Merge remote-tracking branch 'origin/develop' into feature/embeddable-posts

lib/mix/tasks/pleroma/app.ex

@@ -0,0 +1,49 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Mix.Tasks.Pleroma.App do
+  @moduledoc File.read!("docs/administration/CLI_tasks/oauth_app.md")
+  use Mix.Task
+
+  import Mix.Pleroma
+
+  @shortdoc "Creates trusted OAuth App"
+
+  def run(["create" | options]) do
+    start_pleroma()
+
+    {opts, _} =
+      OptionParser.parse!(options,
+        strict: [name: :string, redirect_uri: :string, scopes: :string],
+        aliases: [n: :name, r: :redirect_uri, s: :scopes]
+      )
+
+    scopes =
+      if opts[:scopes] do
+        String.split(opts[:scopes], ",")
+      else
+        ["read", "write", "follow", "push"]
+      end
+
+    params = %{
+      client_name: opts[:name],
+      redirect_uris: opts[:redirect_uri],
+      trusted: true,
+      scopes: scopes
+    }
+
+    with {:ok, app} <- Pleroma.Web.OAuth.App.create(params) do
+      shell_info("#{app.client_name} successfully created:")
+      shell_info("App client_id: " <> app.client_id)
+      shell_info("App client_secret: " <> app.client_secret)
+    else
+      {:error, changeset} ->
+        shell_error("Creating failed:")
+
+        Enum.each(Pleroma.Web.OAuth.App.errors(changeset), fn {key, error} ->
+          shell_error("#{key}: #{error}")
+        end)
+    end
+  end
+end

lib/mix/tasks/pleroma/benchmark.ex

@@ -74,4 +74,43 @@ defmodule Mix.Tasks.Pleroma.Benchmark do
       inputs: inputs
     )
   end
+
+  def run(["adapters"]) do
+    start_pleroma()
+
+    :ok =
+      Pleroma.Gun.Conn.open(
+        "https://httpbin.org/stream-bytes/1500",
+        :gun_connections
+      )
+
+    Process.sleep(1_500)
+
+    Benchee.run(
+      %{
+        "Without conn and without pool" => fn ->
+          {:ok, %Tesla.Env{}} =
+            Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500", [],
+              adapter: [pool: :no_pool, receive_conn: false]
+            )
+        end,
+        "Without conn and with pool" => fn ->
+          {:ok, %Tesla.Env{}} =
+            Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500", [],
+              adapter: [receive_conn: false]
+            )
+        end,
+        "With reused conn and without pool" => fn ->
+          {:ok, %Tesla.Env{}} =
+            Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500", [],
+              adapter: [pool: :no_pool]
+            )
+        end,
+        "With reused conn and with pool" => fn ->
+          {:ok, %Tesla.Env{}} = Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500")
+        end
+      },
+      parallel: 10
+    )
+  end
 end

lib/mix/tasks/pleroma/database.ex

@@ -4,6 +4,7 @@
 
 defmodule Mix.Tasks.Pleroma.Database do
   alias Pleroma.Conversation
+  alias Pleroma.Maintenance
   alias Pleroma.Object
   alias Pleroma.Repo
   alias Pleroma.User
@@ -34,13 +35,7 @@ defmodule Mix.Tasks.Pleroma.Database do
     )
 
     if Keyword.get(options, :vacuum) do
-      Logger.info("Runnning VACUUM FULL")
-
-      Repo.query!(
-        "vacuum full;",
-        [],
-        timeout: :infinity
-      )
+      Maintenance.vacuum("full")
     end
   end
 
@@ -94,13 +89,7 @@ defmodule Mix.Tasks.Pleroma.Database do
     |> Repo.delete_all(timeout: :infinity)
 
     if Keyword.get(options, :vacuum) do
-      Logger.info("Runnning VACUUM FULL")
-
-      Repo.query!(
-        "vacuum full;",
-        [],
-        timeout: :infinity
-      )
+      Maintenance.vacuum("full")
     end
   end
 
@@ -135,4 +124,10 @@ defmodule Mix.Tasks.Pleroma.Database do
     end)
     |> Stream.run()
   end
+
+  def run(["vacuum", args]) do
+    start_pleroma()
+
+    Maintenance.vacuum(args)
+  end
 end

lib/mix/tasks/pleroma/digest.ex

@@ -1,5 +1,6 @@
 defmodule Mix.Tasks.Pleroma.Digest do
   use Mix.Task
+  import Mix.Pleroma
 
   @shortdoc "Manages digest emails"
   @moduledoc File.read!("docs/administration/CLI_tasks/digest.md")
@@ -22,12 +23,10 @@ defmodule Mix.Tasks.Pleroma.Digest do
     with %Swoosh.Email{} = email <- Pleroma.Emails.UserEmail.digest_email(patched_user) do
       {:ok, _} = Pleroma.Emails.Mailer.deliver(email)
 
-      Mix.shell().info("Digest email have been sent to #{nickname} (#{user.email})")
+      shell_info("Digest email have been sent to #{nickname} (#{user.email})")
     else
       _ ->
-        Mix.shell().info(
-          "Cound't find any mentions for #{nickname} since #{last_digest_emailed_at}"
-        )
+        shell_info("Cound't find any mentions for #{nickname} since #{last_digest_emailed_at}")
     end
   end
 end

lib/mix/tasks/pleroma/emoji.ex

@@ -4,18 +4,18 @@
 
 defmodule Mix.Tasks.Pleroma.Emoji do
   use Mix.Task
+  import Mix.Pleroma
 
   @shortdoc "Manages emoji packs"
   @moduledoc File.read!("docs/administration/CLI_tasks/emoji.md")
 
   def run(["ls-packs" | args]) do
-    Mix.Pleroma.start_pleroma()
-    Application.ensure_all_started(:hackney)
+    start_pleroma()
 
     {options, [], []} = parse_global_opts(args)
 
-    manifest =
-      fetch_manifest(if options[:manifest], do: options[:manifest], else: default_manifest())
+    url_or_path = options[:manifest] || default_manifest()
+    manifest = fetch_and_decode(url_or_path)
 
     Enum.each(manifest, fn {name, info} ->
       to_print = [
@@ -36,19 +36,18 @@ defmodule Mix.Tasks.Pleroma.Emoji do
   end
 
   def run(["get-packs" | args]) do
-    Mix.Pleroma.start_pleroma()
-    Application.ensure_all_started(:hackney)
+    start_pleroma()
 
     {options, pack_names, []} = parse_global_opts(args)
 
-    manifest_url = if options[:manifest], do: options[:manifest], else: default_manifest()
+    url_or_path = options[:manifest] || default_manifest()
 
-    manifest = fetch_manifest(manifest_url)
+    manifest = fetch_and_decode(url_or_path)
 
     for pack_name <- pack_names do
       if Map.has_key?(manifest, pack_name) do
         pack = manifest[pack_name]
-        src_url = pack["src"]
+        src = pack["src"]
 
         IO.puts(
           IO.ANSI.format([
@@ -58,11 +57,11 @@ defmodule Mix.Tasks.Pleroma.Emoji do
             :normal,
             " from ",
             :underline,
-            src_url
+            src
           ])
         )
 
-        binary_archive = Tesla.get!(client(), src_url).body
+        {:ok, binary_archive} = fetch(src)
         archive_sha = :crypto.hash(:sha256, binary_archive) |> Base.encode16()
 
         sha_status_text = ["SHA256 of ", :bright, pack_name, :normal, " source file is ", :bright]
@@ -75,8 +74,11 @@ defmodule Mix.Tasks.Pleroma.Emoji do
           raise "Bad SHA256 for #{pack_name}"
         end
 
-        # The url specified in files should be in the same directory
-        files_url = Path.join(Path.dirname(manifest_url), pack["files"])
+        # The location specified in files should be in the same directory
+        files_loc =
+          url_or_path
+          |> Path.dirname()
+          |> Path.join(pack["files"])
 
         IO.puts(
           IO.ANSI.format([
@@ -86,11 +88,11 @@ defmodule Mix.Tasks.Pleroma.Emoji do
             :normal,
             " from ",
             :underline,
-            files_url
+            files_loc
           ])
         )
 
-        files = Tesla.get!(client(), files_url).body |> Jason.decode!()
+        files = fetch_and_decode(files_loc)
 
         IO.puts(IO.ANSI.format(["Unpacking ", :bright, pack_name]))
 
@@ -134,38 +136,51 @@ defmodule Mix.Tasks.Pleroma.Emoji do
     end
   end
 
-  def run(["gen-pack", src]) do
-    Application.ensure_all_started(:hackney)
+  def run(["gen-pack" | args]) do
+    start_pleroma()
 
-    proposed_name = Path.basename(src) |> Path.rootname()
-    name = String.trim(IO.gets("Pack name [#{proposed_name}]: "))
-    # If there's no name, use the default one
-    name = if String.length(name) > 0, do: name, else: proposed_name
-
-    license = String.trim(IO.gets("License: "))
-    homepage = String.trim(IO.gets("Homepage: "))
-    description = String.trim(IO.gets("Description: "))
-
-    proposed_files_name = "#{name}.json"
-    files_name = String.trim(IO.gets("Save file list to [#{proposed_files_name}]: "))
-    files_name = if String.length(files_name) > 0, do: files_name, else: proposed_files_name
-
-    default_exts = [".png", ".gif"]
-    default_exts_str = Enum.join(default_exts, " ")
-
-    exts =
-      String.trim(
-        IO.gets("Emoji file extensions (separated with spaces) [#{default_exts_str}]: ")
+    {opts, [src], []} =
+      OptionParser.parse(
+        args,
+        strict: [
+          name: :string,
+          license: :string,
+          homepage: :string,
+          description: :string,
+          files: :string,
+          extensions: :string
+        ]
       )
 
+    proposed_name = Path.basename(src) |> Path.rootname()
+    name = get_option(opts, :name, "Pack name:", proposed_name)
+    license = get_option(opts, :license, "License:")
+    homepage = get_option(opts, :homepage, "Homepage:")
+    description = get_option(opts, :description, "Description:")
+
+    proposed_files_name = "#{name}_files.json"
+    files_name = get_option(opts, :files, "Save file list to:", proposed_files_name)
+
+    default_exts = [".png", ".gif"]
+
+    custom_exts =
+      get_option(
+        opts,
+        :extensions,
+        "Emoji file extensions (separated with spaces):",
+        Enum.join(default_exts, " ")
+      )
+      |> String.split(" ", trim: true)
+
     exts =
-      if String.length(exts) > 0 do
-        String.split(exts, " ")
-        |> Enum.filter(fn e -> e |> String.trim() |> String.length() > 0 end)
-      else
+      if MapSet.equal?(MapSet.new(default_exts), MapSet.new(custom_exts)) do
         default_exts
+      else
+        custom_exts
       end
 
+    IO.puts("Using #{Enum.join(exts, " ")} extensions")
+
     IO.puts("Downloading the pack and generating SHA256")
 
     binary_archive = Tesla.get!(client(), src).body
@@ -195,14 +210,16 @@ defmodule Mix.Tasks.Pleroma.Emoji do
     IO.puts("""
 
     #{files_name} has been created and contains the list of all found emojis in the pack.
-    Please review the files in the remove those not needed.
+    Please review the files in the pack and remove those not needed.
     """)
 
-    if File.exists?("index.json") do
-      existing_data = File.read!("index.json") |> Jason.decode!()
+    pack_file = "#{name}.json"
+
+    if File.exists?(pack_file) do
+      existing_data = File.read!(pack_file) |> Jason.decode!()
 
       File.write!(
-        "index.json",
+        pack_file,
         Jason.encode!(
           Map.merge(
             existing_data,
@@ -212,24 +229,28 @@ defmodule Mix.Tasks.Pleroma.Emoji do
         )
       )
 
-      IO.puts("index.json file has been update with the #{name} pack")
+      IO.puts("#{pack_file} has been updated with the #{name} pack")
     else
-      File.write!("index.json", Jason.encode!(pack_json, pretty: true))
+      File.write!(pack_file, Jason.encode!(pack_json, pretty: true))
 
-      IO.puts("index.json has been created with the #{name} pack")
+      IO.puts("#{pack_file} has been created with the #{name} pack")
     end
   end
 
-  defp fetch_manifest(from) do
-    Jason.decode!(
-      if String.starts_with?(from, "http") do
-        Tesla.get!(client(), from).body
-      else
-        File.read!(from)
-      end
-    )
+  defp fetch_and_decode(from) do
+    with {:ok, json} <- fetch(from) do
+      Jason.decode!(json)
+    end
   end
 
+  defp fetch("http" <> _ = from) do
+    with {:ok, %{body: body}} <- Tesla.get(client(), from) do
+      {:ok, body}
+    end
+  end
+
+  defp fetch(path), do: File.read(path)
+
   defp parse_global_opts(args) do
     OptionParser.parse(
       args,

lib/mix/tasks/pleroma/instance.ex

@@ -147,6 +147,7 @@ defmodule Mix.Tasks.Pleroma.Instance do
           "What directory should media uploads go in (when using the local uploader)?",
           Pleroma.Config.get([Pleroma.Uploaders.Local, :uploads])
         )
+        |> Path.expand()
 
       static_dir =
         get_option(
@@ -155,6 +156,7 @@ defmodule Mix.Tasks.Pleroma.Instance do
           "What directory should custom public files be read from (custom emojis, frontend bundle overrides, robots.txt, etc.)?",
           Pleroma.Config.get([:instance, :static_dir])
         )
+        |> Path.expand()
 
       Config.put([:instance, :static_dir], static_dir)
 
@@ -204,7 +206,7 @@ defmodule Mix.Tasks.Pleroma.Instance do
       shell_info("Writing the postgres script to #{psql_path}.")
       File.write(psql_path, result_psql)
 
-      write_robots_txt(indexable, template_dir)
+      write_robots_txt(static_dir, indexable, template_dir)
 
       shell_info(
         "\n All files successfully written! Refer to the installation instructions for your platform for next steps."
@@ -224,15 +226,13 @@ defmodule Mix.Tasks.Pleroma.Instance do
     end
   end
 
-  defp write_robots_txt(indexable, template_dir) do
+  defp write_robots_txt(static_dir, indexable, template_dir) do
     robots_txt =
       EEx.eval_file(
         template_dir <> "/robots_txt.eex",
         indexable: indexable
       )
 
-    static_dir = Pleroma.Config.get([:instance, :static_dir], "instance/static/")
-
     unless File.exists?(static_dir) do
       File.mkdir_p!(static_dir)
     end

lib/mix/tasks/pleroma/user.ex

@@ -8,6 +8,8 @@ defmodule Mix.Tasks.Pleroma.User do
   alias Ecto.Changeset
   alias Pleroma.User
   alias Pleroma.UserInviteToken
+  alias Pleroma.Web.ActivityPub.Builder
+  alias Pleroma.Web.ActivityPub.Pipeline
 
   @shortdoc "Manages Pleroma users"
   @moduledoc File.read!("docs/administration/CLI_tasks/user.md")
@@ -96,8 +98,9 @@ defmodule Mix.Tasks.Pleroma.User do
   def run(["rm", nickname]) do
     start_pleroma()
 
-    with %User{local: true} = user <- User.get_cached_by_nickname(nickname) do
-      User.perform(:delete, user)
+    with %User{local: true} = user <- User.get_cached_by_nickname(nickname),
+         {:ok, delete_data, _} <- Builder.delete(user, user.ap_id),
+         {:ok, _delete, _} <- Pipeline.common_pipeline(delete_data, local: true) do
       shell_info("User #{nickname} deleted.")
     else
       _ -> shell_error("No local user #{nickname}")
@@ -141,28 +144,18 @@ defmodule Mix.Tasks.Pleroma.User do
     end
   end
 
-  def run(["unsubscribe", nickname]) do
+  def run(["deactivate", nickname]) do
     start_pleroma()
 
     with %User{} = user <- User.get_cached_by_nickname(nickname) do
       shell_info("Deactivating #{user.nickname}")
       User.deactivate(user)
-
-      user
-      |> User.get_friends()
-      |> Enum.each(fn friend ->
-        user = User.get_cached_by_id(user.id)
-
-        shell_info("Unsubscribing #{friend.nickname} from #{user.nickname}")
-        User.unfollow(user, friend)
-      end)
-
       :timer.sleep(500)
 
       user = User.get_cached_by_id(user.id)
 
-      if Enum.empty?(User.get_friends(user)) do
-        shell_info("Successfully unsubscribed all followers from #{user.nickname}")
+      if Enum.empty?(Enum.filter(User.get_friends(user), & &1.local)) do
+        shell_info("Successfully unsubscribed all local followers from #{user.nickname}")
       end
     else
       _ ->
@@ -170,7 +163,7 @@ defmodule Mix.Tasks.Pleroma.User do
     end
   end
 
-  def run(["unsubscribe_all_from_instance", instance]) do
+  def run(["deactivate_all_from_instance", instance]) do
     start_pleroma()
 
     Pleroma.User.Query.build(%{nickname: "@#{instance}"})
@@ -178,7 +171,7 @@ defmodule Mix.Tasks.Pleroma.User do
     |> Stream.each(fn users ->
       users
       |> Enum.each(fn user ->
-        run(["unsubscribe", user.nickname])
+        run(["deactivate", user.nickname])
       end)
     end)
     |> Stream.run()

lib/pleroma/activity.ex

@@ -27,17 +27,13 @@ defmodule Pleroma.Activity do
   # https://github.com/tootsuite/mastodon/blob/master/app/models/notification.rb#L19
   @mastodon_notification_types %{
     "Create" => "mention",
-    "Follow" => "follow",
+    "Follow" => ["follow", "follow_request"],
     "Announce" => "reblog",
     "Like" => "favourite",
     "Move" => "move",
     "EmojiReact" => "pleroma:emoji_reaction"
   }
 
-  @mastodon_to_ap_notification_types for {k, v} <- @mastodon_notification_types,
-                                         into: %{},
-                                         do: {v, k}
-
   schema "activities" do
     field(:data, :map)
     field(:local, :boolean, default: true)
@@ -95,6 +91,17 @@ defmodule Pleroma.Activity do
     |> preload([activity, object: object], object: object)
   end
 
+  # Note: applies to fake activities (ActivityPub.Utils.get_notified_from_object/1 etc.)
+  def user_actor(%Activity{actor: nil}), do: nil
+
+  def user_actor(%Activity{} = activity) do
+    with %User{} <- activity.user_actor do
+      activity.user_actor
+    else
+      _ -> User.get_cached_by_ap_id(activity.actor)
+    end
+  end
+
   def with_joined_user_actor(query, join_type \\ :inner) do
     join(query, join_type, [activity], u in User,
       on: u.ap_id == activity.actor,
@@ -280,15 +287,43 @@ defmodule Pleroma.Activity do
 
   defp purge_web_resp_cache(nil), do: nil
 
-  for {ap_type, type} <- @mastodon_notification_types do
+  def follow_accepted?(
+        %Activity{data: %{"type" => "Follow", "object" => followed_ap_id}} = activity
+      ) do
+    with %User{} = follower <- Activity.user_actor(activity),
+         %User{} = followed <- User.get_cached_by_ap_id(followed_ap_id) do
+      Pleroma.FollowingRelationship.following?(follower, followed)
+    else
+      _ -> false
+    end
+  end
+
+  def follow_accepted?(_), do: false
+
+  @spec mastodon_notification_type(Activity.t()) :: String.t() | nil
+
+  for {ap_type, type} <- @mastodon_notification_types, not is_list(type) do
     def mastodon_notification_type(%Activity{data: %{"type" => unquote(ap_type)}}),
       do: unquote(type)
   end
 
+  def mastodon_notification_type(%Activity{data: %{"type" => "Follow"}} = activity) do
+    if follow_accepted?(activity) do
+      "follow"
+    else
+      "follow_request"
+    end
+  end
+
   def mastodon_notification_type(%Activity{}), do: nil
 
+  @spec from_mastodon_notification_type(String.t()) :: String.t() | nil
+  @doc "Converts Mastodon notification type to AR activity type"
   def from_mastodon_notification_type(type) do
-    Map.get(@mastodon_to_ap_notification_types, type)
+    with {k, _v} <-
+           Enum.find(@mastodon_notification_types, fn {_k, v} -> type in List.wrap(v) end) do
+      k
+    end
   end
 
   def all_by_actor_and_id(actor, status_ids \\ [])

lib/pleroma/activity/queries.ex

@@ -24,10 +24,7 @@ defmodule Pleroma.Activity.Queries do
 
   @spec by_actor(query, String.t()) :: query
   def by_actor(query \\ Activity, actor) do
-    from(
-      activity in query,
-      where: fragment("(?)->>'actor' = ?", activity.data, ^actor)
-    )
+    from(a in query, where: a.actor == ^actor)
   end
 
   @spec by_author(query, User.t()) :: query
@@ -35,6 +32,13 @@ defmodule Pleroma.Activity.Queries do
     from(a in query, where: a.actor == ^ap_id)
   end
 
+  def find_by_object_ap_id(activities, object_ap_id) do
+    Enum.find(
+      activities,
+      &(object_ap_id in [is_map(&1.data["object"]) && &1.data["object"]["id"], &1.data["object"]])
+    )
+  end
+
   @spec by_object_id(query, String.t() | [String.t()]) :: query
   def by_object_id(query \\ Activity, object_id)
 

lib/pleroma/application.ex

@@ -3,8 +3,12 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Application do
-  import Cachex.Spec
   use Application
+
+  import Cachex.Spec
+
+  alias Pleroma.Config
+
   require Logger
 
   @name Mix.Project.config()[:name]
@@ -18,9 +22,9 @@ defmodule Pleroma.Application do
   def repository, do: @repository
 
   def user_agent do
-    case Pleroma.Config.get([:http, :user_agent], :default) do
+    case Config.get([:http, :user_agent], :default) do
       :default ->
-        info = "#{Pleroma.Web.base_url()} <#{Pleroma.Config.get([:instance, :email], "")}>"
+        info = "#{Pleroma.Web.base_url()} <#{Config.get([:instance, :email], "")}>"
         named_version() <> "; " <> info
 
       custom ->
@@ -33,27 +37,50 @@ defmodule Pleroma.Application do
   def start(_type, _args) do
     Pleroma.Config.Holder.save_default()
     Pleroma.HTML.compile_scrubbers()
-    Pleroma.Config.DeprecationWarnings.warn()
+    Config.DeprecationWarnings.warn()
     Pleroma.Plugs.HTTPSecurityPlug.warn_if_disabled()
     Pleroma.Repo.check_migrations_applied!()
     setup_instrumenters()
     load_custom_modules()
 
+    adapter = Application.get_env(:tesla, :adapter)
+
+    if adapter == Tesla.Adapter.Gun do
+      if version = Pleroma.OTPVersion.version() do
+        [major, minor] =
+          version
+          |> String.split(".")
+          |> Enum.map(&String.to_integer/1)
+          |> Enum.take(2)
+
+        if (major == 22 and minor < 2) or major < 22 do
+          raise "
+            !!!OTP VERSION WARNING!!!
+            You are using gun adapter with OTP version #{version}, which doesn't support correct handling of unordered certificates chains. Please update your Erlang/OTP to at least 22.2.
+            "
+        end
+      else
+        raise "
+          !!!OTP VERSION WARNING!!!
+          To support correct handling of unordered certificates chains - OTP version must be > 22.2.
+          "
+      end
+    end
+
     # Define workers and child supervisors to be supervised
     children =
       [
         Pleroma.Repo,
-        Pleroma.Config.TransferTask,
+        Config.TransferTask,
         Pleroma.Emoji,
-        Pleroma.Captcha,
         Pleroma.Plugs.RateLimiter.Supervisor
       ] ++
         cachex_children() ++
-        hackney_pool_children() ++
+        http_children(adapter, @env) ++
         [
           Pleroma.Stats,
           Pleroma.JobQueueMonitor,
-          {Oban, Pleroma.Config.get(Oban)}
+          {Oban, Config.get(Oban)}
         ] ++
         task_children(@env) ++
         streamer_child(@env) ++
@@ -70,7 +97,7 @@ defmodule Pleroma.Application do
   end
 
   def load_custom_modules do
-    dir = Pleroma.Config.get([:modules, :runtime_dir])
+    dir = Config.get([:modules, :runtime_dir])
 
     if dir && File.exists?(dir) do
       dir
@@ -111,20 +138,6 @@ defmodule Pleroma.Application do
     Pleroma.Web.Endpoint.Instrumenter.setup()
   end
 
-  def enabled_hackney_pools do
-    [:media] ++
-      if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Hackney do
-        [:federation]
-      else
-        []
-      end ++
-      if Pleroma.Config.get([Pleroma.Upload, :proxy_remote]) do
-        [:upload]
-      else
-        []
-      end
-  end
-
   defp cachex_children do
     [
       build_cachex("used_captcha", ttl_interval: seconds_valid_interval()),
@@ -146,7 +159,7 @@ defmodule Pleroma.Application do
     do: expiration(default: :timer.seconds(6 * 60 * 60), interval: :timer.seconds(60))
 
   defp seconds_valid_interval,
-    do: :timer.seconds(Pleroma.Config.get!([Pleroma.Captcha, :seconds_valid]))
+    do: :timer.seconds(Config.get!([Pleroma.Captcha, :seconds_valid]))
 
   defp build_cachex(type, opts),
     do: %{
@@ -155,12 +168,19 @@ defmodule Pleroma.Application do
       type: :worker
     }
 
-  defp chat_enabled?, do: Pleroma.Config.get([:chat, :enabled])
+  defp chat_enabled?, do: Config.get([:chat, :enabled])
 
-  defp streamer_child(:test), do: []
+  defp streamer_child(env) when env in [:test, :benchmark], do: []
 
   defp streamer_child(_) do
-    [Pleroma.Web.Streamer.supervisor()]
+    [
+      {Registry,
+       [
+         name: Pleroma.Web.Streamer.registry(),
+         keys: :duplicate,
+         partitions: System.schedulers_online()
+       ]}
+    ]
   end
 
   defp chat_child(_env, true) do
@@ -169,13 +189,6 @@ defmodule Pleroma.Application do
 
   defp chat_child(_, _), do: []
 
-  defp hackney_pool_children do
-    for pool <- enabled_hackney_pools() do
-      options = Pleroma.Config.get([:hackney_pools, pool])
-      :hackney_pool.child_spec(pool, options)
-    end
-  end
-
   defp task_children(:test) do
     [
       %{
@@ -200,4 +213,31 @@ defmodule Pleroma.Application do
       }
     ]
   end
+
+  # start hackney and gun pools in tests
+  defp http_children(_, :test) do
+    hackney_options = Config.get([:hackney_pools, :federation])
+    hackney_pool = :hackney_pool.child_spec(:federation, hackney_options)
+    [hackney_pool, Pleroma.Pool.Supervisor]
+  end
+
+  defp http_children(Tesla.Adapter.Hackney, _) do
+    pools = [:federation, :media]
+
+    pools =
+      if Config.get([Pleroma.Upload, :proxy_remote]) do
+        [:upload | pools]
+      else
+        pools
+      end
+
+    for pool <- pools do
+      options = Config.get([:hackney_pools, pool])
+      :hackney_pool.child_spec(pool, options)
+    end
+  end
+
+  defp http_children(Tesla.Adapter.Gun, _), do: [Pleroma.Pool.Supervisor]
+
+  defp http_children(_, _), do: []
 end

lib/pleroma/bbs/authenticator.ex

@@ -4,7 +4,7 @@
 
 defmodule Pleroma.BBS.Authenticator do
   use Sshd.PasswordAuthenticator
-  alias Comeonin.Pbkdf2
+  alias Pleroma.Plugs.AuthenticationPlug
   alias Pleroma.User
 
   def authenticate(username, password) do
@@ -12,7 +12,7 @@ defmodule Pleroma.BBS.Authenticator do
     password = to_string(password)
 
     with %User{} = user <- User.get_by_nickname(username) do
-      Pbkdf2.checkpw(password, user.password_hash)
+      AuthenticationPlug.checkpw(password, user.password_hash)
     else
       _e -> false
     end

lib/pleroma/bbs/handler.ex

@@ -66,7 +66,7 @@ defmodule Pleroma.BBS.Handler do
 
     with %Activity{} <- Activity.get_by_id(activity_id),
          {:ok, _activity} <-
-           CommonAPI.post(user, %{"status" => rest, "in_reply_to_status_id" => activity_id}) do
+           CommonAPI.post(user, %{status: rest, in_reply_to_status_id: activity_id}) do
       IO.puts("Replied!")
     else
       _e -> IO.puts("Could not reply...")
@@ -78,7 +78,7 @@ defmodule Pleroma.BBS.Handler do
   def handle_command(%{user: user} = state, "p " <> text) do
     text = String.trim(text)
 
-    with {:ok, _activity} <- CommonAPI.post(user, %{"status" => text}) do
+    with {:ok, _activity} <- CommonAPI.post(user, %{status: text}) do
       IO.puts("Posted!")
     else
       _e -> IO.puts("Could not post...")

lib/pleroma/captcha/captcha.ex

@@ -3,53 +3,22 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Captcha do
-  import Pleroma.Web.Gettext
-
   alias Calendar.DateTime
   alias Plug.Crypto.KeyGenerator
   alias Plug.Crypto.MessageEncryptor
 
-  use GenServer
-
-  @doc false
-  def start_link(_) do
-    GenServer.start_link(__MODULE__, [], name: __MODULE__)
-  end
-
-  @doc false
-  def init(_) do
-    {:ok, nil}
-  end
-
   @doc """
   Ask the configured captcha service for a new captcha
   """
   def new do
-    GenServer.call(__MODULE__, :new)
-  end
-
-  @doc """
-  Ask the configured captcha service to validate the captcha
-  """
-  def validate(token, captcha, answer_data) do
-    GenServer.call(__MODULE__, {:validate, token, captcha, answer_data})
-  end
-
-  @doc false
-  def handle_call(:new, _from, state) do
-    enabled = Pleroma.Config.get([__MODULE__, :enabled])
-
-    if !enabled do
-      {:reply, %{type: :none}, state}
+    if not enabled?() do
+      %{type: :none}
     else
       new_captcha = method().new()
 
-      secret_key_base = Pleroma.Config.get!([Pleroma.Web.Endpoint, :secret_key_base])
-
       # This make salt a little different for two keys
-      token = new_captcha[:token]
-      secret = KeyGenerator.generate(secret_key_base, token <> "_encrypt")
-      sign_secret = KeyGenerator.generate(secret_key_base, token <> "_sign")
+      {secret, sign_secret} = secret_pair(new_captcha[:token])
+
       # Basically copy what Phoenix.Token does here, add the time to
       # the actual data and make it a binary to then encrypt it
       encrypted_captcha_answer =
@@ -60,55 +29,73 @@ defmodule Pleroma.Captcha do
         |> :erlang.term_to_binary()
         |> MessageEncryptor.encrypt(secret, sign_secret)
 
-      {
-        :reply,
-        # Replace the answer with the encrypted answer
-        %{new_captcha | answer_data: encrypted_captcha_answer},
-        state
-      }
+      # Replace the answer with the encrypted answer
+      %{new_captcha | answer_data: encrypted_captcha_answer}
     end
   end
 
-  @doc false
-  def handle_call({:validate, token, captcha, answer_data}, _from, state) do
+  @doc """
+  Ask the configured captcha service to validate the captcha
+  """
+  def validate(token, captcha, answer_data) do
+    with {:ok, %{at: at, answer_data: answer_md5}} <- validate_answer_data(token, answer_data),
+         :ok <- validate_expiration(at),
+         :ok <- validate_usage(token),
+         :ok <- method().validate(token, captcha, answer_md5),
+         {:ok, _} <- mark_captcha_as_used(token) do
+      :ok
+    end
+  end
+
+  def enabled?, do: Pleroma.Config.get([__MODULE__, :enabled], false)
+
+  defp seconds_valid, do: Pleroma.Config.get!([__MODULE__, :seconds_valid])
+
+  defp secret_pair(token) do
     secret_key_base = Pleroma.Config.get!([Pleroma.Web.Endpoint, :secret_key_base])
     secret = KeyGenerator.generate(secret_key_base, token <> "_encrypt")
     sign_secret = KeyGenerator.generate(secret_key_base, token <> "_sign")
 
+    {secret, sign_secret}
+  end
+
+  defp validate_answer_data(token, answer_data) do
+    {secret, sign_secret} = secret_pair(token)
+
+    with false <- is_nil(answer_data),
+         {:ok, data} <- MessageEncryptor.decrypt(answer_data, secret, sign_secret),
+         %{at: at, answer_data: answer_md5} <- :erlang.binary_to_term(data) do
+      {:ok, %{at: at, answer_data: answer_md5}}
+    else
+      _ -> {:error, :invalid_answer_data}
+    end
+  end
+
+  defp validate_expiration(created_at) do
     # If the time found is less than (current_time-seconds_valid) then the time has already passed
     # Later we check that the time found is more than the presumed invalidatation time, that means
     # that the data is still valid and the captcha can be checked
-    seconds_valid = Pleroma.Config.get!([Pleroma.Captcha, :seconds_valid])
-    valid_if_after = DateTime.subtract!(DateTime.now_utc(), seconds_valid)
 
-    result =
-      with false <- is_nil(answer_data),
-           {:ok, data} <- MessageEncryptor.decrypt(answer_data, secret, sign_secret),
-           %{at: at, answer_data: answer_md5} <- :erlang.binary_to_term(data) do
-        try do
-          if DateTime.before?(at, valid_if_after),
-            do: throw({:error, dgettext("errors", "CAPTCHA expired")})
+    valid_if_after = DateTime.subtract!(DateTime.now_utc(), seconds_valid())
 
-          if not is_nil(Cachex.get!(:used_captcha_cache, token)),
-            do: throw({:error, dgettext("errors", "CAPTCHA already used")})
+    if DateTime.before?(created_at, valid_if_after) do
+      {:error, :expired}
+    else
+      :ok
+    end
+  end
 
-          res = method().validate(token, captcha, answer_md5)
-          # Throw if an error occurs
-          if res != :ok, do: throw(res)
+  defp validate_usage(token) do
+    if is_nil(Cachex.get!(:used_captcha_cache, token)) do
+      :ok
+    else
+      {:error, :already_used}
+    end
+  end
 
-          # Mark this captcha as used
-          {:ok, _} =
-            Cachex.put(:used_captcha_cache, token, true, ttl: :timer.seconds(seconds_valid))
-
-          :ok
-        catch
-          :throw, e -> e
-        end
-      else
-        _ -> {:error, dgettext("errors", "Invalid answer data")}
-      end
-
-    {:reply, result, state}
+  defp mark_captcha_as_used(token) do
+    ttl = seconds_valid() |> :timer.seconds()
+    Cachex.put(:used_captcha_cache, token, true, ttl: ttl)
   end
 
   defp method, do: Pleroma.Config.get!([__MODULE__, :method])

lib/pleroma/captcha/kocaptcha.ex

@@ -3,7 +3,6 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Captcha.Kocaptcha do
-  import Pleroma.Web.Gettext
   alias Pleroma.Captcha.Service
   @behaviour Service
 
@@ -13,7 +12,7 @@ defmodule Pleroma.Captcha.Kocaptcha do
 
     case Tesla.get(endpoint <> "/new") do
       {:error, _} ->
-        %{error: dgettext("errors", "Kocaptcha service unavailable")}
+        %{error: :kocaptcha_service_unavailable}
 
       {:ok, res} ->
         json_resp = Jason.decode!(res.body)
@@ -33,6 +32,6 @@ defmodule Pleroma.Captcha.Kocaptcha do
     if not is_nil(captcha) and
          :crypto.hash(:md5, captcha) |> Base.encode16() == String.upcase(answer_data),
        do: :ok,
-       else: {:error, dgettext("errors", "Invalid CAPTCHA")}
+       else: {:error, :invalid}
   end
 end

lib/pleroma/captcha/native.ex

@@ -3,7 +3,6 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Captcha.Native do
-  import Pleroma.Web.Gettext
   alias Pleroma.Captcha.Service
   @behaviour Service
 
@@ -11,7 +10,7 @@ defmodule Pleroma.Captcha.Native do
   def new do
     case Captcha.get() do
       :error ->
-        %{error: dgettext("errors", "Captcha error")}
+        %{error: :captcha_error}
 
       {:ok, answer_data, img_binary} ->
         %{
@@ -25,7 +24,7 @@ defmodule Pleroma.Captcha.Native do
 
   @impl Service
   def validate(_token, captcha, captcha) when not is_nil(captcha), do: :ok
-  def validate(_token, _captcha, _answer), do: {:error, dgettext("errors", "Invalid CAPTCHA")}
+  def validate(_token, _captcha, _answer), do: {:error, :invalid}
 
   defp token do
     10

lib/pleroma/config/loader.ex

@@ -47,7 +47,7 @@ defmodule Pleroma.Config.Loader do
   @spec filter_group(atom(), keyword()) :: keyword()
   def filter_group(group, configs) do
     Enum.reject(configs[group], fn {key, _v} ->
-      key in @reject_keys or (group == :phoenix and key == :serve_endpoints)
+      key in @reject_keys or (group == :phoenix and key == :serve_endpoints) or group == :postgrex
     end)
   end
 end

lib/pleroma/config/transfer_task.ex

@@ -5,6 +5,7 @@
 defmodule Pleroma.Config.TransferTask do
   use Task
 
+  alias Pleroma.Config
   alias Pleroma.ConfigDB
   alias Pleroma.Repo
 
@@ -18,7 +19,9 @@ defmodule Pleroma.Config.TransferTask do
     {:pleroma, Oban},
     {:pleroma, :rate_limit},
     {:pleroma, :markup},
-    {:plerome, :streamer}
+    {:pleroma, :streamer},
+    {:pleroma, :pools},
+    {:pleroma, :connections_pool}
   ]
 
   @reboot_time_subkeys [
@@ -32,45 +35,44 @@ defmodule Pleroma.Config.TransferTask do
     {:pleroma, :gopher, [:enabled]}
   ]
 
-  @reject [nil, :prometheus]
-
   def start_link(_) do
     load_and_update_env()
-    if Pleroma.Config.get(:env) == :test, do: Ecto.Adapters.SQL.Sandbox.checkin(Repo)
+    if Config.get(:env) == :test, do: Ecto.Adapters.SQL.Sandbox.checkin(Repo)
     :ignore
   end
 
-  @spec load_and_update_env([ConfigDB.t()]) :: :ok | false
-  def load_and_update_env(deleted \\ [], restart_pleroma? \\ true) do
-    with {:configurable, true} <-
-           {:configurable, Pleroma.Config.get(:configurable_from_database)},
-         true <- Ecto.Adapters.SQL.table_exists?(Repo, "config"),
-         started_applications <- Application.started_applications() do
+  @spec load_and_update_env([ConfigDB.t()], boolean()) :: :ok
+  def load_and_update_env(deleted_settings \\ [], restart_pleroma? \\ true) do
+    with {_, true} <- {:configurable, Config.get(:configurable_from_database)} do
       # We need to restart applications for loaded settings take effect
 
-      in_db = Repo.all(ConfigDB)
+      {logger, other} =
+        (Repo.all(ConfigDB) ++ deleted_settings)
+        |> Enum.map(&transform_and_merge/1)
+        |> Enum.split_with(fn {group, _, _, _} -> group in [:logger, :quack] end)
 
-      with_deleted = in_db ++ deleted
+      logger
+      |> Enum.sort()
+      |> Enum.each(&configure/1)
 
-      reject_for_restart = if restart_pleroma?, do: @reject, else: [:pleroma | @reject]
+      started_applications = Application.started_applications()
 
-      applications =
-        with_deleted
-        |> Enum.map(&merge_and_update(&1))
-        |> Enum.uniq()
-        # TODO: some problem with prometheus after restart!
-        |> Enum.reject(&(&1 in reject_for_restart))
+      # TODO: some problem with prometheus after restart!
+      reject = [nil, :prometheus, :postgrex]
 
-      # to be ensured that pleroma will be restarted last
-      applications =
-        if :pleroma in applications do
-          List.delete(applications, :pleroma) ++ [:pleroma]
+      reject =
+        if restart_pleroma? do
+          reject
         else
-          Restarter.Pleroma.rebooted()
-          applications
+          [:pleroma | reject]
         end
 
-      Enum.each(applications, &restart(started_applications, &1, Pleroma.Config.get(:env)))
+      other
+      |> Enum.map(&update/1)
+      |> Enum.uniq()
+      |> Enum.reject(&(&1 in reject))
+      |> maybe_set_pleroma_last()
+      |> Enum.each(&restart(started_applications, &1, Config.get(:env)))
 
       :ok
     else
@@ -78,51 +80,83 @@ defmodule Pleroma.Config.TransferTask do
     end
   end
 
-  defp merge_and_update(setting) do
-    try do
-      key = ConfigDB.from_string(setting.key)
-      group = ConfigDB.from_string(setting.group)
+  defp maybe_set_pleroma_last(apps) do
+    # to be ensured that pleroma will be restarted last
+    if :pleroma in apps do
+      apps
+      |> List.delete(:pleroma)
+      |> List.insert_at(-1, :pleroma)
+    else
+      Restarter.Pleroma.rebooted()
+      apps
+    end
+  end
 
-      default = Pleroma.Config.Holder.default_config(group, key)
-      value = ConfigDB.from_binary(setting.value)
+  defp transform_and_merge(%{group: group, key: key, value: value} = setting) do
+    group = ConfigDB.from_string(group)
+    key = ConfigDB.from_string(key)
+    value = ConfigDB.from_binary(value)
 
-      merged_value =
-        if Ecto.get_meta(setting, :state) == :deleted do
-          default
-        else
-          if can_be_merged?(default, value) do
-            ConfigDB.merge_group(group, key, default, value)
-          else
-            value
-          end
-        end
+    default = Config.Holder.default_config(group, key)
 
-      :ok = update_env(group, key, merged_value)
-
-      if group != :logger do
-        if group != :pleroma or pleroma_need_restart?(group, key, value) do
-          group
-        end
-      else
-        # change logger configuration in runtime, without restart
-        if Keyword.keyword?(merged_value) and
-             key not in [:compile_time_application, :backends, :compile_time_purge_matching] do
-          Logger.configure_backend(key, merged_value)
-        else
-          Logger.configure([{key, merged_value}])
-        end
-
-        nil
+    merged =
+      cond do
+        Ecto.get_meta(setting, :state) == :deleted -> default
+        can_be_merged?(default, value) -> ConfigDB.merge_group(group, key, default, value)
+        true -> value
       end
+
+    {group, key, value, merged}
+  end
+
+  # change logger configuration in runtime, without restart
+  defp configure({:quack, key, _, merged}) do
+    Logger.configure_backend(Quack.Logger, [{key, merged}])
+    :ok = update_env(:quack, key, merged)
+  end
+
+  defp configure({_, :backends, _, merged}) do
+    # removing current backends
+    Enum.each(Application.get_env(:logger, :backends), &Logger.remove_backend/1)
+
+    Enum.each(merged, &Logger.add_backend/1)
+
+    :ok = update_env(:logger, :backends, merged)
+  end
+
+  defp configure({_, key, _, merged}) when key in [:console, :ex_syslogger] do
+    merged =
+      if key == :console do
+        put_in(merged[:format], merged[:format] <> "\n")
+      else
+        merged
+      end
+
+    backend =
+      if key == :ex_syslogger,
+        do: {ExSyslogger, :ex_syslogger},
+        else: key
+
+    Logger.configure_backend(backend, merged)
+    :ok = update_env(:logger, key, merged)
+  end
+
+  defp configure({_, key, _, merged}) do
+    Logger.configure([{key, merged}])
+    :ok = update_env(:logger, key, merged)
+  end
+
+  defp update({group, key, value, merged}) do
+    try do
+      :ok = update_env(group, key, merged)
+
+      if group != :pleroma or pleroma_need_restart?(group, key, value), do: group
     rescue
       error ->
         error_msg =
-          "updating env causes error, group: " <>
-            inspect(setting.group) <>
-            " key: " <>
-            inspect(setting.key) <>
-            " value: " <>
-            inspect(ConfigDB.from_binary(setting.value)) <> " error: " <> inspect(error)
+          "updating env causes error, group: #{inspect(group)}, key: #{inspect(key)}, value: #{
+            inspect(value)
+          } error: #{inspect(error)}"
 
         Logger.warn(error_msg)
 
@@ -130,6 +164,9 @@ defmodule Pleroma.Config.TransferTask do
     end
   end
 
+  defp update_env(group, key, nil), do: Application.delete_env(group, key)
+  defp update_env(group, key, value), do: Application.put_env(group, key, value)
+
   @spec pleroma_need_restart?(atom(), atom(), any()) :: boolean()
   def pleroma_need_restart?(group, key, value) do
     group_and_key_need_reboot?(group, key) or group_and_subkey_need_reboot?(group, key, value)
@@ -147,9 +184,6 @@ defmodule Pleroma.Config.TransferTask do
       end)
   end
 
-  defp update_env(group, key, nil), do: Application.delete_env(group, key)
-  defp update_env(group, key, value), do: Application.put_env(group, key, value)
-
   defp restart(_, :pleroma, env), do: Restarter.Pleroma.restart_after_boot(env)
 
   defp restart(started_applications, app, _) do

lib/pleroma/constants.ex

@@ -17,7 +17,13 @@ defmodule Pleroma.Constants do
       "announcement_count",
       "emoji",
       "context_id",
-      "deleted_activity_id"
+      "deleted_activity_id",
+      "pleroma_internal"
     ]
   )
+
+  const(static_only_files,
+    do:
+      ~w(index.html robots.txt static static-fe finmoji emoji packs sounds images instance sw.js sw-pleroma.js favicon.png schemas doc embed.js embed.css)
+  )
 end

lib/pleroma/conversation.ex

@@ -63,7 +63,7 @@ defmodule Pleroma.Conversation do
          ap_id when is_binary(ap_id) and byte_size(ap_id) > 0 <- object.data["context"] do
       {:ok, conversation} = create_for_ap_id(ap_id)
 
-      users = User.get_users_from_set(activity.recipients, false)
+      users = User.get_users_from_set(activity.recipients, local_only: false)
 
       participations =
         Enum.map(users, fn user ->

lib/pleroma/conversation/participation.ex

@@ -128,22 +128,19 @@ defmodule Pleroma.Conversation.Participation do
     |> Pleroma.Pagination.fetch_paginated(params)
   end
 
-  def restrict_recipients(query, user, %{"recipients" => user_ids}) do
-    user_ids =
+  def restrict_recipients(query, user, %{recipients: user_ids}) do
+    user_binary_ids =
       [user.id | user_ids]
       |> Enum.uniq()
-      |> Enum.reduce([], fn user_id, acc ->
-        {:ok, user_id} = FlakeId.Ecto.CompatType.dump(user_id)
-        [user_id | acc]
-      end)
+      |> User.binary_id()
 
     conversation_subquery =
       __MODULE__
       |> group_by([p], p.conversation_id)
       |> having(
         [p],
-        count(p.user_id) == ^length(user_ids) and
-          fragment("array_agg(?) @> ?", p.user_id, ^user_ids)
+        count(p.user_id) == ^length(user_binary_ids) and
+          fragment("array_agg(?) @> ?", p.user_id, ^user_binary_ids)
       )
       |> select([p], %{id: p.conversation_id})
 
@@ -175,7 +172,7 @@ defmodule Pleroma.Conversation.Participation do
         | last_activity_id: activity_id
       }
     end)
-    |> Enum.filter(& &1.last_activity_id)
+    |> Enum.reject(&is_nil(&1.last_activity_id))
   end
 
   def get(_, _ \\ [])

lib/pleroma/docs/json.ex

@@ -18,7 +18,6 @@ defmodule Pleroma.Docs.JSON do
     with config <- Pleroma.Config.Loader.read("config/description.exs") do
       config[:pleroma][:config_description]
       |> Pleroma.Docs.Generator.convert_to_strings()
-      |> Jason.encode!()
     end
   end
 end

lib/pleroma/ecto_enums.ex

@@ -4,10 +4,16 @@
 
 import EctoEnum
 
-defenum(UserRelationshipTypeEnum,
+defenum(Pleroma.UserRelationship.Type,
   block: 1,
   mute: 2,
   reblog_mute: 3,
   notification_mute: 4,
   inverse_subscription: 5
 )
+
+defenum(Pleroma.FollowingRelationship.State,
+  follow_pending: 1,
+  follow_accept: 2,
+  follow_reject: 3
+)

lib/pleroma/emails/new_users_digest_email.ex

@@ -14,8 +14,10 @@ defmodule Pleroma.Emails.NewUsersDigestEmail do
     styling = Pleroma.Config.get([Pleroma.Emails.UserEmail, :styling])
 
     logo_url =
-      Pleroma.Web.Endpoint.url() <>
-        Pleroma.Config.get([:frontend_configurations, :pleroma_fe, :logo])
+      Pleroma.Helpers.UriHelper.maybe_add_base(
+        Pleroma.Config.get([:frontend_configurations, :pleroma_fe, :logo]),
+        Pleroma.Web.Endpoint.url()
+      )
 
     new()
     |> to({to.name, to.email})

lib/pleroma/emoji/formatter.ex

@@ -38,22 +38,14 @@ defmodule Pleroma.Emoji.Formatter do
 
   def demojify(text, nil), do: text
 
-  @doc "Outputs a list of the emoji-shortcodes in a text"
-  def get_emoji(text) when is_binary(text) do
-    Enum.filter(Emoji.get_all(), fn {emoji, %Emoji{}} ->
-      String.contains?(text, ":#{emoji}:")
-    end)
-  end
-
-  def get_emoji(_), do: []
-
   @doc "Outputs a list of the emoji-Maps in a text"
   def get_emoji_map(text) when is_binary(text) do
-    get_emoji(text)
+    Emoji.get_all()
+    |> Enum.filter(fn {emoji, %Emoji{}} -> String.contains?(text, ":#{emoji}:") end)
     |> Enum.reduce(%{}, fn {name, %Emoji{file: file}}, acc ->
       Map.put(acc, name, "#{Pleroma.Web.Endpoint.static_url()}#{file}")
     end)
   end
 
-  def get_emoji_map(_), do: []
+  def get_emoji_map(_), do: %{}
 end

lib/pleroma/emoji/pack.ex

@@ -0,0 +1,541 @@
+defmodule Pleroma.Emoji.Pack do
+  @derive {Jason.Encoder, only: [:files, :pack]}
+  defstruct files: %{},
+            pack_file: nil,
+            path: nil,
+            pack: %{},
+            name: nil
+
+  @type t() :: %__MODULE__{
+          files: %{String.t() => Path.t()},
+          pack_file: Path.t(),
+          path: Path.t(),
+          pack: map(),
+          name: String.t()
+        }
+
+  alias Pleroma.Emoji
+
+  @spec create(String.t()) :: :ok | {:error, File.posix()} | {:error, :empty_values}
+  def create(name) do
+    with :ok <- validate_not_empty([name]),
+         dir <- Path.join(emoji_path(), name),
+         :ok <- File.mkdir(dir) do
+      %__MODULE__{pack_file: Path.join(dir, "pack.json")}
+      |> save_pack()
+    end
+  end
+
+  @spec show(String.t()) :: {:ok, t()} | {:error, atom()}
+  def show(name) do
+    with :ok <- validate_not_empty([name]),
+         {:ok, pack} <- load_pack(name) do
+      {:ok, validate_pack(pack)}
+    end
+  end
+
+  @spec delete(String.t()) ::
+          {:ok, [binary()]} | {:error, File.posix(), binary()} | {:error, :empty_values}
+  def delete(name) do
+    with :ok <- validate_not_empty([name]) do
+      emoji_path()
+      |> Path.join(name)
+      |> File.rm_rf()
+    end
+  end
+
+  @spec add_file(String.t(), String.t(), Path.t(), Plug.Upload.t() | String.t()) ::
+          {:ok, t()} | {:error, File.posix() | atom()}
+  def add_file(name, shortcode, filename, file) do
+    with :ok <- validate_not_empty([name, shortcode, filename]),
+         :ok <- validate_emoji_not_exists(shortcode),
+         {:ok, pack} <- load_pack(name),
+         :ok <- save_file(file, pack, filename),
+         {:ok, updated_pack} <- pack |> put_emoji(shortcode, filename) |> save_pack() do
+      Emoji.reload()
+      {:ok, updated_pack}
+    end
+  end
+
+  @spec delete_file(String.t(), String.t()) ::
+          {:ok, t()} | {:error, File.posix() | atom()}
+  def delete_file(name, shortcode) do
+    with :ok <- validate_not_empty([name, shortcode]),
+         {:ok, pack} <- load_pack(name),
+         :ok <- remove_file(pack, shortcode),
+         {:ok, updated_pack} <- pack |> delete_emoji(shortcode) |> save_pack() do
+      Emoji.reload()
+      {:ok, updated_pack}
+    end
+  end
+
+  @spec update_file(String.t(), String.t(), String.t(), String.t(), boolean()) ::
+          {:ok, t()} | {:error, File.posix() | atom()}
+  def update_file(name, shortcode, new_shortcode, new_filename, force) do
+    with :ok <- validate_not_empty([name, shortcode, new_shortcode, new_filename]),
+         {:ok, pack} <- load_pack(name),
+         {:ok, filename} <- get_filename(pack, shortcode),
+         :ok <- validate_emoji_not_exists(new_shortcode, force),
+         :ok <- rename_file(pack, filename, new_filename),
+         {:ok, updated_pack} <-
+           pack
+           |> delete_emoji(shortcode)
+           |> put_emoji(new_shortcode, new_filename)
+           |> save_pack() do
+      Emoji.reload()
+      {:ok, updated_pack}
+    end
+  end
+
+  @spec import_from_filesystem() :: {:ok, [String.t()]} | {:error, File.posix() | atom()}
+  def import_from_filesystem do
+    emoji_path = emoji_path()
+
+    with {:ok, %{access: :read_write}} <- File.stat(emoji_path),
+         {:ok, results} <- File.ls(emoji_path) do
+      names =
+        results
+        |> Enum.map(&Path.join(emoji_path, &1))
+        |> Enum.reject(fn path ->
+          File.dir?(path) and File.exists?(Path.join(path, "pack.json"))
+        end)
+        |> Enum.map(&write_pack_contents/1)
+        |> Enum.reject(&is_nil/1)
+
+      {:ok, names}
+    else
+      {:ok, %{access: _}} -> {:error, :no_read_write}
+      e -> e
+    end
+  end
+
+  @spec list_remote(String.t()) :: {:ok, map()} | {:error, atom()}
+  def list_remote(url) do
+    uri = url |> String.trim() |> URI.parse()
+
+    with :ok <- validate_shareable_packs_available(uri) do
+      uri
+      |> URI.merge("/api/pleroma/emoji/packs")
+      |> http_get()
+    end
+  end
+
+  @spec list_local() :: {:ok, map()}
+  def list_local do
+    with {:ok, results} <- list_packs_dir() do
+      packs =
+        results
+        |> Enum.map(fn name ->
+          case load_pack(name) do
+            {:ok, pack} -> pack
+            _ -> nil
+          end
+        end)
+        |> Enum.reject(&is_nil/1)
+        |> Map.new(fn pack -> {pack.name, validate_pack(pack)} end)
+
+      {:ok, packs}
+    end
+  end
+
+  @spec get_archive(String.t()) :: {:ok, binary()} | {:error, atom()}
+  def get_archive(name) do
+    with {:ok, pack} <- load_pack(name),
+         :ok <- validate_downloadable(pack) do
+      {:ok, fetch_archive(pack)}
+    end
+  end
+
+  @spec download(String.t(), String.t(), String.t()) :: :ok | {:error, atom()}
+  def download(name, url, as) do
+    uri = url |> String.trim() |> URI.parse()
+
+    with :ok <- validate_shareable_packs_available(uri),
+         {:ok, remote_pack} <- uri |> URI.merge("/api/pleroma/emoji/packs/#{name}") |> http_get(),
+         {:ok, %{sha: sha, url: url} = pack_info} <- fetch_pack_info(remote_pack, uri, name),
+         {:ok, archive} <- download_archive(url, sha),
+         pack <- copy_as(remote_pack, as || name),
+         {:ok, _} = unzip(archive, pack_info, remote_pack, pack) do
+      # Fallback can't contain a pack.json file, since that would cause the fallback-src-sha256
+      # in it to depend on itself
+      if pack_info[:fallback] do
+        save_pack(pack)
+      else
+        {:ok, pack}
+      end
+    end
+  end
+
+  @spec save_metadata(map(), t()) :: {:ok, t()} | {:error, File.posix()}
+  def save_metadata(metadata, %__MODULE__{} = pack) do
+    pack
+    |> Map.put(:pack, metadata)
+    |> save_pack()
+  end
+
+  @spec update_metadata(String.t(), map()) :: {:ok, t()} | {:error, File.posix()}
+  def update_metadata(name, data) do
+    with {:ok, pack} <- load_pack(name) do
+      if fallback_sha_changed?(pack, data) do
+        update_sha_and_save_metadata(pack, data)
+      else
+        save_metadata(data, pack)
+      end
+    end
+  end
+
+  @spec load_pack(String.t()) :: {:ok, t()} | {:error, :not_found}
+  def load_pack(name) do
+    pack_file = Path.join([emoji_path(), name, "pack.json"])
+
+    if File.exists?(pack_file) do
+      pack =
+        pack_file
+        |> File.read!()
+        |> from_json()
+        |> Map.put(:pack_file, pack_file)
+        |> Map.put(:path, Path.dirname(pack_file))
+        |> Map.put(:name, name)
+
+      {:ok, pack}
+    else
+      {:error, :not_found}
+    end
+  end
+
+  @spec emoji_path() :: Path.t()
+  defp emoji_path do
+    [:instance, :static_dir]
+    |> Pleroma.Config.get!()
+    |> Path.join("emoji")
+  end
+
+  defp validate_emoji_not_exists(shortcode, force \\ false)
+  defp validate_emoji_not_exists(_shortcode, true), do: :ok
+
+  defp validate_emoji_not_exists(shortcode, _) do
+    case Emoji.get(shortcode) do
+      nil -> :ok
+      _ -> {:error, :already_exists}
+    end
+  end
+
+  defp write_pack_contents(path) do
+    pack = %__MODULE__{
+      files: files_from_path(path),
+      path: path,
+      pack_file: Path.join(path, "pack.json")
+    }
+
+    case save_pack(pack) do
+      {:ok, _pack} -> Path.basename(path)
+      _ -> nil
+    end
+  end
+
+  defp files_from_path(path) do
+    txt_path = Path.join(path, "emoji.txt")
+
+    if File.exists?(txt_path) do
+      # There's an emoji.txt file, it's likely from a pack installed by the pack manager.
+      # Make a pack.json file from the contents of that emoji.txt file
+
+      # FIXME: Copy-pasted from Pleroma.Emoji/load_from_file_stream/2
+
+      # Create a map of shortcodes to filenames from emoji.txt
+      txt_path
+      |> File.read!()
+      |> String.split("\n")
+      |> Enum.map(&String.trim/1)
+      |> Enum.map(fn line ->
+        case String.split(line, ~r/,\s*/) do
+          # This matches both strings with and without tags
+          # and we don't care about tags here
+          [name, file | _] ->
+            file_dir_name = Path.dirname(file)
+
+            if String.ends_with?(path, file_dir_name) do
+              {name, Path.basename(file)}
+            else
+              {name, file}
+            end
+
+          _ ->
+            nil
+        end
+      end)
+      |> Enum.reject(&is_nil/1)
+      |> Map.new()
+    else
+      # If there's no emoji.txt, assume all files
+      # that are of certain extensions from the config are emojis and import them all
+      pack_extensions = Pleroma.Config.get!([:emoji, :pack_extensions])
+      Emoji.Loader.make_shortcode_to_file_map(path, pack_extensions)
+    end
+  end
+
+  defp validate_pack(pack) do
+    info =
+      if downloadable?(pack) do
+        archive = fetch_archive(pack)
+        archive_sha = :crypto.hash(:sha256, archive) |> Base.encode16()
+
+        pack.pack
+        |> Map.put("can-download", true)
+        |> Map.put("download-sha256", archive_sha)
+      else
+        Map.put(pack.pack, "can-download", false)
+      end
+
+    Map.put(pack, :pack, info)
+  end
+
+  defp downloadable?(pack) do
+    # If the pack is set as shared, check if it can be downloaded
+    # That means that when asked, the pack can be packed and sent to the remote
+    # Otherwise, they'd have to download it from external-src
+    pack.pack["share-files"] &&
+      Enum.all?(pack.files, fn {_, file} ->
+        File.exists?(Path.join(pack.path, file))
+      end)
+  end
+
+  defp create_archive_and_cache(pack, hash) do
+    files = ['pack.json' | Enum.map(pack.files, fn {_, file} -> to_charlist(file) end)]
+
+    {:ok, {_, result}} =
+      :zip.zip('#{pack.name}.zip', files, [:memory, cwd: to_charlist(pack.path)])
+
+    ttl_per_file = Pleroma.Config.get!([:emoji, :shared_pack_cache_seconds_per_file])
+    overall_ttl = :timer.seconds(ttl_per_file * Enum.count(files))
+
+    Cachex.put!(
+      :emoji_packs_cache,
+      pack.name,
+      # if pack.json MD5 changes, the cache is not valid anymore
+      %{hash: hash, pack_data: result},
+      # Add a minute to cache time for every file in the pack
+      ttl: overall_ttl
+    )
+
+    result
+  end
+
+  defp save_pack(pack) do
+    with {:ok, json} <- Jason.encode(pack, pretty: true),
+         :ok <- File.write(pack.pack_file, json) do
+      {:ok, pack}
+    end
+  end
+
+  defp from_json(json) do
+    map = Jason.decode!(json)
+
+    struct(__MODULE__, %{files: map["files"], pack: map["pack"]})
+  end
+
+  defp validate_shareable_packs_available(uri) do
+    with {:ok, %{"links" => links}} <- uri |> URI.merge("/.well-known/nodeinfo") |> http_get(),
+         # Get the actual nodeinfo address and fetch it
+         {:ok, %{"metadata" => %{"features" => features}}} <-
+           links |> List.last() |> Map.get("href") |> http_get() do
+      if Enum.member?(features, "shareable_emoji_packs") do
+        :ok
+      else
+        {:error, :not_shareable}
+      end
+    end
+  end
+
+  defp validate_not_empty(list) do
+    if Enum.all?(list, fn i -> is_binary(i) and i != "" end) do
+      :ok
+    else
+      {:error, :empty_values}
+    end
+  end
+
+  defp save_file(file, pack, filename) do
+    file_path = Path.join(pack.path, filename)
+    create_subdirs(file_path)
+
+    case file do
+      %Plug.Upload{path: upload_path} ->
+        # Copy the uploaded file from the temporary directory
+        with {:ok, _} <- File.copy(upload_path, file_path), do: :ok
+
+      url when is_binary(url) ->
+        # Download and write the file
+        file_contents = Tesla.get!(url).body
+        File.write(file_path, file_contents)
+    end
+  end
+
+  defp put_emoji(pack, shortcode, filename) do
+    files = Map.put(pack.files, shortcode, filename)
+    %{pack | files: files}
+  end
+
+  defp delete_emoji(pack, shortcode) do
+    files = Map.delete(pack.files, shortcode)
+    %{pack | files: files}
+  end
+
+  defp rename_file(pack, filename, new_filename) do
+    old_path = Path.join(pack.path, filename)
+    new_path = Path.join(pack.path, new_filename)
+    create_subdirs(new_path)
+
+    with :ok <- File.rename(old_path, new_path) do
+      remove_dir_if_empty(old_path, filename)
+    end
+  end
+
+  defp create_subdirs(file_path) do
+    if String.contains?(file_path, "/") do
+      file_path
+      |> Path.dirname()
+      |> File.mkdir_p!()
+    end
+  end
+
+  defp remove_file(pack, shortcode) do
+    with {:ok, filename} <- get_filename(pack, shortcode),
+         emoji <- Path.join(pack.path, filename),
+         :ok <- File.rm(emoji) do
+      remove_dir_if_empty(emoji, filename)
+    end
+  end
+
+  defp remove_dir_if_empty(emoji, filename) do
+    dir = Path.dirname(emoji)
+
+    if String.contains?(filename, "/") and File.ls!(dir) == [] do
+      File.rmdir!(dir)
+    else
+      :ok
+    end
+  end
+
+  defp get_filename(pack, shortcode) do
+    with %{^shortcode => filename} when is_binary(filename) <- pack.files,
+         true <- pack.path |> Path.join(filename) |> File.exists?() do
+      {:ok, filename}
+    else
+      _ -> {:error, :doesnt_exist}
+    end
+  end
+
+  defp http_get(%URI{} = url), do: url |> to_string() |> http_get()
+
+  defp http_get(url) do
+    with {:ok, %{body: body}} <- url |> Pleroma.HTTP.get() do
+      Jason.decode(body)
+    end
+  end
+
+  defp list_packs_dir do
+    emoji_path = emoji_path()
+    # Create the directory first if it does not exist. This is probably the first request made
+    # with the API so it should be sufficient
+    with {:create_dir, :ok} <- {:create_dir, File.mkdir_p(emoji_path)},
+         {:ls, {:ok, results}} <- {:ls, File.ls(emoji_path)} do
+      {:ok, results}
+    else
+      {:create_dir, {:error, e}} -> {:error, :create_dir, e}
+      {:ls, {:error, e}} -> {:error, :ls, e}
+    end
+  end
+
+  defp validate_downloadable(pack) do
+    if downloadable?(pack), do: :ok, else: {:error, :cant_download}
+  end
+
+  defp copy_as(remote_pack, local_name) do
+    path = Path.join(emoji_path(), local_name)
+
+    %__MODULE__{
+      name: local_name,
+      path: path,
+      files: remote_pack["files"],
+      pack_file: Path.join(path, "pack.json")
+    }
+  end
+
+  defp unzip(archive, pack_info, remote_pack, local_pack) do
+    with :ok <- File.mkdir_p!(local_pack.path) do
+      files = Enum.map(remote_pack["files"], fn {_, path} -> to_charlist(path) end)
+      # Fallback cannot contain a pack.json file
+      files = if pack_info[:fallback], do: files, else: ['pack.json' | files]
+
+      :zip.unzip(archive, cwd: to_charlist(local_pack.path), file_list: files)
+    end
+  end
+
+  defp fetch_pack_info(remote_pack, uri, name) do
+    case remote_pack["pack"] do
+      %{"share-files" => true, "can-download" => true, "download-sha256" => sha} ->
+        {:ok,
+         %{
+           sha: sha,
+           url: URI.merge(uri, "/api/pleroma/emoji/packs/#{name}/archive") |> to_string()
+         }}
+
+      %{"fallback-src" => src, "fallback-src-sha256" => sha} when is_binary(src) ->
+        {:ok,
+         %{
+           sha: sha,
+           url: src,
+           fallback: true
+         }}
+
+      _ ->
+        {:error, "The pack was not set as shared and there is no fallback src to download from"}
+    end
+  end
+
+  defp download_archive(url, sha) do
+    with {:ok, %{body: archive}} <- Tesla.get(url) do
+      if Base.decode16!(sha) == :crypto.hash(:sha256, archive) do
+        {:ok, archive}
+      else
+        {:error, :invalid_checksum}
+      end
+    end
+  end
+
+  defp fetch_archive(pack) do
+    hash = :crypto.hash(:md5, File.read!(pack.pack_file))
+
+    case Cachex.get!(:emoji_packs_cache, pack.name) do
+      %{hash: ^hash, pack_data: archive} -> archive
+      _ -> create_archive_and_cache(pack, hash)
+    end
+  end
+
+  defp fallback_sha_changed?(pack, data) do
+    is_binary(data[:"fallback-src"]) and data[:"fallback-src"] != pack.pack["fallback-src"]
+  end
+
+  defp update_sha_and_save_metadata(pack, data) do
+    with {:ok, %{body: zip}} <- Tesla.get(data[:"fallback-src"]),
+         :ok <- validate_has_all_files(pack, zip) do
+      fallback_sha = :sha256 |> :crypto.hash(zip) |> Base.encode16()
+
+      data
+      |> Map.put("fallback-src-sha256", fallback_sha)
+      |> save_metadata(pack)
+    end
+  end
+
+  defp validate_has_all_files(pack, zip) do
+    with {:ok, f_list} <- :zip.unzip(zip, [:memory]) do
+      # Check if all files from the pack.json are in the archive
+      pack.files
+      |> Enum.all?(fn {_, from_manifest} ->
+        List.keyfind(f_list, to_charlist(from_manifest), 0)
+      end)
+      |> if(do: :ok, else: {:error, :incomplete})
+    end
+  end
+end

lib/pleroma/filter.ex

@@ -89,11 +89,10 @@ defmodule Pleroma.Filter do
     |> Repo.delete()
   end
 
-  def update(%Pleroma.Filter{} = filter) do
-    destination = Map.from_struct(filter)
-
-    Pleroma.Filter.get(filter.filter_id, %{id: filter.user_id})
-    |> cast(destination, [:phrase, :context, :hide, :expires_at, :whole_word])
+  def update(%Pleroma.Filter{} = filter, params) do
+    filter
+    |> cast(params, [:phrase, :context, :hide, :expires_at, :whole_word])
+    |> validate_required([:phrase, :context])
     |> Repo.update()
   end
 end

lib/pleroma/following_relationship.ex

@@ -8,12 +8,14 @@ defmodule Pleroma.FollowingRelationship do
   import Ecto.Changeset
   import Ecto.Query
 
+  alias Ecto.Changeset
   alias FlakeId.Ecto.CompatType
+  alias Pleroma.FollowingRelationship.State
   alias Pleroma.Repo
   alias Pleroma.User
 
   schema "following_relationships" do
-    field(:state, :string, default: "accept")
+    field(:state, State, default: :follow_pending)
 
     belongs_to(:follower, User, type: CompatType)
     belongs_to(:following, User, type: CompatType)
@@ -21,12 +23,29 @@ defmodule Pleroma.FollowingRelationship do
     timestamps()
   end
 
+  @doc "Returns underlying integer code for state atom"
+  def state_int_code(state_atom), do: State.__enum_map__() |> Keyword.fetch!(state_atom)
+
+  def accept_state_code, do: state_int_code(:follow_accept)
+
   def changeset(%__MODULE__{} = following_relationship, attrs) do
     following_relationship
     |> cast(attrs, [:state])
     |> put_assoc(:follower, attrs.follower)
     |> put_assoc(:following, attrs.following)
     |> validate_required([:state, :follower, :following])
+    |> unique_constraint(:follower_id,
+      name: :following_relationships_follower_id_following_id_index
+    )
+    |> validate_not_self_relationship()
+  end
+
+  def state_to_enum(state) when state in ["pending", "accept", "reject"] do
+    String.to_existing_atom("follow_#{state}")
+  end
+
+  def state_to_enum(state) do
+    raise "State is not convertible to Pleroma.FollowingRelationship.State: #{state}"
   end
 
   def get(%User{} = follower, %User{} = following) do
@@ -35,7 +54,7 @@ defmodule Pleroma.FollowingRelationship do
     |> Repo.one()
   end
 
-  def update(follower, following, "reject"), do: unfollow(follower, following)
+  def update(follower, following, :follow_reject), do: unfollow(follower, following)
 
   def update(%User{} = follower, %User{} = following, state) do
     case get(follower, following) do
@@ -50,7 +69,7 @@ defmodule Pleroma.FollowingRelationship do
     end
   end
 
-  def follow(%User{} = follower, %User{} = following, state \\ "accept") do
+  def follow(%User{} = follower, %User{} = following, state \\ :follow_accept) do
     %__MODULE__{}
     |> changeset(%{follower: follower, following: following, state: state})
     |> Repo.insert(on_conflict: :nothing)
@@ -69,6 +88,29 @@ defmodule Pleroma.FollowingRelationship do
     |> Repo.aggregate(:count, :id)
   end
 
+  def followers_query(%User{} = user) do
+    __MODULE__
+    |> join(:inner, [r], u in User, on: r.follower_id == u.id)
+    |> where([r], r.following_id == ^user.id)
+    |> where([r], r.state == ^:follow_accept)
+  end
+
+  def followers_ap_ids(%User{} = user, from_ap_ids \\ nil) do
+    query =
+      user
+      |> followers_query()
+      |> select([r, u], u.ap_id)
+
+    query =
+      if from_ap_ids do
+        where(query, [r, u], u.ap_id in ^from_ap_ids)
+      else
+        query
+      end
+
+    Repo.all(query)
+  end
+
   def following_count(%User{id: nil}), do: 0
 
   def following_count(%User{} = user) do
@@ -80,7 +122,7 @@ defmodule Pleroma.FollowingRelationship do
   def get_follow_requests(%User{id: id}) do
     __MODULE__
     |> join(:inner, [r], f in assoc(r, :follower))
-    |> where([r], r.state == "pending")
+    |> where([r], r.state == ^:follow_pending)
     |> where([r], r.following_id == ^id)
     |> select([r, f], f)
     |> Repo.all()
@@ -88,16 +130,20 @@ defmodule Pleroma.FollowingRelationship do
 
   def following?(%User{id: follower_id}, %User{id: followed_id}) do
     __MODULE__
-    |> where(follower_id: ^follower_id, following_id: ^followed_id, state: "accept")
+    |> where(follower_id: ^follower_id, following_id: ^followed_id, state: ^:follow_accept)
     |> Repo.exists?()
   end
 
+  def following_query(%User{} = user) do
+    __MODULE__
+    |> join(:inner, [r], u in User, on: r.following_id == u.id)
+    |> where([r], r.follower_id == ^user.id)
+    |> where([r], r.state == ^:follow_accept)
+  end
+
   def following(%User{} = user) do
     following =
-      __MODULE__
-      |> join(:inner, [r], u in User, on: r.following_id == u.id)
-      |> where([r], r.follower_id == ^user.id)
-      |> where([r], r.state == "accept")
+      following_query(user)
       |> select([r, u], u.follower_address)
       |> Repo.all()
 
@@ -129,4 +175,82 @@ defmodule Pleroma.FollowingRelationship do
         move_following(origin, target)
     end
   end
+
+  def all_between_user_sets(
+        source_users,
+        target_users
+      )
+      when is_list(source_users) and is_list(target_users) do
+    source_user_ids = User.binary_id(source_users)
+    target_user_ids = User.binary_id(target_users)
+
+    __MODULE__
+    |> where(
+      fragment(
+        "(follower_id = ANY(?) AND following_id = ANY(?)) OR \
+        (follower_id = ANY(?) AND following_id = ANY(?))",
+        ^source_user_ids,
+        ^target_user_ids,
+        ^target_user_ids,
+        ^source_user_ids
+      )
+    )
+    |> Repo.all()
+  end
+
+  def find(following_relationships, follower, following) do
+    Enum.find(following_relationships, fn
+      fr -> fr.follower_id == follower.id and fr.following_id == following.id
+    end)
+  end
+
+  @doc """
+  For a query with joined activity,
+  keeps rows where activity's actor is followed by user -or- is NOT domain-blocked by user.
+  """
+  def keep_following_or_not_domain_blocked(query, user) do
+    where(
+      query,
+      [_, activity],
+      fragment(
+        # "(actor's domain NOT in domain_blocks) OR (actor IS in followed AP IDs)"
+        """
+        NOT (substring(? from '.*://([^/]*)') = ANY(?)) OR
+          ? = ANY(SELECT ap_id FROM users AS u INNER JOIN following_relationships AS fr
+            ON u.id = fr.following_id WHERE fr.follower_id = ? AND fr.state = ?)
+        """,
+        activity.actor,
+        ^user.domain_blocks,
+        activity.actor,
+        ^User.binary_id(user.id),
+        ^accept_state_code()
+      )
+    )
+  end
+
+  defp validate_not_self_relationship(%Changeset{} = changeset) do
+    changeset
+    |> validate_follower_id_following_id_inequality()
+    |> validate_following_id_follower_id_inequality()
+  end
+
+  defp validate_follower_id_following_id_inequality(%Changeset{} = changeset) do
+    validate_change(changeset, :follower_id, fn _, follower_id ->
+      if follower_id == get_field(changeset, :following_id) do
+        [source_id: "can't be equal to following_id"]
+      else
+        []
+      end
+    end)
+  end
+
+  defp validate_following_id_follower_id_inequality(%Changeset{} = changeset) do
+    validate_change(changeset, :following_id, fn _, following_id ->
+      if following_id == get_field(changeset, :follower_id) do
+        [target_id: "can't be equal to follower_id"]
+      else
+        []
+      end
+    end)
+  end
 end

lib/pleroma/formatter.ex

@@ -31,13 +31,23 @@ defmodule Pleroma.Formatter do
   def mention_handler("@" <> nickname, buffer, opts, acc) do
     case User.get_cached_by_nickname(nickname) do
       %User{id: id} = user ->
-        ap_id = get_ap_id(user)
+        user_url = user.uri || user.ap_id
         nickname_text = get_nickname_text(nickname, opts)
 
         link =
-          ~s(<span class="h-card"><a data-user="#{id}" class="u-url mention" href="#{ap_id}" rel="ugc">@<span>#{
-            nickname_text
-          }</span></a></span>)
+          Phoenix.HTML.Tag.content_tag(
+            :span,
+            Phoenix.HTML.Tag.content_tag(
+              :a,
+              ["@", Phoenix.HTML.Tag.content_tag(:span, nickname_text)],
+              "data-user": id,
+              class: "u-url mention",
+              href: user_url,
+              rel: "ugc"
+            ),
+            class: "h-card"
+          )
+          |> Phoenix.HTML.safe_to_string()
 
         {link, %{acc | mentions: MapSet.put(acc.mentions, {"@" <> nickname, user})}}
 
@@ -49,7 +59,15 @@ defmodule Pleroma.Formatter do
   def hashtag_handler("#" <> tag = tag_text, _buffer, _opts, acc) do
     tag = String.downcase(tag)
     url = "#{Pleroma.Web.base_url()}/tag/#{tag}"
-    link = ~s(<a class="hashtag" data-tag="#{tag}" href="#{url}" rel="tag ugc">#{tag_text}</a>)
+
+    link =
+      Phoenix.HTML.Tag.content_tag(:a, tag_text,
+        class: "hashtag",
+        "data-tag": tag,
+        href: url,
+        rel: "tag ugc"
+      )
+      |> Phoenix.HTML.safe_to_string()
 
     {link, %{acc | tags: MapSet.put(acc.tags, {tag_text, tag})}}
   end
@@ -128,9 +146,6 @@ defmodule Pleroma.Formatter do
     end
   end
 
-  defp get_ap_id(%User{source_data: %{"url" => url}}) when is_binary(url), do: url
-  defp get_ap_id(%User{ap_id: ap_id}), do: ap_id
-
   defp get_nickname_text(nickname, %{mentions_format: :full}), do: User.full_nickname(nickname)
   defp get_nickname_text(nickname, _), do: User.local_nickname(nickname)
 end

lib/pleroma/gun/api.ex

@@ -0,0 +1,45 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Gun.API do
+  @behaviour Pleroma.Gun
+
+  alias Pleroma.Gun
+
+  @gun_keys [
+    :connect_timeout,
+    :http_opts,
+    :http2_opts,
+    :protocols,
+    :retry,
+    :retry_timeout,
+    :trace,
+    :transport,
+    :tls_opts,
+    :tcp_opts,
+    :socks_opts,
+    :ws_opts
+  ]
+
+  @impl Gun
+  def open(host, port, opts \\ %{}), do: :gun.open(host, port, Map.take(opts, @gun_keys))
+
+  @impl Gun
+  defdelegate info(pid), to: :gun
+
+  @impl Gun
+  defdelegate close(pid), to: :gun
+
+  @impl Gun
+  defdelegate await_up(pid, timeout \\ 5_000), to: :gun
+
+  @impl Gun
+  defdelegate connect(pid, opts), to: :gun
+
+  @impl Gun
+  defdelegate await(pid, ref), to: :gun
+
+  @impl Gun
+  defdelegate set_owner(pid, owner), to: :gun
+end

lib/pleroma/gun/conn.ex

@@ -0,0 +1,198 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Gun.Conn do
+  @moduledoc """
+  Struct for gun connection data
+  """
+  alias Pleroma.Gun
+  alias Pleroma.Pool.Connections
+
+  require Logger
+
+  @type gun_state :: :up | :down
+  @type conn_state :: :active | :idle
+
+  @type t :: %__MODULE__{
+          conn: pid(),
+          gun_state: gun_state(),
+          conn_state: conn_state(),
+          used_by: [pid()],
+          last_reference: pos_integer(),
+          crf: float(),
+          retries: pos_integer()
+        }
+
+  defstruct conn: nil,
+            gun_state: :open,
+            conn_state: :init,
+            used_by: [],
+            last_reference: 0,
+            crf: 1,
+            retries: 0
+
+  @spec open(String.t() | URI.t(), atom(), keyword()) :: :ok | nil
+  def open(url, name, opts \\ [])
+  def open(url, name, opts) when is_binary(url), do: open(URI.parse(url), name, opts)
+
+  def open(%URI{} = uri, name, opts) do
+    pool_opts = Pleroma.Config.get([:connections_pool], [])
+
+    opts =
+      opts
+      |> Enum.into(%{})
+      |> Map.put_new(:retry, pool_opts[:retry] || 1)
+      |> Map.put_new(:retry_timeout, pool_opts[:retry_timeout] || 1000)
+      |> Map.put_new(:await_up_timeout, pool_opts[:await_up_timeout] || 5_000)
+      |> maybe_add_tls_opts(uri)
+
+    key = "#{uri.scheme}:#{uri.host}:#{uri.port}"
+
+    max_connections = pool_opts[:max_connections] || 250
+
+    conn_pid =
+      if Connections.count(name) < max_connections do
+        do_open(uri, opts)
+      else
+        close_least_used_and_do_open(name, uri, opts)
+      end
+
+    if is_pid(conn_pid) do
+      conn = %Pleroma.Gun.Conn{
+        conn: conn_pid,
+        gun_state: :up,
+        conn_state: :active,
+        last_reference: :os.system_time(:second)
+      }
+
+      :ok = Gun.set_owner(conn_pid, Process.whereis(name))
+      Connections.add_conn(name, key, conn)
+    end
+  end
+
+  defp maybe_add_tls_opts(opts, %URI{scheme: "http"}), do: opts
+
+  defp maybe_add_tls_opts(opts, %URI{scheme: "https", host: host}) do
+    tls_opts = [
+      verify: :verify_peer,
+      cacertfile: CAStore.file_path(),
+      depth: 20,
+      reuse_sessions: false,
+      verify_fun:
+        {&:ssl_verify_hostname.verify_fun/3,
+         [check_hostname: Pleroma.HTTP.Connection.format_host(host)]}
+    ]
+
+    tls_opts =
+      if Keyword.keyword?(opts[:tls_opts]) do
+        Keyword.merge(tls_opts, opts[:tls_opts])
+      else
+        tls_opts
+      end
+
+    Map.put(opts, :tls_opts, tls_opts)
+  end
+
+  defp do_open(uri, %{proxy: {proxy_host, proxy_port}} = opts) do
+    connect_opts =
+      uri
+      |> destination_opts()
+      |> add_http2_opts(uri.scheme, Map.get(opts, :tls_opts, []))
+
+    with open_opts <- Map.delete(opts, :tls_opts),
+         {:ok, conn} <- Gun.open(proxy_host, proxy_port, open_opts),
+         {:ok, _} <- Gun.await_up(conn, opts[:await_up_timeout]),
+         stream <- Gun.connect(conn, connect_opts),
+         {:response, :fin, 200, _} <- Gun.await(conn, stream) do
+      conn
+    else
+      error ->
+        Logger.warn(
+          "Opening proxied connection to #{compose_uri_log(uri)} failed with error #{
+            inspect(error)
+          }"
+        )
+
+        error
+    end
+  end
+
+  defp do_open(uri, %{proxy: {proxy_type, proxy_host, proxy_port}} = opts) do
+    version =
+      proxy_type
+      |> to_string()
+      |> String.last()
+      |> case do
+        "4" -> 4
+        _ -> 5
+      end
+
+    socks_opts =
+      uri
+      |> destination_opts()
+      |> add_http2_opts(uri.scheme, Map.get(opts, :tls_opts, []))
+      |> Map.put(:version, version)
+
+    opts =
+      opts
+      |> Map.put(:protocols, [:socks])
+      |> Map.put(:socks_opts, socks_opts)
+
+    with {:ok, conn} <- Gun.open(proxy_host, proxy_port, opts),
+         {:ok, _} <- Gun.await_up(conn, opts[:await_up_timeout]) do
+      conn
+    else
+      error ->
+        Logger.warn(
+          "Opening socks proxied connection to #{compose_uri_log(uri)} failed with error #{
+            inspect(error)
+          }"
+        )
+
+        error
+    end
+  end
+
+  defp do_open(%URI{host: host, port: port} = uri, opts) do
+    host = Pleroma.HTTP.Connection.parse_host(host)
+
+    with {:ok, conn} <- Gun.open(host, port, opts),
+         {:ok, _} <- Gun.await_up(conn, opts[:await_up_timeout]) do
+      conn
+    else
+      error ->
+        Logger.warn(
+          "Opening connection to #{compose_uri_log(uri)} failed with error #{inspect(error)}"
+        )
+
+        error
+    end
+  end
+
+  defp destination_opts(%URI{host: host, port: port}) do
+    host = Pleroma.HTTP.Connection.parse_host(host)
+    %{host: host, port: port}
+  end
+
+  defp add_http2_opts(opts, "https", tls_opts) do
+    Map.merge(opts, %{protocols: [:http2], transport: :tls, tls_opts: tls_opts})
+  end
+
+  defp add_http2_opts(opts, _, _), do: opts
+
+  defp close_least_used_and_do_open(name, uri, opts) do
+    with [{key, conn} | _conns] <- Connections.get_unused_conns(name),
+         :ok <- Gun.close(conn.conn) do
+      Connections.remove_conn(name, key)
+
+      do_open(uri, opts)
+    else
+      [] -> {:error, :pool_overflowed}
+    end
+  end
+
+  def compose_uri_log(%URI{scheme: scheme, host: host, path: path}) do
+    "#{scheme}://#{host}#{path}"
+  end
+end

lib/pleroma/gun/gun.ex

@@ -0,0 +1,31 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Gun do
+  @callback open(charlist(), pos_integer(), map()) :: {:ok, pid()}
+  @callback info(pid()) :: map()
+  @callback close(pid()) :: :ok
+  @callback await_up(pid, pos_integer()) :: {:ok, atom()} | {:error, atom()}
+  @callback connect(pid(), map()) :: reference()
+  @callback await(pid(), reference()) :: {:response, :fin, 200, []}
+  @callback set_owner(pid(), pid()) :: :ok
+
+  @api Pleroma.Config.get([Pleroma.Gun], Pleroma.Gun.API)
+
+  defp api, do: @api
+
+  def open(host, port, opts), do: api().open(host, port, opts)
+
+  def info(pid), do: api().info(pid)
+
+  def close(pid), do: api().close(pid)
+
+  def await_up(pid, timeout \\ 5_000), do: api().await_up(pid, timeout)
+
+  def connect(pid, opts), do: api().connect(pid, opts)
+
+  def await(pid, ref), do: api().await(pid, ref)
+
+  def set_owner(pid, owner), do: api().set_owner(pid, owner)
+end

lib/pleroma/healthcheck.ex

@@ -29,7 +29,7 @@ defmodule Pleroma.Healthcheck do
   @spec system_info() :: t()
   def system_info do
     %Healthcheck{
-      memory_used: Float.round(:erlang.memory(:total) / 1024 / 1024, 2)
+      memory_used: Float.round(:recon_alloc.memory(:allocated) / 1024 / 1024, 2)
     }
     |> assign_db_info()
     |> assign_job_queue_stats()

lib/pleroma/helpers/uri_helper.ex

@@ -24,4 +24,7 @@ defmodule Pleroma.Helpers.UriHelper do
       params
     end
   end
+
+  def maybe_add_base("/" <> uri, base), do: Path.join([base, uri])
+  def maybe_add_base(uri, _base), do: uri
 end

lib/pleroma/http/adapter_helper.ex

@@ -0,0 +1,41 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.HTTP.AdapterHelper do
+  alias Pleroma.HTTP.Connection
+
+  @type proxy ::
+          {Connection.host(), pos_integer()}
+          | {Connection.proxy_type(), Connection.host(), pos_integer()}
+
+  @callback options(keyword(), URI.t()) :: keyword()
+  @callback after_request(keyword()) :: :ok
+
+  @spec options(keyword(), URI.t()) :: keyword()
+  def options(opts, _uri) do
+    proxy = Pleroma.Config.get([:http, :proxy_url], nil)
+    maybe_add_proxy(opts, format_proxy(proxy))
+  end
+
+  @spec maybe_get_conn(URI.t(), keyword()) :: keyword()
+  def maybe_get_conn(_uri, opts), do: opts
+
+  @spec after_request(keyword()) :: :ok
+  def after_request(_opts), do: :ok
+
+  @spec format_proxy(String.t() | tuple() | nil) :: proxy() | nil
+  def format_proxy(nil), do: nil
+
+  def format_proxy(proxy_url) do
+    case Connection.parse_proxy(proxy_url) do
+      {:ok, host, port} -> {host, port}
+      {:ok, type, host, port} -> {type, host, port}
+      _ -> nil
+    end
+  end
+
+  @spec maybe_add_proxy(keyword(), proxy() | nil) :: keyword()
+  def maybe_add_proxy(opts, nil), do: opts
+  def maybe_add_proxy(opts, proxy), do: Keyword.put_new(opts, :proxy, proxy)
+end

lib/pleroma/http/adapter_helper/gun.ex

@@ -0,0 +1,77 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.HTTP.AdapterHelper.Gun do
+  @behaviour Pleroma.HTTP.AdapterHelper
+
+  alias Pleroma.HTTP.AdapterHelper
+  alias Pleroma.Pool.Connections
+
+  require Logger
+
+  @defaults [
+    connect_timeout: 5_000,
+    domain_lookup_timeout: 5_000,
+    tls_handshake_timeout: 5_000,
+    retry: 1,
+    retry_timeout: 1000,
+    await_up_timeout: 5_000
+  ]
+
+  @spec options(keyword(), URI.t()) :: keyword()
+  def options(incoming_opts \\ [], %URI{} = uri) do
+    proxy =
+      Pleroma.Config.get([:http, :proxy_url])
+      |> AdapterHelper.format_proxy()
+
+    config_opts = Pleroma.Config.get([:http, :adapter], [])
+
+    @defaults
+    |> Keyword.merge(config_opts)
+    |> add_scheme_opts(uri)
+    |> AdapterHelper.maybe_add_proxy(proxy)
+    |> maybe_get_conn(uri, incoming_opts)
+  end
+
+  @spec after_request(keyword()) :: :ok
+  def after_request(opts) do
+    if opts[:conn] && opts[:body_as] != :chunks do
+      Connections.checkout(opts[:conn], self(), :gun_connections)
+    end
+
+    :ok
+  end
+
+  defp add_scheme_opts(opts, %{scheme: "http"}), do: opts
+
+  defp add_scheme_opts(opts, %{scheme: "https"}) do
+    opts
+    |> Keyword.put(:certificates_verification, true)
+    |> Keyword.put(:tls_opts, log_level: :warning)
+  end
+
+  defp maybe_get_conn(adapter_opts, uri, incoming_opts) do
+    {receive_conn?, opts} =
+      adapter_opts
+      |> Keyword.merge(incoming_opts)
+      |> Keyword.pop(:receive_conn, true)
+
+    if Connections.alive?(:gun_connections) and receive_conn? do
+      checkin_conn(uri, opts)
+    else
+      opts
+    end
+  end
+
+  defp checkin_conn(uri, opts) do
+    case Connections.checkin(uri, :gun_connections) do
+      nil ->
+        Task.start(Pleroma.Gun.Conn, :open, [uri, :gun_connections, opts])
+        opts
+
+      conn when is_pid(conn) ->
+        Keyword.merge(opts, conn: conn, close_conn: false)
+    end
+  end
+end

lib/pleroma/http/adapter_helper/hackney.ex

@@ -0,0 +1,28 @@
+defmodule Pleroma.HTTP.AdapterHelper.Hackney do
+  @behaviour Pleroma.HTTP.AdapterHelper
+
+  @defaults [
+    connect_timeout: 10_000,
+    recv_timeout: 20_000,
+    follow_redirect: true,
+    force_redirect: true,
+    pool: :federation
+  ]
+
+  @spec options(keyword(), URI.t()) :: keyword()
+  def options(connection_opts \\ [], %URI{} = uri) do
+    proxy = Pleroma.Config.get([:http, :proxy_url])
+
+    config_opts = Pleroma.Config.get([:http, :adapter], [])
+
+    @defaults
+    |> Keyword.merge(config_opts)
+    |> Keyword.merge(connection_opts)
+    |> add_scheme_opts(uri)
+    |> Pleroma.HTTP.AdapterHelper.maybe_add_proxy(proxy)
+  end
+
+  defp add_scheme_opts(opts, _), do: opts
+
+  def after_request(_), do: :ok
+end

lib/pleroma/http/connection.ex

@@ -4,40 +4,121 @@
 
 defmodule Pleroma.HTTP.Connection do
   @moduledoc """
-  Connection for http-requests.
+  Configure Tesla.Client with default and customized adapter options.
   """
 
-  @hackney_options [
-    connect_timeout: 10_000,
-    recv_timeout: 20_000,
-    follow_redirect: true,
-    force_redirect: true,
-    pool: :federation
-  ]
-  @adapter Application.get_env(:tesla, :adapter)
+  alias Pleroma.Config
+  alias Pleroma.HTTP.AdapterHelper
+
+  require Logger
+
+  @defaults [pool: :federation]
+
+  @type ip_address :: ipv4_address() | ipv6_address()
+  @type ipv4_address :: {0..255, 0..255, 0..255, 0..255}
+  @type ipv6_address ::
+          {0..65_535, 0..65_535, 0..65_535, 0..65_535, 0..65_535, 0..65_535, 0..65_535, 0..65_535}
+  @type proxy_type() :: :socks4 | :socks5
+  @type host() :: charlist() | ip_address()
 
   @doc """
-  Configure a client connection
-
-  # Returns
-
-  Tesla.Env.client
+  Merge default connection & adapter options with received ones.
   """
-  @spec new(Keyword.t()) :: Tesla.Env.client()
-  def new(opts \\ []) do
-    Tesla.client([], {@adapter, hackney_options(opts)})
+
+  @spec options(URI.t(), keyword()) :: keyword()
+  def options(%URI{} = uri, opts \\ []) do
+    @defaults
+    |> pool_timeout()
+    |> Keyword.merge(opts)
+    |> adapter_helper().options(uri)
   end
 
-  # fetch Hackney options
-  #
-  def hackney_options(opts) do
-    options = Keyword.get(opts, :adapter, [])
-    adapter_options = Pleroma.Config.get([:http, :adapter], [])
-    proxy_url = Pleroma.Config.get([:http, :proxy_url], nil)
+  defp pool_timeout(opts) do
+    {config_key, default} =
+      if adapter() == Tesla.Adapter.Gun do
+        {:pools, Config.get([:pools, :default, :timeout])}
+      else
+        {:hackney_pools, 10_000}
+      end
 
-    @hackney_options
-    |> Keyword.merge(adapter_options)
-    |> Keyword.merge(options)
-    |> Keyword.merge(proxy: proxy_url)
+    timeout = Config.get([config_key, opts[:pool], :timeout], default)
+
+    Keyword.merge(opts, timeout: timeout)
+  end
+
+  @spec after_request(keyword()) :: :ok
+  def after_request(opts), do: adapter_helper().after_request(opts)
+
+  defp adapter, do: Application.get_env(:tesla, :adapter)
+
+  defp adapter_helper do
+    case adapter() do
+      Tesla.Adapter.Gun -> AdapterHelper.Gun
+      Tesla.Adapter.Hackney -> AdapterHelper.Hackney
+      _ -> AdapterHelper
+    end
+  end
+
+  @spec parse_proxy(String.t() | tuple() | nil) ::
+          {:ok, host(), pos_integer()}
+          | {:ok, proxy_type(), host(), pos_integer()}
+          | {:error, atom()}
+          | nil
+
+  def parse_proxy(nil), do: nil
+
+  def parse_proxy(proxy) when is_binary(proxy) do
+    with [host, port] <- String.split(proxy, ":"),
+         {port, ""} <- Integer.parse(port) do
+      {:ok, parse_host(host), port}
+    else
+      {_, _} ->
+        Logger.warn("Parsing port failed #{inspect(proxy)}")
+        {:error, :invalid_proxy_port}
+
+      :error ->
+        Logger.warn("Parsing port failed #{inspect(proxy)}")
+        {:error, :invalid_proxy_port}
+
+      _ ->
+        Logger.warn("Parsing proxy failed #{inspect(proxy)}")
+        {:error, :invalid_proxy}
+    end
+  end
+
+  def parse_proxy(proxy) when is_tuple(proxy) do
+    with {type, host, port} <- proxy do
+      {:ok, type, parse_host(host), port}
+    else
+      _ ->
+        Logger.warn("Parsing proxy failed #{inspect(proxy)}")
+        {:error, :invalid_proxy}
+    end
+  end
+
+  @spec parse_host(String.t() | atom() | charlist()) :: charlist() | ip_address()
+  def parse_host(host) when is_list(host), do: host
+  def parse_host(host) when is_atom(host), do: to_charlist(host)
+
+  def parse_host(host) when is_binary(host) do
+    host = to_charlist(host)
+
+    case :inet.parse_address(host) do
+      {:error, :einval} -> host
+      {:ok, ip} -> ip
+    end
+  end
+
+  @spec format_host(String.t()) :: charlist()
+  def format_host(host) do
+    host_charlist = to_charlist(host)
+
+    case :inet.parse_address(host_charlist) do
+      {:error, :einval} ->
+        :idna.encode(host_charlist)
+
+      {:ok, _ip} ->
+        host_charlist
+    end
   end
 end

lib/pleroma/http/http.ex

@@ -4,21 +4,47 @@
 
 defmodule Pleroma.HTTP do
   @moduledoc """
-
+    Wrapper for `Tesla.request/2`.
   """
 
   alias Pleroma.HTTP.Connection
+  alias Pleroma.HTTP.Request
   alias Pleroma.HTTP.RequestBuilder, as: Builder
+  alias Tesla.Client
+  alias Tesla.Env
+
+  require Logger
 
   @type t :: __MODULE__
 
   @doc """
-  Builds and perform http request.
+  Performs GET request.
+
+  See `Pleroma.HTTP.request/5`
+  """
+  @spec get(Request.url() | nil, Request.headers(), keyword()) ::
+          nil | {:ok, Env.t()} | {:error, any()}
+  def get(url, headers \\ [], options \\ [])
+  def get(nil, _, _), do: nil
+  def get(url, headers, options), do: request(:get, url, "", headers, options)
+
+  @doc """
+  Performs POST request.
+
+  See `Pleroma.HTTP.request/5`
+  """
+  @spec post(Request.url(), String.t(), Request.headers(), keyword()) ::
+          {:ok, Env.t()} | {:error, any()}
+  def post(url, body, headers \\ [], options \\ []),
+    do: request(:post, url, body, headers, options)
+
+  @doc """
+  Builds and performs http request.
 
   # Arguments:
   `method` - :get, :post, :put, :delete
-  `url`
-  `body`
+  `url` - full url
+  `body` - request body
   `headers` - a keyworld list of headers, e.g. `[{"content-type", "text/plain"}]`
   `options` - custom, per-request middleware or adapter options
 
@@ -26,61 +52,66 @@ defmodule Pleroma.HTTP do
   `{:ok, %Tesla.Env{}}` or `{:error, error}`
 
   """
-  def request(method, url, body \\ "", headers \\ [], options \\ []) do
-    try do
-      options =
-        process_request_options(options)
-        |> process_sni_options(url)
-
-      params = Keyword.get(options, :params, [])
-
-      %{}
-      |> Builder.method(method)
-      |> Builder.headers(headers)
-      |> Builder.opts(options)
-      |> Builder.url(url)
-      |> Builder.add_param(:body, :body, body)
-      |> Builder.add_param(:query, :query, params)
-      |> Enum.into([])
-      |> (&Tesla.request(Connection.new(options), &1)).()
-    rescue
-      e ->
-        {:error, e}
-    catch
-      :exit, e ->
-        {:error, e}
-    end
-  end
-
-  defp process_sni_options(options, nil), do: options
-
-  defp process_sni_options(options, url) do
+  @spec request(atom(), Request.url(), String.t(), Request.headers(), keyword()) ::
+          {:ok, Env.t()} | {:error, any()}
+  def request(method, url, body, headers, options) when is_binary(url) do
     uri = URI.parse(url)
-    host = uri.host |> to_charlist()
+    adapter_opts = Connection.options(uri, options[:adapter] || [])
+    options = put_in(options[:adapter], adapter_opts)
+    params = options[:params] || []
+    request = build_request(method, headers, options, url, body, params)
 
-    case uri.scheme do
-      "https" -> options ++ [ssl: [server_name_indication: host]]
-      _ -> options
-    end
+    adapter = Application.get_env(:tesla, :adapter)
+    client = Tesla.client([Tesla.Middleware.FollowRedirects], adapter)
+
+    pid = Process.whereis(adapter_opts[:pool])
+
+    pool_alive? =
+      if adapter == Tesla.Adapter.Gun && pid do
+        Process.alive?(pid)
+      else
+        false
+      end
+
+    request_opts =
+      adapter_opts
+      |> Enum.into(%{})
+      |> Map.put(:env, Pleroma.Config.get([:env]))
+      |> Map.put(:pool_alive?, pool_alive?)
+
+    response = request(client, request, request_opts)
+
+    Connection.after_request(adapter_opts)
+
+    response
   end
 
-  def process_request_options(options) do
-    Keyword.merge(Pleroma.HTTP.Connection.hackney_options([]), options)
+  @spec request(Client.t(), keyword(), map()) :: {:ok, Env.t()} | {:error, any()}
+  def request(%Client{} = client, request, %{env: :test}), do: request(client, request)
+
+  def request(%Client{} = client, request, %{body_as: :chunks}), do: request(client, request)
+
+  def request(%Client{} = client, request, %{pool_alive?: false}), do: request(client, request)
+
+  def request(%Client{} = client, request, %{pool: pool, timeout: timeout}) do
+    :poolboy.transaction(
+      pool,
+      &Pleroma.Pool.Request.execute(&1, client, request, timeout),
+      timeout
+    )
   end
 
-  @doc """
-  Performs GET request.
+  @spec request(Client.t(), keyword()) :: {:ok, Env.t()} | {:error, any()}
+  def request(client, request), do: Tesla.request(client, request)
 
-  See `Pleroma.HTTP.request/5`
-  """
-  def get(url, headers \\ [], options \\ []),
-    do: request(:get, url, "", headers, options)
-
-  @doc """
-  Performs POST request.
-
-  See `Pleroma.HTTP.request/5`
-  """
-  def post(url, body, headers \\ [], options \\ []),
-    do: request(:post, url, body, headers, options)
+  defp build_request(method, headers, options, url, body, params) do
+    Builder.new()
+    |> Builder.method(method)
+    |> Builder.headers(headers)
+    |> Builder.opts(options)
+    |> Builder.url(url)
+    |> Builder.add_param(:body, :body, body)
+    |> Builder.add_param(:query, :query, params)
+    |> Builder.convert_to_keyword()
+  end
 end

lib/pleroma/http/request.ex

@@ -0,0 +1,23 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.HTTP.Request do
+  @moduledoc """
+  Request struct.
+  """
+  defstruct method: :get, url: "", query: [], headers: [], body: "", opts: []
+
+  @type method :: :head | :get | :delete | :trace | :options | :post | :put | :patch
+  @type url :: String.t()
+  @type headers :: [{String.t(), String.t()}]
+
+  @type t :: %__MODULE__{
+          method: method(),
+          url: url(),
+          query: keyword(),
+          headers: headers(),
+          body: String.t(),
+          opts: keyword()
+        }
+end

lib/pleroma/http/request_builder.ex

@@ -7,136 +7,87 @@ defmodule Pleroma.HTTP.RequestBuilder do
   Helper functions for building Tesla requests
   """
 
+  alias Pleroma.HTTP.Request
+  alias Tesla.Multipart
+
   @doc """
-  Specify the request method when building a request
-
-  ## Parameters
-
-  - request (Map) - Collected request options
-  - m (atom) - Request method
-
-  ## Returns
-
-  Map
+  Creates new request
   """
-  @spec method(map(), atom) :: map()
-  def method(request, m) do
-    Map.put_new(request, :method, m)
-  end
+  @spec new(Request.t()) :: Request.t()
+  def new(%Request{} = request \\ %Request{}), do: request
 
   @doc """
   Specify the request method when building a request
-
-  ## Parameters
-
-  - request (Map) - Collected request options
-  - u (String) - Request URL
-
-  ## Returns
-
-  Map
   """
-  @spec url(map(), String.t()) :: map()
-  def url(request, u) do
-    Map.put_new(request, :url, u)
-  end
+  @spec method(Request.t(), Request.method()) :: Request.t()
+  def method(request, m), do: %{request | method: m}
+
+  @doc """
+  Specify the request method when building a request
+  """
+  @spec url(Request.t(), Request.url()) :: Request.t()
+  def url(request, u), do: %{request | url: u}
 
   @doc """
   Add headers to the request
   """
-  @spec headers(map(), list(tuple)) :: map()
-  def headers(request, header_list) do
-    header_list =
+  @spec headers(Request.t(), Request.headers()) :: Request.t()
+  def headers(request, headers) do
+    headers_list =
       if Pleroma.Config.get([:http, :send_user_agent]) do
-        header_list ++ [{"User-Agent", Pleroma.Application.user_agent()}]
+        [{"user-agent", Pleroma.Application.user_agent()} | headers]
       else
-        header_list
+        headers
       end
 
-    Map.put_new(request, :headers, header_list)
+    %{request | headers: headers_list}
   end
 
   @doc """
   Add custom, per-request middleware or adapter options to the request
   """
-  @spec opts(map(), Keyword.t()) :: map()
-  def opts(request, options) do
-    Map.put_new(request, :opts, options)
-  end
+  @spec opts(Request.t(), keyword()) :: Request.t()
+  def opts(request, options), do: %{request | opts: options}
 
   @doc """
   Add optional parameters to the request
-
-  ## Parameters
-
-  - request (Map) - Collected request options
-  - definitions (Map) - Map of parameter name to parameter location.
-  - options (KeywordList) - The provided optional parameters
-
-  ## Returns
-
-  Map
   """
-  @spec add_optional_params(map(), %{optional(atom) => atom}, keyword()) :: map()
-  def add_optional_params(request, _, []), do: request
+  @spec add_param(Request.t(), atom(), atom(), any()) :: Request.t()
+  def add_param(request, :query, :query, values), do: %{request | query: values}
 
-  def add_optional_params(request, definitions, [{key, value} | tail]) do
-    case definitions do
-      %{^key => location} ->
-        request
-        |> add_param(location, key, value)
-        |> add_optional_params(definitions, tail)
-
-      _ ->
-        add_optional_params(request, definitions, tail)
-    end
-  end
-
-  @doc """
-  Add optional parameters to the request
-
-  ## Parameters
-
-  - request (Map) - Collected request options
-  - location (atom) - Where to put the parameter
-  - key (atom) - The name of the parameter
-  - value (any) - The value of the parameter
-
-  ## Returns
-
-  Map
-  """
-  @spec add_param(map(), atom, atom, any()) :: map()
-  def add_param(request, :query, :query, values), do: Map.put(request, :query, values)
-
-  def add_param(request, :body, :body, value), do: Map.put(request, :body, value)
+  def add_param(request, :body, :body, value), do: %{request | body: value}
 
   def add_param(request, :body, key, value) do
     request
-    |> Map.put_new_lazy(:body, &Tesla.Multipart.new/0)
+    |> Map.put(:body, Multipart.new())
     |> Map.update!(
       :body,
-      &Tesla.Multipart.add_field(
+      &Multipart.add_field(
         &1,
         key,
         Jason.encode!(value),
-        headers: [{:"Content-Type", "application/json"}]
+        headers: [{"content-type", "application/json"}]
       )
     )
   end
 
   def add_param(request, :file, name, path) do
     request
-    |> Map.put_new_lazy(:body, &Tesla.Multipart.new/0)
-    |> Map.update!(:body, &Tesla.Multipart.add_file(&1, path, name: name))
+    |> Map.put(:body, Multipart.new())
+    |> Map.update!(:body, &Multipart.add_file(&1, path, name: name))
   end
 
   def add_param(request, :form, name, value) do
-    request
-    |> Map.update(:body, %{name => value}, &Map.put(&1, name, value))
+    Map.update(request, :body, %{name => value}, &Map.put(&1, name, value))
   end
 
   def add_param(request, location, key, value) do
     Map.update(request, location, [{key, value}], &(&1 ++ [{key, value}]))
   end
+
+  def convert_to_keyword(request) do
+    request
+    |> Map.from_struct()
+    |> Enum.into([])
+  end
 end

lib/pleroma/maintenance.ex

@@ -0,0 +1,37 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Maintenance do
+  alias Pleroma.Repo
+  require Logger
+
+  def vacuum(args) do
+    case args do
+      "analyze" ->
+        Logger.info("Runnning VACUUM ANALYZE.")
+
+        Repo.query!(
+          "vacuum analyze;",
+          [],
+          timeout: :infinity
+        )
+
+      "full" ->
+        Logger.info("Runnning VACUUM FULL.")
+
+        Logger.warn(
+          "Re-packing your entire database may take a while and will consume extra disk space during the process."
+        )
+
+        Repo.query!(
+          "vacuum full;",
+          [],
+          timeout: :infinity
+        )
+
+      _ ->
+        Logger.error("Error: invalid vacuum argument.")
+    end
+  end
+end

lib/pleroma/marker.ex

@@ -9,24 +9,34 @@ defmodule Pleroma.Marker do
   import Ecto.Query
 
   alias Ecto.Multi
+  alias Pleroma.Notification
   alias Pleroma.Repo
   alias Pleroma.User
+  alias __MODULE__
 
   @timelines ["notifications"]
+  @type t :: %__MODULE__{}
 
   schema "markers" do
     field(:last_read_id, :string, default: "")
     field(:timeline, :string, default: "")
     field(:lock_version, :integer, default: 0)
+    field(:unread_count, :integer, default: 0, virtual: true)
 
     belongs_to(:user, User, type: FlakeId.Ecto.CompatType)
     timestamps()
   end
 
+  @doc "Gets markers by user and timeline."
+  @spec get_markers(User.t(), list(String)) :: list(t())
   def get_markers(user, timelines \\ []) do
-    Repo.all(get_query(user, timelines))
+    user
+    |> get_query(timelines)
+    |> unread_count_query()
+    |> Repo.all()
   end
 
+  @spec upsert(User.t(), map()) :: {:ok | :error, any()}
   def upsert(%User{} = user, attrs) do
     attrs
     |> Map.take(@timelines)
@@ -45,6 +55,27 @@ defmodule Pleroma.Marker do
     |> Repo.transaction()
   end
 
+  @spec multi_set_last_read_id(Multi.t(), User.t(), String.t()) :: Multi.t()
+  def multi_set_last_read_id(multi, %User{} = user, "notifications") do
+    multi
+    |> Multi.run(:counters, fn _repo, _changes ->
+      {:ok, %{last_read_id: Repo.one(Notification.last_read_query(user))}}
+    end)
+    |> Multi.insert(
+      :marker,
+      fn %{counters: attrs} ->
+        %Marker{timeline: "notifications", user_id: user.id}
+        |> struct(attrs)
+        |> Ecto.Changeset.change()
+      end,
+      returning: true,
+      on_conflict: {:replace, [:last_read_id]},
+      conflict_target: [:user_id, :timeline]
+    )
+  end
+
+  def multi_set_last_read_id(multi, _, _), do: multi
+
   defp get_marker(user, timeline) do
     case Repo.find_resource(get_query(user, timeline)) do
       {:ok, marker} -> %__MODULE__{marker | user: user}
@@ -71,4 +102,16 @@ defmodule Pleroma.Marker do
     |> by_user_id(user.id)
     |> by_timeline(timelines)
   end
+
+  defp unread_count_query(query) do
+    from(
+      q in query,
+      left_join: n in "notifications",
+      on: n.user_id == q.user_id and n.seen == false,
+      group_by: [:id],
+      select_merge: %{
+        unread_count: fragment("count(?)", n.id)
+      }
+    )
+  end
 end

lib/pleroma/mfa.ex

@@ -0,0 +1,155 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.MFA do
+  @moduledoc """
+  The MFA context.
+  """
+
+  alias Pleroma.User
+
+  alias Pleroma.MFA.BackupCodes
+  alias Pleroma.MFA.Changeset
+  alias Pleroma.MFA.Settings
+  alias Pleroma.MFA.TOTP
+
+  @doc """
+  Returns MFA methods the user has enabled.
+
+  ## Examples
+
+    iex> Pleroma.MFA.supported_method(User)
+    "totp, u2f"
+  """
+  @spec supported_methods(User.t()) :: String.t()
+  def supported_methods(user) do
+    settings = fetch_settings(user)
+
+    Settings.mfa_methods()
+    |> Enum.reduce([], fn m, acc ->
+      if method_enabled?(m, settings) do
+        acc ++ [m]
+      else
+        acc
+      end
+    end)
+    |> Enum.join(",")
+  end
+
+  @doc "Checks that user enabled MFA"
+  def require?(user) do
+    fetch_settings(user).enabled
+  end
+
+  @doc """
+  Display MFA settings of user
+  """
+  def mfa_settings(user) do
+    settings = fetch_settings(user)
+
+    Settings.mfa_methods()
+    |> Enum.map(fn m -> [m, method_enabled?(m, settings)] end)
+    |> Enum.into(%{enabled: settings.enabled}, fn [a, b] -> {a, b} end)
+  end
+
+  @doc false
+  def fetch_settings(%User{} = user) do
+    user.multi_factor_authentication_settings || %Settings{}
+  end
+
+  @doc "clears backup codes"
+  def invalidate_backup_code(%User{} = user, hash_code) do
+    %{backup_codes: codes} = fetch_settings(user)
+
+    user
+    |> Changeset.cast_backup_codes(codes -- [hash_code])
+    |> User.update_and_set_cache()
+  end
+
+  @doc "generates backup codes"
+  @spec generate_backup_codes(User.t()) :: {:ok, list(binary)} | {:error, String.t()}
+  def generate_backup_codes(%User{} = user) do
+    with codes <- BackupCodes.generate(),
+         hashed_codes <- Enum.map(codes, &Pbkdf2.hash_pwd_salt/1),
+         changeset <- Changeset.cast_backup_codes(user, hashed_codes),
+         {:ok, _} <- User.update_and_set_cache(changeset) do
+      {:ok, codes}
+    else
+      {:error, msg} ->
+        %{error: msg}
+    end
+  end
+
+  @doc """
+  Generates secret key and set delivery_type to 'app' for TOTP method.
+  """
+  @spec setup_totp(User.t()) :: {:ok, User.t()} | {:error, Ecto.Changeset.t()}
+  def setup_totp(user) do
+    user
+    |> Changeset.setup_totp(%{secret: TOTP.generate_secret(), delivery_type: "app"})
+    |> User.update_and_set_cache()
+  end
+
+  @doc """
+  Confirms the TOTP method for user.
+
+  `attrs`:
+    `password` - current user password
+    `code` - TOTP token
+  """
+  @spec confirm_totp(User.t(), map()) :: {:ok, User.t()} | {:error, Ecto.Changeset.t() | atom()}
+  def confirm_totp(%User{} = user, attrs) do
+    with settings <- user.multi_factor_authentication_settings.totp,
+         {:ok, :pass} <- TOTP.validate_token(settings.secret, attrs["code"]) do
+      user
+      |> Changeset.confirm_totp()
+      |> User.update_and_set_cache()
+    end
+  end
+
+  @doc """
+  Disables the TOTP method for user.
+
+  `attrs`:
+    `password` - current user password
+  """
+  @spec disable_totp(User.t()) :: {:ok, User.t()} | {:error, Ecto.Changeset.t()}
+  def disable_totp(%User{} = user) do
+    user
+    |> Changeset.disable_totp()
+    |> Changeset.disable()
+    |> User.update_and_set_cache()
+  end
+
+  @doc """
+  Force disables all MFA methods for user.
+  """
+  @spec disable(User.t()) :: {:ok, User.t()} | {:error, Ecto.Changeset.t()}
+  def disable(%User{} = user) do
+    user
+    |> Changeset.disable_totp()
+    |> Changeset.disable(true)
+    |> User.update_and_set_cache()
+  end
+
+  @doc """
+  Checks if the user has MFA method enabled.
+  """
+  def method_enabled?(method, settings) do
+    with {:ok, %{confirmed: true} = _} <- Map.fetch(settings, method) do
+      true
+    else
+      _ -> false
+    end
+  end
+
+  @doc """
+  Checks if the user has enabled at least one MFA method.
+  """
+  def enabled?(settings) do
+    Settings.mfa_methods()
+    |> Enum.map(fn m -> method_enabled?(m, settings) end)
+    |> Enum.any?()
+  end
+end

lib/pleroma/mfa/backup_codes.ex

@@ -0,0 +1,31 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.MFA.BackupCodes do
+  @moduledoc """
+  This module contains functions for generating backup codes.
+  """
+  alias Pleroma.Config
+
+  @config_ns [:instance, :multi_factor_authentication, :backup_codes]
+
+  @doc """
+  Generates backup codes.
+  """
+  @spec generate(Keyword.t()) :: list(String.t())
+  def generate(opts \\ []) do
+    number_of_codes = Keyword.get(opts, :number_of_codes, default_backup_codes_number())
+    code_length = Keyword.get(opts, :length, default_backup_codes_code_length())
+
+    Enum.map(1..number_of_codes, fn _ ->
+      :crypto.strong_rand_bytes(div(code_length, 2))
+      |> Base.encode16(case: :lower)
+    end)
+  end
+
+  defp default_backup_codes_number, do: Config.get(@config_ns ++ [:number], 5)
+
+  defp default_backup_codes_code_length,
+    do: Config.get(@config_ns ++ [:length], 16)
+end

lib/pleroma/mfa/changeset.ex

@@ -0,0 +1,64 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.MFA.Changeset do
+  alias Pleroma.MFA
+  alias Pleroma.MFA.Settings
+  alias Pleroma.User
+
+  def disable(%Ecto.Changeset{} = changeset, force \\ false) do
+    settings =
+      changeset
+      |> Ecto.Changeset.apply_changes()
+      |> MFA.fetch_settings()
+
+    if force || not MFA.enabled?(settings) do
+      put_change(changeset, %Settings{settings | enabled: false})
+    else
+      changeset
+    end
+  end
+
+  def disable_totp(%User{multi_factor_authentication_settings: settings} = user) do
+    user
+    |> put_change(%Settings{settings | totp: %Settings.TOTP{}})
+  end
+
+  def confirm_totp(%User{multi_factor_authentication_settings: settings} = user) do
+    totp_settings = %Settings.TOTP{settings.totp | confirmed: true}
+
+    user
+    |> put_change(%Settings{settings | totp: totp_settings, enabled: true})
+  end
+
+  def setup_totp(%User{} = user, attrs) do
+    mfa_settings = MFA.fetch_settings(user)
+
+    totp_settings =
+      %Settings.TOTP{}
+      |> Ecto.Changeset.cast(attrs, [:secret, :delivery_type])
+
+    user
+    |> put_change(%Settings{mfa_settings | totp: Ecto.Changeset.apply_changes(totp_settings)})
+  end
+
+  def cast_backup_codes(%User{} = user, codes) do
+    user
+    |> put_change(%Settings{
+      user.multi_factor_authentication_settings
+      | backup_codes: codes
+    })
+  end
+
+  defp put_change(%User{} = user, settings) do
+    user
+    |> Ecto.Changeset.change()
+    |> put_change(settings)
+  end
+
+  defp put_change(%Ecto.Changeset{} = changeset, settings) do
+    changeset
+    |> Ecto.Changeset.put_change(:multi_factor_authentication_settings, settings)
+  end
+end

lib/pleroma/mfa/settings.ex

@@ -0,0 +1,24 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.MFA.Settings do
+  use Ecto.Schema
+
+  @primary_key false
+
+  @mfa_methods [:totp]
+  embedded_schema do
+    field(:enabled, :boolean, default: false)
+    field(:backup_codes, {:array, :string}, default: [])
+
+    embeds_one :totp, TOTP, on_replace: :delete, primary_key: false do
+      field(:secret, :string)
+      # app | sms
+      field(:delivery_type, :string, default: "app")
+      field(:confirmed, :boolean, default: false)
+    end
+  end
+
+  def mfa_methods, do: @mfa_methods
+end

lib/pleroma/mfa/token.ex

@@ -0,0 +1,106 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.MFA.Token do
+  use Ecto.Schema
+  import Ecto.Query
+  import Ecto.Changeset
+
+  alias Pleroma.Repo
+  alias Pleroma.User
+  alias Pleroma.Web.OAuth.Authorization
+  alias Pleroma.Web.OAuth.Token, as: OAuthToken
+
+  @expires 300
+
+  schema "mfa_tokens" do
+    field(:token, :string)
+    field(:valid_until, :naive_datetime_usec)
+
+    belongs_to(:user, User, type: FlakeId.Ecto.CompatType)
+    belongs_to(:authorization, Authorization)
+
+    timestamps()
+  end
+
+  def get_by_token(token) do
+    from(
+      t in __MODULE__,
+      where: t.token == ^token,
+      preload: [:user, :authorization]
+    )
+    |> Repo.find_resource()
+  end
+
+  def validate(token) do
+    with {:fetch_token, {:ok, token}} <- {:fetch_token, get_by_token(token)},
+         {:expired, false} <- {:expired, is_expired?(token)} do
+      {:ok, token}
+    else
+      {:expired, _} -> {:error, :expired_token}
+      {:fetch_token, _} -> {:error, :not_found}
+      error -> {:error, error}
+    end
+  end
+
+  def create_token(%User{} = user) do
+    %__MODULE__{}
+    |> change
+    |> assign_user(user)
+    |> put_token
+    |> put_valid_until
+    |> Repo.insert()
+  end
+
+  def create_token(user, authorization) do
+    %__MODULE__{}
+    |> change
+    |> assign_user(user)
+    |> assign_authorization(authorization)
+    |> put_token
+    |> put_valid_until
+    |> Repo.insert()
+  end
+
+  defp assign_user(changeset, user) do
+    changeset
+    |> put_assoc(:user, user)
+    |> validate_required([:user])
+  end
+
+  defp assign_authorization(changeset, authorization) do
+    changeset
+    |> put_assoc(:authorization, authorization)
+    |> validate_required([:authorization])
+  end
+
+  defp put_token(changeset) do
+    changeset
+    |> change(%{token: OAuthToken.Utils.generate_token()})
+    |> validate_required([:token])
+    |> unique_constraint(:token)
+  end
+
+  defp put_valid_until(changeset) do
+    expires_in = NaiveDateTime.add(NaiveDateTime.utc_now(), @expires)
+
+    changeset
+    |> change(%{valid_until: expires_in})
+    |> validate_required([:valid_until])
+  end
+
+  def is_expired?(%__MODULE__{valid_until: valid_until}) do
+    NaiveDateTime.diff(NaiveDateTime.utc_now(), valid_until) > 0
+  end
+
+  def is_expired?(_), do: false
+
+  def delete_expired_tokens do
+    from(
+      q in __MODULE__,
+      where: fragment("?", q.valid_until) < ^Timex.now()
+    )
+    |> Repo.delete_all()
+  end
+end

lib/pleroma/mfa/totp.ex

@@ -0,0 +1,86 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.MFA.TOTP do
+  @moduledoc """
+  This module represents functions to create secrets for
+  TOTP Application as well as validate them with a time based token.
+  """
+  alias Pleroma.Config
+
+  @config_ns [:instance, :multi_factor_authentication, :totp]
+
+  @doc """
+  https://github.com/google/google-authenticator/wiki/Key-Uri-Format
+  """
+  def provisioning_uri(secret, label, opts \\ []) do
+    query =
+      %{
+        secret: secret,
+        issuer: Keyword.get(opts, :issuer, default_issuer()),
+        digits: Keyword.get(opts, :digits, default_digits()),
+        period: Keyword.get(opts, :period, default_period())
+      }
+      |> Enum.filter(fn {_, v} -> not is_nil(v) end)
+      |> Enum.into(%{})
+      |> URI.encode_query()
+
+    %URI{scheme: "otpauth", host: "totp", path: "/" <> label, query: query}
+    |> URI.to_string()
+  end
+
+  defp default_period, do: Config.get(@config_ns ++ [:period])
+  defp default_digits, do: Config.get(@config_ns ++ [:digits])
+
+  defp default_issuer,
+    do: Config.get(@config_ns ++ [:issuer], Config.get([:instance, :name]))
+
+  @doc "Creates a random Base 32 encoded string"
+  def generate_secret do
+    Base.encode32(:crypto.strong_rand_bytes(10))
+  end
+
+  @doc "Generates a valid token based on a secret"
+  def generate_token(secret) do
+    :pot.totp(secret)
+  end
+
+  @doc """
+  Validates a given token based on a secret.
+
+  optional parameters:
+  `token_length` default `6`
+  `interval_length` default `30`
+  `window` default 0
+
+  Returns {:ok, :pass} if the token is valid and
+  {:error, :invalid_token} if it is not.
+  """
+  @spec validate_token(String.t(), String.t()) ::
+          {:ok, :pass} | {:error, :invalid_token | :invalid_secret_and_token}
+  def validate_token(secret, token)
+      when is_binary(secret) and is_binary(token) do
+    opts = [
+      token_length: default_digits(),
+      interval_length: default_period()
+    ]
+
+    validate_token(secret, token, opts)
+  end
+
+  def validate_token(_, _), do: {:error, :invalid_secret_and_token}
+
+  @doc "See `validate_token/2`"
+  @spec validate_token(String.t(), String.t(), Keyword.t()) ::
+          {:ok, :pass} | {:error, :invalid_token | :invalid_secret_and_token}
+  def validate_token(secret, token, options)
+      when is_binary(secret) and is_binary(token) do
+    case :pot.valid_totp(token, secret, options) do
+      true -> {:ok, :pass}
+      false -> {:error, :invalid_token}
+    end
+  end
+
+  def validate_token(_, _, _), do: {:error, :invalid_secret_and_token}
+end

lib/pleroma/moderation_log.ex

@@ -605,6 +605,17 @@ defmodule Pleroma.ModerationLog do
     }"
   end
 
+  @spec get_log_entry_message(ModerationLog) :: String.t()
+  def get_log_entry_message(%ModerationLog{
+        data: %{
+          "actor" => %{"nickname" => actor_nickname},
+          "action" => "updated_users",
+          "subject" => subjects
+        }
+      }) do
+    "@#{actor_nickname} updated users: #{users_to_nicknames_string(subjects)}"
+  end
+
   defp nicknames_to_string(nicknames) do
     nicknames
     |> Enum.map(&"@#{&1}")

lib/pleroma/notification.ex

@@ -5,11 +5,15 @@
 defmodule Pleroma.Notification do
   use Ecto.Schema
 
+  alias Ecto.Multi
   alias Pleroma.Activity
+  alias Pleroma.FollowingRelationship
+  alias Pleroma.Marker
   alias Pleroma.Notification
   alias Pleroma.Object
   alias Pleroma.Pagination
   alias Pleroma.Repo
+  alias Pleroma.ThreadMute
   alias Pleroma.User
   alias Pleroma.Web.CommonAPI.Utils
   alias Pleroma.Web.Push
@@ -17,6 +21,7 @@ defmodule Pleroma.Notification do
 
   import Ecto.Query
   import Ecto.Changeset
+
   require Logger
 
   @type t :: %__MODULE__{}
@@ -31,17 +36,36 @@ defmodule Pleroma.Notification do
     timestamps()
   end
 
+  @spec unread_notifications_count(User.t()) :: integer()
+  def unread_notifications_count(%User{id: user_id}) do
+    from(q in __MODULE__,
+      where: q.user_id == ^user_id and q.seen == false
+    )
+    |> Repo.aggregate(:count, :id)
+  end
+
   def changeset(%Notification{} = notification, attrs) do
     notification
     |> cast(attrs, [:seen])
   end
 
+  @spec last_read_query(User.t()) :: Ecto.Queryable.t()
+  def last_read_query(user) do
+    from(q in Pleroma.Notification,
+      where: q.user_id == ^user.id,
+      where: q.seen == true,
+      select: type(q.id, :string),
+      limit: 1,
+      order_by: [desc: :id]
+    )
+  end
+
   defp for_user_query_ap_id_opts(user, opts) do
-    ap_id_relations =
+    ap_id_relationships =
       [:block] ++
         if opts[@include_muted_option], do: [], else: [:notification_mute]
 
-    preloaded_ap_ids = User.outgoing_relations_ap_ids(user, ap_id_relations)
+    preloaded_ap_ids = User.outgoing_relationships_ap_ids(user, ap_id_relationships)
 
     exclude_blocked_opts = Map.merge(%{blocked_users_ap_ids: preloaded_ap_ids[:block]}, opts)
 
@@ -68,8 +92,9 @@ defmodule Pleroma.Notification do
     |> join(:left, [n, a], object in Object,
       on:
         fragment(
-          "(?->>'id') = COALESCE((? -> 'object'::text) ->> 'id'::text)",
+          "(?->>'id') = COALESCE(?->'object'->>'id', ?->>'object')",
           object.data,
+          a.data,
           a.data
         )
     )
@@ -79,15 +104,13 @@ defmodule Pleroma.Notification do
     |> exclude_visibility(opts)
   end
 
+  # Excludes blocked users and non-followed domain-blocked users
   defp exclude_blocked(query, user, opts) do
     blocked_ap_ids = opts[:blocked_users_ap_ids] || User.blocked_users_ap_ids(user)
 
     query
     |> where([n, a], a.actor not in ^blocked_ap_ids)
-    |> where(
-      [n, a],
-      fragment("substring(? from '.*://([^/]*)')", a.actor) not in ^user.domain_blocks
-    )
+    |> FollowingRelationship.keep_following_or_not_domain_blocked(user)
   end
 
   defp exclude_notification_muted(query, _, %{@include_muted_option => true}) do
@@ -100,7 +123,7 @@ defmodule Pleroma.Notification do
 
     query
     |> where([n, a], a.actor not in ^notification_muted_ap_ids)
-    |> join(:left, [n, a], tm in Pleroma.ThreadMute,
+    |> join(:left, [n, a], tm in ThreadMute,
       on: tm.user_id == ^user.id and tm.context == fragment("?->>'context'", a.data)
     )
     |> where([n, a, o, tm], is_nil(tm.user_id))
@@ -184,46 +207,41 @@ defmodule Pleroma.Notification do
     |> Repo.all()
   end
 
-  def set_read_up_to(%{id: user_id} = _user, id) do
+  def set_read_up_to(%{id: user_id} = user, id) do
     query =
       from(
         n in Notification,
         where: n.user_id == ^user_id,
         where: n.id <= ^id,
         where: n.seen == false,
-        update: [
-          set: [
-            seen: true,
-            updated_at: ^NaiveDateTime.utc_now()
-          ]
-        ],
         # Ideally we would preload object and activities here
         # but Ecto does not support preloads in update_all
         select: n.id
       )
 
-    {_, notification_ids} = Repo.update_all(query, [])
+    {:ok, %{ids: {_, notification_ids}}} =
+      Multi.new()
+      |> Multi.update_all(:ids, query, set: [seen: true, updated_at: NaiveDateTime.utc_now()])
+      |> Marker.multi_set_last_read_id(user, "notifications")
+      |> Repo.transaction()
 
-    Notification
+    for_user_query(user)
     |> where([n], n.id in ^notification_ids)
-    |> join(:inner, [n], activity in assoc(n, :activity))
-    |> join(:left, [n, a], object in Object,
-      on:
-        fragment(
-          "(?->>'id') = COALESCE((? -> 'object'::text) ->> 'id'::text)",
-          object.data,
-          a.data
-        )
-    )
-    |> preload([n, a, o], activity: {a, object: o})
     |> Repo.all()
   end
 
+  @spec read_one(User.t(), String.t()) ::
+          {:ok, Notification.t()} | {:error, Ecto.Changeset.t()} | nil
   def read_one(%User{} = user, notification_id) do
     with {:ok, %Notification{} = notification} <- get(user, notification_id) do
-      notification
-      |> changeset(%{seen: true})
-      |> Repo.update()
+      Multi.new()
+      |> Multi.update(:update, changeset(notification, %{seen: true}))
+      |> Marker.multi_set_last_read_id(user, "notifications")
+      |> Repo.transaction()
+      |> case do
+        {:ok, %{update: notification}} -> {:ok, notification}
+        {:error, :update, changeset, _} -> {:error, changeset}
+      end
     end
   end
 
@@ -260,6 +278,16 @@ defmodule Pleroma.Notification do
     |> Repo.delete_all()
   end
 
+  def dismiss(%Pleroma.Activity{} = activity) do
+    Notification
+    |> where([n], n.activity_id == ^activity.id)
+    |> Repo.delete_all()
+    |> case do
+      {_, notifications} -> {:ok, notifications}
+      _ -> {:error, "Cannot dismiss notification"}
+    end
+  end
+
   def dismiss(%{id: user_id} = _user, id) do
     notification = Repo.get(Notification, id)
 
@@ -275,58 +303,160 @@ defmodule Pleroma.Notification do
   def create_notifications(%Activity{data: %{"to" => _, "type" => "Create"}} = activity) do
     object = Object.normalize(activity)
 
-    unless object && object.data["type"] == "Answer" do
-      users = get_notified_from_activity(activity)
-      notifications = Enum.map(users, fn user -> create_notification(activity, user) end)
-      {:ok, notifications}
-    else
+    if object && object.data["type"] == "Answer" do
       {:ok, []}
+    else
+      do_create_notifications(activity)
     end
   end
 
   def create_notifications(%Activity{data: %{"type" => type}} = activity)
-      when type in ["Like", "Announce", "Follow", "Move", "EmojiReact"] do
-    notifications =
-      activity
-      |> get_notified_from_activity()
-      |> Enum.map(&create_notification(activity, &1))
-
-    {:ok, notifications}
+      when type in ["Follow", "Like", "Announce", "Move", "EmojiReact"] do
+    do_create_notifications(activity)
   end
 
   def create_notifications(_), do: {:ok, []}
 
+  defp do_create_notifications(%Activity{} = activity) do
+    {enabled_receivers, disabled_receivers} = get_notified_from_activity(activity)
+    potential_receivers = enabled_receivers ++ disabled_receivers
+
+    notifications =
+      Enum.map(potential_receivers, fn user ->
+        do_send = user in enabled_receivers
+        create_notification(activity, user, do_send)
+      end)
+
+    {:ok, notifications}
+  end
+
   # TODO move to sql, too.
-  def create_notification(%Activity{} = activity, %User{} = user) do
+  def create_notification(%Activity{} = activity, %User{} = user, do_send \\ true) do
     unless skip?(activity, user) do
-      notification = %Notification{user_id: user.id, activity: activity}
-      {:ok, notification} = Repo.insert(notification)
+      {:ok, %{notification: notification}} =
+        Multi.new()
+        |> Multi.insert(:notification, %Notification{user_id: user.id, activity: activity})
+        |> Marker.multi_set_last_read_id(user, "notifications")
+        |> Repo.transaction()
 
-      ["user", "user:notification"]
-      |> Streamer.stream(notification)
+      if do_send do
+        Streamer.stream(["user", "user:notification"], notification)
+        Push.send(notification)
+      end
 
-      Push.send(notification)
       notification
     end
   end
 
+  @doc """
+  Returns a tuple with 2 elements:
+    {notification-enabled receivers, currently disabled receivers (blocking / [thread] muting)}
+
+  NOTE: might be called for FAKE Activities, see ActivityPub.Utils.get_notified_from_object/1
+  """
+  @spec get_notified_from_activity(Activity.t(), boolean()) :: {list(User.t()), list(User.t())}
   def get_notified_from_activity(activity, local_only \\ true)
 
   def get_notified_from_activity(%Activity{data: %{"type" => type}} = activity, local_only)
       when type in ["Create", "Like", "Announce", "Follow", "Move", "EmojiReact"] do
+    potential_receiver_ap_ids = get_potential_receiver_ap_ids(activity)
+
+    potential_receivers =
+      User.get_users_from_set(potential_receiver_ap_ids, local_only: local_only)
+
+    notification_enabled_ap_ids =
+      potential_receiver_ap_ids
+      |> exclude_domain_blocker_ap_ids(activity, potential_receivers)
+      |> exclude_relationship_restricted_ap_ids(activity)
+      |> exclude_thread_muter_ap_ids(activity)
+
+    notification_enabled_users =
+      Enum.filter(potential_receivers, fn u -> u.ap_id in notification_enabled_ap_ids end)
+
+    {notification_enabled_users, potential_receivers -- notification_enabled_users}
+  end
+
+  def get_notified_from_activity(_, _local_only), do: {[], []}
+
+  # For some activities, only notify the author of the object
+  def get_potential_receiver_ap_ids(%{data: %{"type" => type, "object" => object_id}})
+      when type in ~w{Like Announce EmojiReact} do
+    case Object.get_cached_by_ap_id(object_id) do
+      %Object{data: %{"actor" => actor}} ->
+        [actor]
+
+      _ ->
+        []
+    end
+  end
+
+  def get_potential_receiver_ap_ids(activity) do
     []
     |> Utils.maybe_notify_to_recipients(activity)
     |> Utils.maybe_notify_mentioned_recipients(activity)
     |> Utils.maybe_notify_subscribers(activity)
     |> Utils.maybe_notify_followers(activity)
     |> Enum.uniq()
-    |> User.get_users_from_set(local_only)
   end
 
-  def get_notified_from_activity(_, _local_only), do: []
+  @doc "Filters out AP IDs domain-blocking and not following the activity's actor"
+  def exclude_domain_blocker_ap_ids(ap_ids, activity, preloaded_users \\ [])
+
+  def exclude_domain_blocker_ap_ids([], _activity, _preloaded_users), do: []
+
+  def exclude_domain_blocker_ap_ids(ap_ids, %Activity{} = activity, preloaded_users) do
+    activity_actor_domain = activity.actor && URI.parse(activity.actor).host
+
+    users =
+      ap_ids
+      |> Enum.map(fn ap_id ->
+        Enum.find(preloaded_users, &(&1.ap_id == ap_id)) ||
+          User.get_cached_by_ap_id(ap_id)
+      end)
+      |> Enum.filter(& &1)
+
+    domain_blocker_ap_ids = for u <- users, activity_actor_domain in u.domain_blocks, do: u.ap_id
+
+    domain_blocker_follower_ap_ids =
+      if Enum.any?(domain_blocker_ap_ids) do
+        activity
+        |> Activity.user_actor()
+        |> FollowingRelationship.followers_ap_ids(domain_blocker_ap_ids)
+      else
+        []
+      end
+
+    ap_ids
+    |> Kernel.--(domain_blocker_ap_ids)
+    |> Kernel.++(domain_blocker_follower_ap_ids)
+  end
+
+  @doc "Filters out AP IDs of users basing on their relationships with activity actor user"
+  def exclude_relationship_restricted_ap_ids([], _activity), do: []
+
+  def exclude_relationship_restricted_ap_ids(ap_ids, %Activity{} = activity) do
+    relationship_restricted_ap_ids =
+      activity
+      |> Activity.user_actor()
+      |> User.incoming_relationships_ungrouped_ap_ids([
+        :block,
+        :notification_mute
+      ])
+
+    Enum.uniq(ap_ids) -- relationship_restricted_ap_ids
+  end
+
+  @doc "Filters out AP IDs of users who mute activity thread"
+  def exclude_thread_muter_ap_ids([], _activity), do: []
+
+  def exclude_thread_muter_ap_ids(ap_ids, %Activity{} = activity) do
+    thread_muter_ap_ids = ThreadMute.muter_ap_ids(activity.data["context"])
+
+    Enum.uniq(ap_ids) -- thread_muter_ap_ids
+  end
 
   @spec skip?(Activity.t(), User.t()) :: boolean()
-  def skip?(activity, user) do
+  def skip?(%Activity{} = activity, %User{} = user) do
     [
       :self,
       :followers,
@@ -335,18 +465,20 @@ defmodule Pleroma.Notification do
       :non_follows,
       :recently_followed
     ]
-    |> Enum.any?(&skip?(&1, activity, user))
+    |> Enum.find(&skip?(&1, activity, user))
   end
 
+  def skip?(_, _), do: false
+
   @spec skip?(atom(), Activity.t(), User.t()) :: boolean()
-  def skip?(:self, activity, user) do
+  def skip?(:self, %Activity{} = activity, %User{} = user) do
     activity.data["actor"] == user.ap_id
   end
 
   def skip?(
         :followers,
-        activity,
-        %{notification_settings: %{followers: false}} = user
+        %Activity{} = activity,
+        %User{notification_settings: %{followers: false}} = user
       ) do
     actor = activity.data["actor"]
     follower = User.get_cached_by_ap_id(actor)
@@ -355,15 +487,19 @@ defmodule Pleroma.Notification do
 
   def skip?(
         :non_followers,
-        activity,
-        %{notification_settings: %{non_followers: false}} = user
+        %Activity{} = activity,
+        %User{notification_settings: %{non_followers: false}} = user
       ) do
     actor = activity.data["actor"]
     follower = User.get_cached_by_ap_id(actor)
     !User.following?(follower, user)
   end
 
-  def skip?(:follows, activity, %{notification_settings: %{follows: false}} = user) do
+  def skip?(
+        :follows,
+        %Activity{} = activity,
+        %User{notification_settings: %{follows: false}} = user
+      ) do
     actor = activity.data["actor"]
     followed = User.get_cached_by_ap_id(actor)
     User.following?(user, followed)
@@ -371,15 +507,16 @@ defmodule Pleroma.Notification do
 
   def skip?(
         :non_follows,
-        activity,
-        %{notification_settings: %{non_follows: false}} = user
+        %Activity{} = activity,
+        %User{notification_settings: %{non_follows: false}} = user
       ) do
     actor = activity.data["actor"]
     followed = User.get_cached_by_ap_id(actor)
     !User.following?(user, followed)
   end
 
-  def skip?(:recently_followed, %{data: %{"type" => "Follow"}} = activity, user) do
+  # To do: consider defining recency in hours and checking FollowingRelationship with a single SQL
+  def skip?(:recently_followed, %Activity{data: %{"type" => "Follow"}} = activity, %User{} = user) do
     actor = activity.data["actor"]
 
     Notification.for_user(user)

lib/pleroma/object.ex

@@ -9,11 +9,13 @@ defmodule Pleroma.Object do
   import Ecto.Changeset
 
   alias Pleroma.Activity
+  alias Pleroma.Config
   alias Pleroma.Object
   alias Pleroma.Object.Fetcher
   alias Pleroma.ObjectTombstone
   alias Pleroma.Repo
   alias Pleroma.User
+  alias Pleroma.Workers.AttachmentsCleanupWorker
 
   require Logger
 
@@ -138,12 +140,17 @@ defmodule Pleroma.Object do
 
   def normalize(_, _, _), do: nil
 
-  # Owned objects can only be mutated by their owner
-  def authorize_mutation(%Object{data: %{"actor" => actor}}, %User{ap_id: ap_id}),
-    do: actor == ap_id
+  # Owned objects can only be accessed by their owner
+  def authorize_access(%Object{data: %{"actor" => actor}}, %User{ap_id: ap_id}) do
+    if actor == ap_id do
+      :ok
+    else
+      {:error, :forbidden}
+    end
+  end
 
-  # Legacy objects can be mutated by anybody
-  def authorize_mutation(%Object{}, %User{}), do: true
+  # Legacy objects can be accessed by anybody
+  def authorize_access(%Object{}, %User{}), do: :ok
 
   @spec get_cached_by_ap_id(String.t()) :: Object.t() | nil
   def get_cached_by_ap_id(ap_id) do
@@ -183,27 +190,37 @@ defmodule Pleroma.Object do
   def delete(%Object{data: %{"id" => id}} = object) do
     with {:ok, _obj} = swap_object_with_tombstone(object),
          deleted_activity = Activity.delete_all_by_object_ap_id(id),
-         {:ok, true} <- Cachex.del(:object_cache, "object:#{id}"),
-         {:ok, _} <- Cachex.del(:web_resp_cache, URI.parse(id).path) do
-      with true <- Pleroma.Config.get([:instance, :cleanup_attachments]) do
-        {:ok, _} =
-          Pleroma.Workers.AttachmentsCleanupWorker.enqueue("cleanup_attachments", %{
-            "object" => object
-          })
-      end
+         {:ok, _} <- invalid_object_cache(object) do
+      cleanup_attachments(
+        Config.get([:instance, :cleanup_attachments]),
+        %{"object" => object}
+      )
 
       {:ok, object, deleted_activity}
     end
   end
 
-  def prune(%Object{data: %{"id" => id}} = object) do
+  @spec cleanup_attachments(boolean(), %{required(:object) => map()}) ::
+          {:ok, Oban.Job.t() | nil}
+  def cleanup_attachments(true, %{"object" => _} = params) do
+    AttachmentsCleanupWorker.enqueue("cleanup_attachments", params)
+  end
+
+  def cleanup_attachments(_, _), do: {:ok, nil}
+
+  def prune(%Object{data: %{"id" => _id}} = object) do
     with {:ok, object} <- Repo.delete(object),
-         {:ok, true} <- Cachex.del(:object_cache, "object:#{id}"),
-         {:ok, _} <- Cachex.del(:web_resp_cache, URI.parse(id).path) do
+         {:ok, _} <- invalid_object_cache(object) do
       {:ok, object}
     end
   end
 
+  def invalid_object_cache(%Object{data: %{"id" => id}}) do
+    with {:ok, true} <- Cachex.del(:object_cache, "object:#{id}") do
+      Cachex.del(:web_resp_cache, URI.parse(id).path)
+    end
+  end
+
   def set_cache(%Object{data: %{"id" => ap_id}} = object) do
     Cachex.put(:object_cache, "object:#{ap_id}", object)
     {:ok, object}
@@ -261,7 +278,7 @@ defmodule Pleroma.Object do
     end
   end
 
-  def increase_vote_count(ap_id, name) do
+  def increase_vote_count(ap_id, name, actor) do
     with %Object{} = object <- Object.normalize(ap_id),
          "Question" <- object.data["type"] do
       multiple = Map.has_key?(object.data, "anyOf")
@@ -276,12 +293,15 @@ defmodule Pleroma.Object do
             option
         end)
 
+      voters = [actor | object.data["voters"] || []] |> Enum.uniq()
+
       data =
         if multiple do
           Map.put(object.data, "anyOf", options)
         else
           Map.put(object.data, "oneOf", options)
         end
+        |> Map.put("voters", voters)
 
       object
       |> Object.change(%{data: data})

lib/pleroma/object/containment.ex

@@ -32,6 +32,18 @@ defmodule Pleroma.Object.Containment do
     get_actor(%{"actor" => actor})
   end
 
+  def get_object(%{"object" => id}) when is_binary(id) do
+    id
+  end
+
+  def get_object(%{"object" => %{"id" => id}}) when is_binary(id) do
+    id
+  end
+
+  def get_object(_) do
+    nil
+  end
+
   # TODO: We explicitly allow 'tag' URIs through, due to references to legacy OStatus
   # objects being present in the test suite environment.  Once these objects are
   # removed, please also remove this.

lib/pleroma/object/fetcher.ex

@@ -141,7 +141,7 @@ defmodule Pleroma.Object.Fetcher do
         date: date
       })
 
-    [{:Signature, signature}]
+    [{"signature", signature}]
   end
 
   defp sign_fetch(headers, id, date) do
@@ -154,7 +154,7 @@ defmodule Pleroma.Object.Fetcher do
 
   defp maybe_date_fetch(headers, date) do
     if Pleroma.Config.get([:activitypub, :sign_object_fetches]) do
-      headers ++ [{:Date, date}]
+      headers ++ [{"date", date}]
     else
       headers
     end
@@ -166,7 +166,7 @@ defmodule Pleroma.Object.Fetcher do
     date = Pleroma.Signature.signed_date()
 
     headers =
-      [{:Accept, "application/activity+json"}]
+      [{"accept", "application/activity+json"}]
       |> maybe_date_fetch(date)
       |> sign_fetch(id, date)
 

lib/pleroma/otp_version.ex

@@ -0,0 +1,28 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.OTPVersion do
+  @spec version() :: String.t() | nil
+  def version do
+    # OTP Version https://erlang.org/doc/system_principles/versions.html#otp-version
+    [
+      Path.join(:code.root_dir(), "OTP_VERSION"),
+      Path.join([:code.root_dir(), "releases", :erlang.system_info(:otp_release), "OTP_VERSION"])
+    ]
+    |> get_version_from_files()
+  end
+
+  @spec get_version_from_files([Path.t()]) :: String.t() | nil
+  def get_version_from_files([]), do: nil
+
+  def get_version_from_files([path | paths]) do
+    if File.exists?(path) do
+      path
+      |> File.read!()
+      |> String.replace(~r/\r|\n|\s/, "")
+    else
+      get_version_from_files(paths)
+    end
+  end
+end

lib/pleroma/plugs/authentication_plug.ex

@@ -3,9 +3,11 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Plugs.AuthenticationPlug do
-  alias Comeonin.Pbkdf2
-  import Plug.Conn
+  alias Pleroma.Plugs.OAuthScopesPlug
   alias Pleroma.User
+
+  import Plug.Conn
+
   require Logger
 
   def init(options), do: options
@@ -14,8 +16,13 @@ defmodule Pleroma.Plugs.AuthenticationPlug do
     :crypt.crypt(password, password_hash) == password_hash
   end
 
+  def checkpw(password, "$2" <> _ = password_hash) do
+    # Handle bcrypt passwords for Mastodon migration
+    Bcrypt.verify_pass(password, password_hash)
+  end
+
   def checkpw(password, "$pbkdf2" <> _ = password_hash) do
-    Pbkdf2.checkpw(password, password_hash)
+    Pbkdf2.verify_pass(password, password_hash)
   end
 
   def checkpw(_password, _password_hash) do
@@ -23,6 +30,25 @@ defmodule Pleroma.Plugs.AuthenticationPlug do
     false
   end
 
+  def maybe_update_password(%User{password_hash: "$2" <> _} = user, password) do
+    do_update_password(user, password)
+  end
+
+  def maybe_update_password(%User{password_hash: "$6" <> _} = user, password) do
+    do_update_password(user, password)
+  end
+
+  def maybe_update_password(user, _), do: {:ok, user}
+
+  defp do_update_password(user, password) do
+    user
+    |> User.password_update_changeset(%{
+      "password" => password,
+      "password_confirmation" => password
+    })
+    |> Pleroma.Repo.update()
+  end
+
   def call(%{assigns: %{user: %User{}}} = conn, _), do: conn
 
   def call(
@@ -34,16 +60,19 @@ defmodule Pleroma.Plugs.AuthenticationPlug do
         } = conn,
         _
       ) do
-    if Pbkdf2.checkpw(password, password_hash) do
+    if checkpw(password, password_hash) do
+      {:ok, auth_user} = maybe_update_password(auth_user, password)
+
       conn
       |> assign(:user, auth_user)
+      |> OAuthScopesPlug.skip_plug()
     else
       conn
     end
   end
 
   def call(%{assigns: %{auth_credentials: %{password: _}}} = conn, _) do
-    Pbkdf2.dummy_checkpw()
+    Pbkdf2.no_user_verify()
     conn
   end
 

lib/pleroma/plugs/ensure_authenticated_plug.ex

@@ -5,32 +5,35 @@
 defmodule Pleroma.Plugs.EnsureAuthenticatedPlug do
   import Plug.Conn
   import Pleroma.Web.TranslationHelpers
+
   alias Pleroma.User
 
+  use Pleroma.Web, :plug
+
   def init(options) do
     options
   end
 
-  def call(%{assigns: %{user: %User{}}} = conn, _) do
+  @impl true
+  def perform(
+        %{
+          assigns: %{
+            auth_credentials: %{password: _},
+            user: %User{multi_factor_authentication_settings: %{enabled: true}}
+          }
+        } = conn,
+        _
+      ) do
+    conn
+    |> render_error(:forbidden, "Two-factor authentication enabled, you must use a access token.")
+    |> halt()
+  end
+
+  def perform(%{assigns: %{user: %User{}}} = conn, _) do
     conn
   end
 
-  def call(conn, options) do
-    perform =
-      cond do
-        options[:if_func] -> options[:if_func].()
-        options[:unless_func] -> !options[:unless_func].()
-        true -> true
-      end
-
-    if perform do
-      fail(conn)
-    else
-      conn
-    end
-  end
-
-  def fail(conn) do
+  def perform(conn, _) do
     conn
     |> render_error(:forbidden, "Invalid credentials.")
     |> halt()

lib/pleroma/plugs/ensure_public_or_authenticated_plug.ex

@@ -5,14 +5,18 @@
 defmodule Pleroma.Plugs.EnsurePublicOrAuthenticatedPlug do
   import Pleroma.Web.TranslationHelpers
   import Plug.Conn
+
   alias Pleroma.Config
   alias Pleroma.User
 
+  use Pleroma.Web, :plug
+
   def init(options) do
     options
   end
 
-  def call(conn, _) do
+  @impl true
+  def perform(conn, _) do
     public? = Config.get!([:instance, :public])
 
     case {public?, conn} do

lib/pleroma/plugs/expect_authenticated_check_plug.ex

@@ -0,0 +1,20 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Plugs.ExpectAuthenticatedCheckPlug do
+  @moduledoc """
+  Marks `Pleroma.Plugs.EnsureAuthenticatedPlug` as expected to be executed later in plug chain.
+
+  No-op plug which affects `Pleroma.Web` operation (is checked with `PlugHelper.plug_called?/2`).
+  """
+
+  use Pleroma.Web, :plug
+
+  def init(options), do: options
+
+  @impl true
+  def perform(conn, _) do
+    conn
+  end
+end

lib/pleroma/plugs/expect_public_or_authenticated_check_plug.ex

@@ -0,0 +1,21 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Plugs.ExpectPublicOrAuthenticatedCheckPlug do
+  @moduledoc """
+  Marks `Pleroma.Plugs.EnsurePublicOrAuthenticatedPlug` as expected to be executed later in plug
+  chain.
+
+  No-op plug which affects `Pleroma.Web` operation (is checked with `PlugHelper.plug_called?/2`).
+  """
+
+  use Pleroma.Web, :plug
+
+  def init(options), do: options
+
+  @impl true
+  def perform(conn, _) do
+    conn
+  end
+end

lib/pleroma/plugs/federating_plug.ex

@@ -19,6 +19,9 @@ defmodule Pleroma.Web.FederatingPlug do
 
   def federating?, do: Pleroma.Config.get([:instance, :federating])
 
+  # Definition for the use in :if_func / :unless_func plug options
+  def federating?(_conn), do: federating?()
+
   defp fail(conn) do
     conn
     |> put_status(404)

lib/pleroma/plugs/http_security_plug.ex

@@ -31,7 +31,7 @@ defmodule Pleroma.Plugs.HTTPSecurityPlug do
       {"x-content-type-options", "nosniff"},
       {"referrer-policy", referrer_policy},
       {"x-download-options", "noopen"},
-      {"content-security-policy", csp_string() <> ";"}
+      {"content-security-policy", csp_string()}
     ]
 
     if report_uri do
@@ -43,23 +43,46 @@ defmodule Pleroma.Plugs.HTTPSecurityPlug do
         ]
       }
 
-      headers ++ [{"reply-to", Jason.encode!(report_group)}]
+      [{"reply-to", Jason.encode!(report_group)} | headers]
     else
       headers
     end
   end
 
+  static_csp_rules = [
+    "default-src 'none'",
+    "base-uri 'self'",
+    "frame-ancestors 'none'",
+    "style-src 'self' 'unsafe-inline'",
+    "font-src 'self'",
+    "manifest-src 'self'"
+  ]
+
+  @csp_start [Enum.join(static_csp_rules, ";") <> ";"]
+
   defp csp_string do
     scheme = Config.get([Pleroma.Web.Endpoint, :url])[:scheme]
     static_url = Pleroma.Web.Endpoint.static_url()
     websocket_url = Pleroma.Web.Endpoint.websocket_url()
     report_uri = Config.get([:http_security, :report_uri])
 
-    connect_src = "connect-src 'self' #{static_url} #{websocket_url}"
+    img_src = "img-src 'self' data: blob:"
+    media_src = "media-src 'self'"
+
+    {img_src, media_src} =
+      if Config.get([:media_proxy, :enabled]) &&
+           !Config.get([:media_proxy, :proxy_opts, :redirect_on_failure]) do
+        sources = get_proxy_and_attachment_sources()
+        {[img_src, sources], [media_src, sources]}
+      else
+        {[img_src, " https:"], [media_src, " https:"]}
+      end
+
+    connect_src = ["connect-src 'self' blob: ", static_url, ?\s, websocket_url]
 
     connect_src =
       if Pleroma.Config.get(:env) == :dev do
-        connect_src <> " http://localhost:3035/"
+        [connect_src, " http://localhost:3035/"]
       else
         connect_src
       end
@@ -71,27 +94,46 @@ defmodule Pleroma.Plugs.HTTPSecurityPlug do
         "script-src 'self'"
       end
 
-    main_part = [
-      "default-src 'none'",
-      "base-uri 'self'",
-      "frame-ancestors 'none'",
-      "img-src 'self' data: https:",
-      "media-src 'self' https:",
-      "style-src 'self' 'unsafe-inline'",
-      "font-src 'self'",
-      "manifest-src 'self'",
-      connect_src,
-      script_src
-    ]
+    report = if report_uri, do: ["report-uri ", report_uri, ";report-to csp-endpoint"]
+    insecure = if scheme == "https", do: "upgrade-insecure-requests"
 
-    report = if report_uri, do: ["report-uri #{report_uri}; report-to csp-endpoint"], else: []
-
-    insecure = if scheme == "https", do: ["upgrade-insecure-requests"], else: []
-
-    (main_part ++ report ++ insecure)
-    |> Enum.join("; ")
+    @csp_start
+    |> add_csp_param(img_src)
+    |> add_csp_param(media_src)
+    |> add_csp_param(connect_src)
+    |> add_csp_param(script_src)
+    |> add_csp_param(insecure)
+    |> add_csp_param(report)
+    |> :erlang.iolist_to_binary()
   end
 
+  defp get_proxy_and_attachment_sources do
+    media_proxy_whitelist =
+      Enum.reduce(Config.get([:media_proxy, :whitelist]), [], fn host, acc ->
+        add_source(acc, host)
+      end)
+
+    upload_base_url =
+      if Config.get([Pleroma.Upload, :base_url]),
+        do: URI.parse(Config.get([Pleroma.Upload, :base_url])).host
+
+    s3_endpoint =
+      if Config.get([Pleroma.Upload, :uploader]) == Pleroma.Uploaders.S3,
+        do: URI.parse(Config.get([Pleroma.Uploaders.S3, :public_endpoint])).host
+
+    []
+    |> add_source(upload_base_url)
+    |> add_source(s3_endpoint)
+    |> add_source(media_proxy_whitelist)
+  end
+
+  defp add_source(iodata, nil), do: iodata
+  defp add_source(iodata, source), do: [[?\s, source] | iodata]
+
+  defp add_csp_param(csp_iodata, nil), do: csp_iodata
+
+  defp add_csp_param(csp_iodata, param), do: [[param, ?;] | csp_iodata]
+
   def warn_if_disabled do
     unless Config.get([:http_security, :enabled]) do
       Logger.warn("

lib/pleroma/plugs/instance_static.ex

@@ -3,6 +3,8 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Plugs.InstanceStatic do
+  require Pleroma.Constants
+
   @moduledoc """
   This is a shim to call `Plug.Static` but with runtime `from` configuration.
 
@@ -21,9 +23,6 @@ defmodule Pleroma.Plugs.InstanceStatic do
     end
   end
 
-  @only ~w(index.html robots.txt static emoji packs sounds images instance favicon.png sw.js
-  sw-pleroma.js)
-
   def init(opts) do
     opts
     |> Keyword.put(:from, "__unconfigured_instance_static_plug")
@@ -31,7 +30,7 @@ defmodule Pleroma.Plugs.InstanceStatic do
     |> Plug.Static.init()
   end
 
-  for only <- @only do
+  for only <- Pleroma.Constants.static_only_files() do
     at = Plug.Router.Utils.split("/")
 
     def call(%{request_path: "/" <> unquote(only) <> _} = conn, opts) do

lib/pleroma/plugs/legacy_authentication_plug.ex

@@ -4,6 +4,8 @@
 
 defmodule Pleroma.Plugs.LegacyAuthenticationPlug do
   import Plug.Conn
+
+  alias Pleroma.Plugs.OAuthScopesPlug
   alias Pleroma.User
 
   def init(options) do
@@ -27,6 +29,7 @@ defmodule Pleroma.Plugs.LegacyAuthenticationPlug do
       conn
       |> assign(:auth_user, user)
       |> assign(:user, user)
+      |> OAuthScopesPlug.skip_plug()
     else
       _ ->
         conn

lib/pleroma/plugs/mapped_signature_to_identity_plug.ex

@@ -13,8 +13,9 @@ defmodule Pleroma.Web.Plugs.MappedSignatureToIdentityPlug do
   def init(options), do: options
 
   defp key_id_from_conn(conn) do
-    with %{"keyId" => key_id} <- HTTPSignatures.signature_for_conn(conn) do
-      Signature.key_id_to_actor_id(key_id)
+    with %{"keyId" => key_id} <- HTTPSignatures.signature_for_conn(conn),
+         {:ok, ap_id} <- Signature.key_id_to_actor_id(key_id) do
+      ap_id
     else
       _ ->
         nil
@@ -42,13 +43,13 @@ defmodule Pleroma.Web.Plugs.MappedSignatureToIdentityPlug do
     else
       {:user_match, false} ->
         Logger.debug("Failed to map identity from signature (payload actor mismatch)")
-        Logger.debug("key_id=#{key_id_from_conn(conn)}, actor=#{actor}")
+        Logger.debug("key_id=#{inspect(key_id_from_conn(conn))}, actor=#{inspect(actor)}")
         assign(conn, :valid_signature, false)
 
       # remove me once testsuite uses mapped capabilities instead of what we do now
       {:user, nil} ->
         Logger.debug("Failed to map identity from signature (lookup failure)")
-        Logger.debug("key_id=#{key_id_from_conn(conn)}, actor=#{actor}")
+        Logger.debug("key_id=#{inspect(key_id_from_conn(conn))}, actor=#{actor}")
         conn
     end
   end
@@ -60,7 +61,7 @@ defmodule Pleroma.Web.Plugs.MappedSignatureToIdentityPlug do
     else
       _ ->
         Logger.debug("Failed to map identity from signature (no payload actor mismatch)")
-        Logger.debug("key_id=#{key_id_from_conn(conn)}")
+        Logger.debug("key_id=#{inspect(key_id_from_conn(conn))}")
         assign(conn, :valid_signature, false)
     end
   end

lib/pleroma/plugs/oauth_scopes_plug.ex

@@ -7,13 +7,13 @@ defmodule Pleroma.Plugs.OAuthScopesPlug do
   import Pleroma.Web.Gettext
 
   alias Pleroma.Config
-  alias Pleroma.Plugs.EnsurePublicOrAuthenticatedPlug
 
-  @behaviour Plug
+  use Pleroma.Web, :plug
 
   def init(%{scopes: _} = options), do: options
 
-  def call(%Plug.Conn{assigns: assigns} = conn, %{scopes: scopes} = options) do
+  @impl true
+  def perform(%Plug.Conn{assigns: assigns} = conn, %{scopes: scopes} = options) do
     op = options[:op] || :|
     token = assigns[:token]
 
@@ -28,10 +28,7 @@ defmodule Pleroma.Plugs.OAuthScopesPlug do
         conn
 
       options[:fallback] == :proceed_unauthenticated ->
-        conn
-        |> assign(:user, nil)
-        |> assign(:token, nil)
-        |> maybe_perform_instance_privacy_check(options)
+        drop_auth_info(conn)
 
       true ->
         missing_scopes = scopes -- matched_scopes
@@ -47,6 +44,15 @@ defmodule Pleroma.Plugs.OAuthScopesPlug do
     end
   end
 
+  @doc "Drops authentication info from connection"
+  def drop_auth_info(conn) do
+    # To simplify debugging, setting a private variable on `conn` if auth info is dropped
+    conn
+    |> put_private(:authentication_ignored, true)
+    |> assign(:user, nil)
+    |> assign(:token, nil)
+  end
+
   @doc "Filters descendants of supported scopes"
   def filter_descendants(scopes, supported_scopes) do
     Enum.filter(
@@ -68,12 +74,4 @@ defmodule Pleroma.Plugs.OAuthScopesPlug do
       scopes
     end
   end
-
-  defp maybe_perform_instance_privacy_check(%Plug.Conn{} = conn, options) do
-    if options[:skip_instance_privacy_check] do
-      conn
-    else
-      EnsurePublicOrAuthenticatedPlug.call(conn, [])
-    end
-  end
 end

lib/pleroma/plugs/plug_helper.ex

@@ -0,0 +1,40 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Plugs.PlugHelper do
+  @moduledoc "Pleroma Plug helper"
+
+  @called_plugs_list_id :called_plugs
+  def called_plugs_list_id, do: @called_plugs_list_id
+
+  @skipped_plugs_list_id :skipped_plugs
+  def skipped_plugs_list_id, do: @skipped_plugs_list_id
+
+  @doc "Returns `true` if specified plug was called."
+  def plug_called?(conn, plug_module) do
+    contained_in_private_list?(conn, @called_plugs_list_id, plug_module)
+  end
+
+  @doc "Returns `true` if specified plug was explicitly marked as skipped."
+  def plug_skipped?(conn, plug_module) do
+    contained_in_private_list?(conn, @skipped_plugs_list_id, plug_module)
+  end
+
+  @doc "Returns `true` if specified plug was either called or explicitly marked as skipped."
+  def plug_called_or_skipped?(conn, plug_module) do
+    plug_called?(conn, plug_module) || plug_skipped?(conn, plug_module)
+  end
+
+  # Appends plug to known list (skipped, called). Intended to be used from within plug code only.
+  def append_to_private_list(conn, list_id, value) do
+    list = conn.private[list_id] || []
+    modified_list = Enum.uniq(list ++ [value])
+    Plug.Conn.put_private(conn, list_id, modified_list)
+  end
+
+  defp contained_in_private_list?(conn, private_variable, value) do
+    list = conn.private[private_variable] || []
+    value in list
+  end
+end

lib/pleroma/plugs/rate_limiter/rate_limiter.ex

@@ -110,20 +110,9 @@ defmodule Pleroma.Plugs.RateLimiter do
   end
 
   def disabled?(conn) do
-    localhost_or_socket =
-      case Config.get([Pleroma.Web.Endpoint, :http, :ip]) do
-        {127, 0, 0, 1} -> true
-        {0, 0, 0, 0, 0, 0, 0, 1} -> true
-        {:local, _} -> true
-        _ -> false
-      end
-
-    remote_ip_not_found =
-      if Map.has_key?(conn.assigns, :remote_ip_found),
-        do: !conn.assigns.remote_ip_found,
-        else: false
-
-    localhost_or_socket and remote_ip_not_found
+    if Map.has_key?(conn.assigns, :remote_ip_found),
+      do: !conn.assigns.remote_ip_found,
+      else: false
   end
 
   @inspect_bucket_not_found {:error, :not_found}

lib/pleroma/plugs/remote_ip.ex

@@ -7,8 +7,6 @@ defmodule Pleroma.Plugs.RemoteIp do
   This is a shim to call [`RemoteIp`](https://git.pleroma.social/pleroma/remote_ip) but with runtime configuration.
   """
 
-  import Plug.Conn
-
   @behaviour Plug
 
   @headers ~w[
@@ -28,12 +26,11 @@ defmodule Pleroma.Plugs.RemoteIp do
 
   def init(_), do: nil
 
-  def call(%{remote_ip: original_remote_ip} = conn, _) do
+  def call(conn, _) do
     config = Pleroma.Config.get(__MODULE__, [])
 
     if Keyword.get(config, :enabled, false) do
-      %{remote_ip: new_remote_ip} = conn = RemoteIp.call(conn, remote_ip_opts(config))
-      assign(conn, :remote_ip_found, original_remote_ip != new_remote_ip)
+      RemoteIp.call(conn, remote_ip_opts(config))
     else
       conn
     end

lib/pleroma/plugs/uploaded_media.ex

@@ -41,6 +41,7 @@ defmodule Pleroma.Plugs.UploadedMedia do
         conn ->
           conn
       end
+      |> merge_resp_headers([{"content-security-policy", "sandbox"}])
 
     config = Pleroma.Config.get(Pleroma.Upload)
 

lib/pleroma/pool/connections.ex

@@ -0,0 +1,283 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Pool.Connections do
+  use GenServer
+
+  alias Pleroma.Config
+  alias Pleroma.Gun
+
+  require Logger
+
+  @type domain :: String.t()
+  @type conn :: Pleroma.Gun.Conn.t()
+
+  @type t :: %__MODULE__{
+          conns: %{domain() => conn()},
+          opts: keyword()
+        }
+
+  defstruct conns: %{}, opts: []
+
+  @spec start_link({atom(), keyword()}) :: {:ok, pid()}
+  def start_link({name, opts}) do
+    GenServer.start_link(__MODULE__, opts, name: name)
+  end
+
+  @impl true
+  def init(opts), do: {:ok, %__MODULE__{conns: %{}, opts: opts}}
+
+  @spec checkin(String.t() | URI.t(), atom()) :: pid() | nil
+  def checkin(url, name)
+  def checkin(url, name) when is_binary(url), do: checkin(URI.parse(url), name)
+
+  def checkin(%URI{} = uri, name) do
+    timeout = Config.get([:connections_pool, :checkin_timeout], 250)
+
+    GenServer.call(name, {:checkin, uri}, timeout)
+  end
+
+  @spec alive?(atom()) :: boolean()
+  def alive?(name) do
+    if pid = Process.whereis(name) do
+      Process.alive?(pid)
+    else
+      false
+    end
+  end
+
+  @spec get_state(atom()) :: t()
+  def get_state(name) do
+    GenServer.call(name, :state)
+  end
+
+  @spec count(atom()) :: pos_integer()
+  def count(name) do
+    GenServer.call(name, :count)
+  end
+
+  @spec get_unused_conns(atom()) :: [{domain(), conn()}]
+  def get_unused_conns(name) do
+    GenServer.call(name, :unused_conns)
+  end
+
+  @spec checkout(pid(), pid(), atom()) :: :ok
+  def checkout(conn, pid, name) do
+    GenServer.cast(name, {:checkout, conn, pid})
+  end
+
+  @spec add_conn(atom(), String.t(), Pleroma.Gun.Conn.t()) :: :ok
+  def add_conn(name, key, conn) do
+    GenServer.cast(name, {:add_conn, key, conn})
+  end
+
+  @spec remove_conn(atom(), String.t()) :: :ok
+  def remove_conn(name, key) do
+    GenServer.cast(name, {:remove_conn, key})
+  end
+
+  @impl true
+  def handle_cast({:add_conn, key, conn}, state) do
+    state = put_in(state.conns[key], conn)
+
+    Process.monitor(conn.conn)
+    {:noreply, state}
+  end
+
+  @impl true
+  def handle_cast({:checkout, conn_pid, pid}, state) do
+    state =
+      with true <- Process.alive?(conn_pid),
+           {key, conn} <- find_conn(state.conns, conn_pid),
+           used_by <- List.keydelete(conn.used_by, pid, 0) do
+        conn_state = if used_by == [], do: :idle, else: conn.conn_state
+
+        put_in(state.conns[key], %{conn | conn_state: conn_state, used_by: used_by})
+      else
+        false ->
+          Logger.debug("checkout for closed conn #{inspect(conn_pid)}")
+          state
+
+        nil ->
+          Logger.debug("checkout for alive conn #{inspect(conn_pid)}, but is not in state")
+          state
+      end
+
+    {:noreply, state}
+  end
+
+  @impl true
+  def handle_cast({:remove_conn, key}, state) do
+    state = put_in(state.conns, Map.delete(state.conns, key))
+    {:noreply, state}
+  end
+
+  @impl true
+  def handle_call({:checkin, uri}, from, state) do
+    key = "#{uri.scheme}:#{uri.host}:#{uri.port}"
+
+    case state.conns[key] do
+      %{conn: pid, gun_state: :up} = conn ->
+        time = :os.system_time(:second)
+        last_reference = time - conn.last_reference
+        crf = crf(last_reference, 100, conn.crf)
+
+        state =
+          put_in(state.conns[key], %{
+            conn
+            | last_reference: time,
+              crf: crf,
+              conn_state: :active,
+              used_by: [from | conn.used_by]
+          })
+
+        {:reply, pid, state}
+
+      %{gun_state: :down} ->
+        {:reply, nil, state}
+
+      nil ->
+        {:reply, nil, state}
+    end
+  end
+
+  @impl true
+  def handle_call(:state, _from, state), do: {:reply, state, state}
+
+  @impl true
+  def handle_call(:count, _from, state) do
+    {:reply, Enum.count(state.conns), state}
+  end
+
+  @impl true
+  def handle_call(:unused_conns, _from, state) do
+    unused_conns =
+      state.conns
+      |> Enum.filter(&filter_conns/1)
+      |> Enum.sort(&sort_conns/2)
+
+    {:reply, unused_conns, state}
+  end
+
+  defp filter_conns({_, %{conn_state: :idle, used_by: []}}), do: true
+  defp filter_conns(_), do: false
+
+  defp sort_conns({_, c1}, {_, c2}) do
+    c1.crf <= c2.crf and c1.last_reference <= c2.last_reference
+  end
+
+  @impl true
+  def handle_info({:gun_up, conn_pid, _protocol}, state) do
+    %{origin_host: host, origin_scheme: scheme, origin_port: port} = Gun.info(conn_pid)
+
+    host =
+      case :inet.ntoa(host) do
+        {:error, :einval} -> host
+        ip -> ip
+      end
+
+    key = "#{scheme}:#{host}:#{port}"
+
+    state =
+      with {key, conn} <- find_conn(state.conns, conn_pid, key),
+           {true, key} <- {Process.alive?(conn_pid), key} do
+        put_in(state.conns[key], %{
+          conn
+          | gun_state: :up,
+            conn_state: :active,
+            retries: 0
+        })
+      else
+        {false, key} ->
+          put_in(
+            state.conns,
+            Map.delete(state.conns, key)
+          )
+
+        nil ->
+          :ok = Gun.close(conn_pid)
+
+          state
+      end
+
+    {:noreply, state}
+  end
+
+  @impl true
+  def handle_info({:gun_down, conn_pid, _protocol, _reason, _killed}, state) do
+    retries = Config.get([:connections_pool, :retry], 1)
+    # we can't get info on this pid, because pid is dead
+    state =
+      with {key, conn} <- find_conn(state.conns, conn_pid),
+           {true, key} <- {Process.alive?(conn_pid), key} do
+        if conn.retries == retries do
+          :ok = Gun.close(conn.conn)
+
+          put_in(
+            state.conns,
+            Map.delete(state.conns, key)
+          )
+        else
+          put_in(state.conns[key], %{
+            conn
+            | gun_state: :down,
+              retries: conn.retries + 1
+          })
+        end
+      else
+        {false, key} ->
+          put_in(
+            state.conns,
+            Map.delete(state.conns, key)
+          )
+
+        nil ->
+          Logger.debug(":gun_down for conn which isn't found in state")
+
+          state
+      end
+
+    {:noreply, state}
+  end
+
+  @impl true
+  def handle_info({:DOWN, _ref, :process, conn_pid, reason}, state) do
+    Logger.debug("received DOWN message for #{inspect(conn_pid)} reason -> #{inspect(reason)}")
+
+    state =
+      with {key, conn} <- find_conn(state.conns, conn_pid) do
+        Enum.each(conn.used_by, fn {pid, _ref} ->
+          Process.exit(pid, reason)
+        end)
+
+        put_in(
+          state.conns,
+          Map.delete(state.conns, key)
+        )
+      else
+        nil ->
+          Logger.debug(":DOWN for conn which isn't found in state")
+
+          state
+      end
+
+    {:noreply, state}
+  end
+
+  defp find_conn(conns, conn_pid) do
+    Enum.find(conns, fn {_key, conn} ->
+      conn.conn == conn_pid
+    end)
+  end
+
+  defp find_conn(conns, conn_pid, conn_key) do
+    Enum.find(conns, fn {key, conn} ->
+      key == conn_key and conn.conn == conn_pid
+    end)
+  end
+
+  def crf(current, steps, crf) do
+    1 + :math.pow(0.5, current / steps) * crf
+  end
+end

lib/pleroma/pool/pool.ex

@@ -0,0 +1,22 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Pool do
+  def child_spec(opts) do
+    poolboy_opts =
+      opts
+      |> Keyword.put(:worker_module, Pleroma.Pool.Request)
+      |> Keyword.put(:name, {:local, opts[:name]})
+      |> Keyword.put(:size, opts[:size])
+      |> Keyword.put(:max_overflow, opts[:max_overflow])
+
+    %{
+      id: opts[:id] || {__MODULE__, make_ref()},
+      start: {:poolboy, :start_link, [poolboy_opts, [name: opts[:name]]]},
+      restart: :permanent,
+      shutdown: 5000,
+      type: :worker
+    }
+  end
+end

lib/pleroma/pool/request.ex

@@ -0,0 +1,65 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Pool.Request do
+  use GenServer
+
+  require Logger
+
+  def start_link(args) do
+    GenServer.start_link(__MODULE__, args)
+  end
+
+  @impl true
+  def init(_), do: {:ok, []}
+
+  @spec execute(pid() | atom(), Tesla.Client.t(), keyword(), pos_integer()) ::
+          {:ok, Tesla.Env.t()} | {:error, any()}
+  def execute(pid, client, request, timeout) do
+    GenServer.call(pid, {:execute, client, request}, timeout)
+  end
+
+  @impl true
+  def handle_call({:execute, client, request}, _from, state) do
+    response = Pleroma.HTTP.request(client, request)
+
+    {:reply, response, state}
+  end
+
+  @impl true
+  def handle_info({:gun_data, _conn, _stream, _, _}, state) do
+    {:noreply, state}
+  end
+
+  @impl true
+  def handle_info({:gun_up, _conn, _protocol}, state) do
+    {:noreply, state}
+  end
+
+  @impl true
+  def handle_info({:gun_down, _conn, _protocol, _reason, _killed}, state) do
+    {:noreply, state}
+  end
+
+  @impl true
+  def handle_info({:gun_error, _conn, _stream, _error}, state) do
+    {:noreply, state}
+  end
+
+  @impl true
+  def handle_info({:gun_push, _conn, _stream, _new_stream, _method, _uri, _headers}, state) do
+    {:noreply, state}
+  end
+
+  @impl true
+  def handle_info({:gun_response, _conn, _stream, _, _status, _headers}, state) do
+    {:noreply, state}
+  end
+
+  @impl true
+  def handle_info(msg, state) do
+    Logger.warn("Received unexpected message #{inspect(__MODULE__)} #{inspect(msg)}")
+    {:noreply, state}
+  end
+end

lib/pleroma/pool/supervisor.ex

@@ -0,0 +1,42 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Pool.Supervisor do
+  use Supervisor
+
+  alias Pleroma.Config
+  alias Pleroma.Pool
+
+  def start_link(args) do
+    Supervisor.start_link(__MODULE__, args, name: __MODULE__)
+  end
+
+  def init(_) do
+    conns_child = %{
+      id: Pool.Connections,
+      start:
+        {Pool.Connections, :start_link, [{:gun_connections, Config.get([:connections_pool])}]}
+    }
+
+    Supervisor.init([conns_child | pools()], strategy: :one_for_one)
+  end
+
+  defp pools do
+    pools = Config.get(:pools)
+
+    pools =
+      if Config.get([Pleroma.Upload, :proxy_remote]) == false do
+        Keyword.delete(pools, :upload)
+      else
+        pools
+      end
+
+    for {pool_name, pool_opts} <- pools do
+      pool_opts
+      |> Keyword.put(:id, {Pool, pool_name})
+      |> Keyword.put(:name, pool_name)
+      |> Pool.child_spec()
+    end
+  end
+end

lib/pleroma/reverse_proxy/client.ex

@@ -3,19 +3,23 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.ReverseProxy.Client do
-  @callback request(atom(), String.t(), [tuple()], String.t(), list()) ::
-              {:ok, pos_integer(), [tuple()], reference() | map()}
-              | {:ok, pos_integer(), [tuple()]}
+  @type status :: pos_integer()
+  @type header_name :: String.t()
+  @type header_value :: String.t()
+  @type headers :: [{header_name(), header_value()}]
+
+  @callback request(atom(), String.t(), headers(), String.t(), list()) ::
+              {:ok, status(), headers(), reference() | map()}
+              | {:ok, status(), headers()}
               | {:ok, reference()}
               | {:error, term()}
 
-  @callback stream_body(reference() | pid() | map()) ::
-              {:ok, binary()} | :done | {:error, String.t()}
+  @callback stream_body(map()) :: {:ok, binary(), map()} | :done | {:error, atom() | String.t()}
 
   @callback close(reference() | pid() | map()) :: :ok
 
-  def request(method, url, headers, "", opts \\ []) do
-    client().request(method, url, headers, "", opts)
+  def request(method, url, headers, body \\ "", opts \\ []) do
+    client().request(method, url, headers, body, opts)
   end
 
   def stream_body(ref), do: client().stream_body(ref)
@@ -23,6 +27,12 @@ defmodule Pleroma.ReverseProxy.Client do
   def close(ref), do: client().close(ref)
 
   defp client do
-    Pleroma.Config.get([Pleroma.ReverseProxy.Client], :hackney)
+    :tesla
+    |> Application.get_env(:adapter)
+    |> client()
   end
+
+  defp client(Tesla.Adapter.Hackney), do: Pleroma.ReverseProxy.Client.Hackney
+  defp client(Tesla.Adapter.Gun), do: Pleroma.ReverseProxy.Client.Tesla
+  defp client(_), do: Pleroma.Config.get!(Pleroma.ReverseProxy.Client)
 end

lib/pleroma/reverse_proxy/client/hackney.ex

@@ -0,0 +1,24 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.ReverseProxy.Client.Hackney do
+  @behaviour Pleroma.ReverseProxy.Client
+
+  @impl true
+  def request(method, url, headers, body, opts \\ []) do
+    :hackney.request(method, url, headers, body, opts)
+  end
+
+  @impl true
+  def stream_body(ref) do
+    case :hackney.stream_body(ref) do
+      :done -> :done
+      {:ok, data} -> {:ok, data, ref}
+      {:error, error} -> {:error, error}
+    end
+  end
+
+  @impl true
+  def close(ref), do: :hackney.close(ref)
+end

lib/pleroma/reverse_proxy/client/tesla.ex

@@ -0,0 +1,90 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.ReverseProxy.Client.Tesla do
+  @behaviour Pleroma.ReverseProxy.Client
+
+  @type headers() :: [{String.t(), String.t()}]
+  @type status() :: pos_integer()
+
+  @spec request(atom(), String.t(), headers(), String.t(), keyword()) ::
+          {:ok, status(), headers}
+          | {:ok, status(), headers, map()}
+          | {:error, atom() | String.t()}
+          | no_return()
+
+  @impl true
+  def request(method, url, headers, body, opts \\ []) do
+    check_adapter()
+
+    opts = Keyword.put(opts, :body_as, :chunks)
+
+    with {:ok, response} <-
+           Pleroma.HTTP.request(
+             method,
+             url,
+             body,
+             headers,
+             Keyword.put(opts, :adapter, opts)
+           ) do
+      if is_map(response.body) and method != :head do
+        {:ok, response.status, response.headers, response.body}
+      else
+        {:ok, response.status, response.headers}
+      end
+    else
+      {:error, error} -> {:error, error}
+    end
+  end
+
+  @impl true
+  @spec stream_body(map()) ::
+          {:ok, binary(), map()} | {:error, atom() | String.t()} | :done | no_return()
+  def stream_body(%{pid: pid, opts: opts, fin: true}) do
+    # if connection was reused, but in tesla were redirects,
+    # tesla returns new opened connection, which must be closed manually
+    if opts[:old_conn], do: Tesla.Adapter.Gun.close(pid)
+    # if there were redirects we need to checkout old conn
+    conn = opts[:old_conn] || opts[:conn]
+
+    if conn, do: :ok = Pleroma.Pool.Connections.checkout(conn, self(), :gun_connections)
+
+    :done
+  end
+
+  def stream_body(client) do
+    case read_chunk!(client) do
+      {:fin, body} ->
+        {:ok, body, Map.put(client, :fin, true)}
+
+      {:nofin, part} ->
+        {:ok, part, client}
+
+      {:error, error} ->
+        {:error, error}
+    end
+  end
+
+  defp read_chunk!(%{pid: pid, stream: stream, opts: opts}) do
+    adapter = check_adapter()
+    adapter.read_chunk(pid, stream, opts)
+  end
+
+  @impl true
+  @spec close(map) :: :ok | no_return()
+  def close(%{pid: pid}) do
+    adapter = check_adapter()
+    adapter.close(pid)
+  end
+
+  defp check_adapter do
+    adapter = Application.get_env(:tesla, :adapter)
+
+    unless adapter == Tesla.Adapter.Gun do
+      raise "#{adapter} doesn't support reading body in chunks"
+    end
+
+    adapter
+  end
+end

lib/pleroma/reverse_proxy/reverse_proxy.ex

@@ -3,8 +3,6 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.ReverseProxy do
-  alias Pleroma.HTTP
-
   @keep_req_headers ~w(accept user-agent accept-encoding cache-control if-modified-since) ++
                       ~w(if-unmodified-since if-none-match if-range range)
   @resp_cache_headers ~w(etag date last-modified)
@@ -58,10 +56,10 @@ defmodule Pleroma.ReverseProxy do
 
   * `req_headers`, `resp_headers` additional headers.
 
-  * `http`: options for [hackney](https://github.com/benoitc/hackney).
+  * `http`: options for [hackney](https://github.com/benoitc/hackney) or [gun](https://github.com/ninenines/gun).
 
   """
-  @default_hackney_options [pool: :media]
+  @default_options [pool: :media]
 
   @inline_content_types [
     "image/gif",
@@ -94,11 +92,7 @@ defmodule Pleroma.ReverseProxy do
   def call(_conn, _url, _opts \\ [])
 
   def call(conn = %{method: method}, url, opts) when method in @methods do
-    hackney_opts =
-      Pleroma.HTTP.Connection.hackney_options([])
-      |> Keyword.merge(@default_hackney_options)
-      |> Keyword.merge(Keyword.get(opts, :http, []))
-      |> HTTP.process_request_options()
+    client_opts = Keyword.merge(@default_options, Keyword.get(opts, :http, []))
 
     req_headers = build_req_headers(conn.req_headers, opts)
 
@@ -110,7 +104,7 @@ defmodule Pleroma.ReverseProxy do
       end
 
     with {:ok, nil} <- Cachex.get(:failed_proxy_url_cache, url),
-         {:ok, code, headers, client} <- request(method, url, req_headers, hackney_opts),
+         {:ok, code, headers, client} <- request(method, url, req_headers, client_opts),
          :ok <-
            header_length_constraint(
              headers,
@@ -156,11 +150,11 @@ defmodule Pleroma.ReverseProxy do
     |> halt()
   end
 
-  defp request(method, url, headers, hackney_opts) do
+  defp request(method, url, headers, opts) do
     Logger.debug("#{__MODULE__} #{method} #{url} #{inspect(headers)}")
     method = method |> String.downcase() |> String.to_existing_atom()
 
-    case client().request(method, url, headers, "", hackney_opts) do
+    case client().request(method, url, headers, "", opts) do
       {:ok, code, headers, client} when code in @valid_resp_codes ->
         {:ok, code, downcase_headers(headers), client}
 
@@ -210,7 +204,7 @@ defmodule Pleroma.ReverseProxy do
              duration,
              Keyword.get(opts, :max_read_duration, @max_read_duration)
            ),
-         {:ok, data} <- client().stream_body(client),
+         {:ok, data, client} <- client().stream_body(client),
          {:ok, duration} <- increase_read_duration(duration),
          sent_so_far = sent_so_far + byte_size(data),
          :ok <-

lib/pleroma/scheduled_activity.ex

@@ -40,7 +40,7 @@ defmodule Pleroma.ScheduledActivity do
          %{changes: %{params: %{"media_ids" => media_ids} = params}} = changeset
        )
        when is_list(media_ids) do
-    media_attachments = Utils.attachments_from_ids(%{"media_ids" => media_ids})
+    media_attachments = Utils.attachments_from_ids(%{media_ids: media_ids})
 
     params =
       params

lib/pleroma/signature.ex

@@ -8,6 +8,7 @@ defmodule Pleroma.Signature do
   alias Pleroma.Keys
   alias Pleroma.User
   alias Pleroma.Web.ActivityPub.ActivityPub
+  alias Pleroma.Web.ActivityPub.ObjectValidators.Types
 
   def key_id_to_actor_id(key_id) do
     uri =
@@ -21,12 +22,23 @@ defmodule Pleroma.Signature do
         uri
       end
 
-    URI.to_string(uri)
+    maybe_ap_id = URI.to_string(uri)
+
+    case Types.ObjectID.cast(maybe_ap_id) do
+      {:ok, ap_id} ->
+        {:ok, ap_id}
+
+      _ ->
+        case Pleroma.Web.WebFinger.finger(maybe_ap_id) do
+          %{"ap_id" => ap_id} -> {:ok, ap_id}
+          _ -> {:error, maybe_ap_id}
+        end
+    end
   end
 
   def fetch_public_key(conn) do
     with %{"keyId" => kid} <- HTTPSignatures.signature_for_conn(conn),
-         actor_id <- key_id_to_actor_id(kid),
+         {:ok, actor_id} <- key_id_to_actor_id(kid),
          {:ok, public_key} <- User.get_public_key_for_ap_id(actor_id) do
       {:ok, public_key}
     else
@@ -37,7 +49,7 @@ defmodule Pleroma.Signature do
 
   def refetch_public_key(conn) do
     with %{"keyId" => kid} <- HTTPSignatures.signature_for_conn(conn),
-         actor_id <- key_id_to_actor_id(kid),
+         {:ok, actor_id} <- key_id_to_actor_id(kid),
          {:ok, _user} <- ActivityPub.make_user_from_ap_id(actor_id),
          {:ok, public_key} <- User.get_public_key_for_ap_id(actor_id) do
       {:ok, public_key}

lib/pleroma/stats.ex

@@ -45,11 +45,11 @@ defmodule Pleroma.Stats do
   end
 
   def init(_args) do
-    {:ok, get_stat_data()}
+    {:ok, calculate_stat_data()}
   end
 
   def handle_call(:force_update, _from, _state) do
-    new_stats = get_stat_data()
+    new_stats = calculate_stat_data()
     {:reply, new_stats, new_stats}
   end
 
@@ -58,12 +58,12 @@ defmodule Pleroma.Stats do
   end
 
   def handle_cast(:run_update, _state) do
-    new_stats = get_stat_data()
+    new_stats = calculate_stat_data()
 
     {:noreply, new_stats}
   end
 
-  defp get_stat_data do
+  def calculate_stat_data do
     peers =
       from(
         u in User,
@@ -77,13 +77,21 @@ defmodule Pleroma.Stats do
 
     status_count = Repo.aggregate(User.Query.build(%{local: true}), :sum, :note_count)
 
-    user_count = Repo.aggregate(User.Query.build(%{local: true, active: true}), :count, :id)
+    users_query =
+      from(u in User,
+        where: u.deactivated != true,
+        where: u.local == true,
+        where: not is_nil(u.nickname),
+        where: not u.invisible
+      )
+
+    user_count = Repo.aggregate(users_query, :count, :id)
 
     %{
       peers: peers,
       stats: %{
         domain_count: domain_count,
-        status_count: status_count,
+        status_count: status_count || 0,
         user_count: user_count
       }
     }

lib/pleroma/tests/auth_test_controller.ex

@@ -0,0 +1,93 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+# A test controller reachable only in :test env.
+defmodule Pleroma.Tests.AuthTestController do
+  @moduledoc false
+
+  use Pleroma.Web, :controller
+
+  alias Pleroma.Plugs.EnsurePublicOrAuthenticatedPlug
+  alias Pleroma.Plugs.OAuthScopesPlug
+  alias Pleroma.User
+
+  # Serves only with proper OAuth token (:api and :authenticated_api)
+  # Skipping EnsurePublicOrAuthenticatedPlug has no effect in this case
+  #
+  # Suggested use case: all :authenticated_api endpoints (makes no sense for :api endpoints)
+  plug(OAuthScopesPlug, %{scopes: ["read"]} when action == :do_oauth_check)
+
+  # Via :api, keeps :user if token has requested scopes (if :user is dropped, serves if public)
+  # Via :authenticated_api, serves if token is present and has requested scopes
+  #
+  # Suggested use case: vast majority of :api endpoints (no sense for :authenticated_api ones)
+  plug(
+    OAuthScopesPlug,
+    %{scopes: ["read"], fallback: :proceed_unauthenticated}
+    when action == :fallback_oauth_check
+  )
+
+  # Keeps :user if present, executes regardless of token / token scopes
+  # Fails with no :user for :authenticated_api / no user for :api on private instance
+  # Note: EnsurePublicOrAuthenticatedPlug is not skipped (private instance fails on no :user)
+  # Note: Basic Auth processing results in :skip_plug call for OAuthScopesPlug
+  #
+  # Suggested use: suppressing OAuth checks for other auth mechanisms (like Basic Auth)
+  # For controller-level use, see :skip_oauth_skip_publicity_check instead
+  plug(
+    :skip_plug,
+    OAuthScopesPlug when action == :skip_oauth_check
+  )
+
+  # (Shouldn't be executed since the plug is skipped)
+  plug(OAuthScopesPlug, %{scopes: ["admin"]} when action == :skip_oauth_check)
+
+  # Via :api, keeps :user if token has requested scopes, and continues with nil :user otherwise
+  # Via :authenticated_api, serves if token is present and has requested scopes
+  #
+  # Suggested use: as :fallback_oauth_check but open with nil :user for :api on private instances
+  plug(
+    :skip_plug,
+    EnsurePublicOrAuthenticatedPlug when action == :fallback_oauth_skip_publicity_check
+  )
+
+  plug(
+    OAuthScopesPlug,
+    %{scopes: ["read"], fallback: :proceed_unauthenticated}
+    when action == :fallback_oauth_skip_publicity_check
+  )
+
+  # Via :api, keeps :user if present, serves regardless of token presence / scopes / :user presence
+  # Via :authenticated_api, serves if :user is set (regardless of token presence and its scopes)
+  #
+  # Suggested use: making an :api endpoint always accessible (e.g. email confirmation endpoint)
+  plug(
+    :skip_plug,
+    [OAuthScopesPlug, EnsurePublicOrAuthenticatedPlug]
+    when action == :skip_oauth_skip_publicity_check
+  )
+
+  # Via :authenticated_api, always fails with 403 (endpoint is insecure)
+  # Via :api, drops :user if present and serves if public (private instance rejects on no user)
+  #
+  # Suggested use: none; please define OAuth rules for all :api / :authenticated_api endpoints
+  plug(:skip_plug, [] when action == :missing_oauth_check_definition)
+
+  def do_oauth_check(conn, _params), do: conn_state(conn)
+
+  def fallback_oauth_check(conn, _params), do: conn_state(conn)
+
+  def skip_oauth_check(conn, _params), do: conn_state(conn)
+
+  def fallback_oauth_skip_publicity_check(conn, _params), do: conn_state(conn)
+
+  def skip_oauth_skip_publicity_check(conn, _params), do: conn_state(conn)
+
+  def missing_oauth_check_definition(conn, _params), do: conn_state(conn)
+
+  defp conn_state(%{assigns: %{user: %User{} = user}} = conn),
+    do: json(conn, %{user_id: user.id})
+
+  defp conn_state(conn), do: json(conn, %{user_id: nil})
+end

lib/pleroma/thread_mute.ex

@@ -9,7 +9,8 @@ defmodule Pleroma.ThreadMute do
   alias Pleroma.ThreadMute
   alias Pleroma.User
 
-  require Ecto.Query
+  import Ecto.Changeset
+  import Ecto.Query
 
   schema "thread_mutes" do
     belongs_to(:user, User, type: FlakeId.Ecto.CompatType)
@@ -18,19 +19,44 @@ defmodule Pleroma.ThreadMute do
 
   def changeset(mute, params \\ %{}) do
     mute
-    |> Ecto.Changeset.cast(params, [:user_id, :context])
-    |> Ecto.Changeset.foreign_key_constraint(:user_id)
-    |> Ecto.Changeset.unique_constraint(:user_id, name: :unique_index)
+    |> cast(params, [:user_id, :context])
+    |> foreign_key_constraint(:user_id)
+    |> unique_constraint(:user_id, name: :unique_index)
   end
 
   def query(user_id, context) do
-    {:ok, user_id} = FlakeId.Ecto.CompatType.dump(user_id)
+    user_binary_id = User.binary_id(user_id)
 
     ThreadMute
-    |> Ecto.Query.where(user_id: ^user_id)
-    |> Ecto.Query.where(context: ^context)
+    |> where(user_id: ^user_binary_id)
+    |> where(context: ^context)
   end
 
+  def muters_query(context) do
+    ThreadMute
+    |> join(:inner, [tm], u in assoc(tm, :user))
+    |> where([tm], tm.context == ^context)
+    |> select([tm, u], u.ap_id)
+  end
+
+  def muter_ap_ids(context, ap_ids \\ nil)
+
+  # Note: applies to fake activities (ActivityPub.Utils.get_notified_from_object/1 etc.)
+  def muter_ap_ids(context, _ap_ids) when is_nil(context), do: []
+
+  def muter_ap_ids(context, ap_ids) do
+    context
+    |> muters_query()
+    |> maybe_filter_on_ap_id(ap_ids)
+    |> Repo.all()
+  end
+
+  defp maybe_filter_on_ap_id(query, ap_ids) when is_list(ap_ids) do
+    where(query, [tm, u], u.ap_id in ^ap_ids)
+  end
+
+  defp maybe_filter_on_ap_id(query, _ap_ids), do: query
+
   def add_mute(user_id, context) do
     %ThreadMute{}
     |> changeset(%{user_id: user_id, context: context})
@@ -42,8 +68,8 @@ defmodule Pleroma.ThreadMute do
     |> Repo.delete_all()
   end
 
-  def check_muted(user_id, context) do
+  def exists?(user_id, context) do
     query(user_id, context)
-    |> Repo.all()
+    |> Repo.exists?()
   end
 end

lib/pleroma/upload.ex

@@ -134,7 +134,7 @@ defmodule Pleroma.Upload do
     end
   end
 
-  defp prepare_upload(%{"img" => "data:image/" <> image_data}, opts) do
+  defp prepare_upload(%{img: "data:image/" <> image_data}, opts) do
     parsed = Regex.named_captures(~r/(?<filetype>jpeg|png|gif);base64,(?<data>.*)/, image_data)
     data = Base.decode64!(parsed["data"], ignore: :whitespace)
     hash = String.downcase(Base.encode16(:crypto.hash(:sha256, data)))

lib/pleroma/user.ex

@@ -9,15 +9,17 @@ defmodule Pleroma.User do
   import Ecto.Query
   import Ecto, only: [assoc: 2]
 
-  alias Comeonin.Pbkdf2
   alias Ecto.Multi
   alias Pleroma.Activity
   alias Pleroma.Config
   alias Pleroma.Conversation.Participation
   alias Pleroma.Delivery
+  alias Pleroma.Emoji
   alias Pleroma.FollowingRelationship
+  alias Pleroma.Formatter
   alias Pleroma.HTML
   alias Pleroma.Keys
+  alias Pleroma.MFA
   alias Pleroma.Notification
   alias Pleroma.Object
   alias Pleroma.Registration
@@ -27,6 +29,9 @@ defmodule Pleroma.User do
   alias Pleroma.UserRelationship
   alias Pleroma.Web
   alias Pleroma.Web.ActivityPub.ActivityPub
+  alias Pleroma.Web.ActivityPub.Builder
+  alias Pleroma.Web.ActivityPub.ObjectValidators.Types
+  alias Pleroma.Web.ActivityPub.Pipeline
   alias Pleroma.Web.ActivityPub.Utils
   alias Pleroma.Web.CommonAPI
   alias Pleroma.Web.CommonAPI.Utils, as: CommonUtils
@@ -81,6 +86,7 @@ defmodule Pleroma.User do
     field(:password, :string, virtual: true)
     field(:password_confirmation, :string, virtual: true)
     field(:keys, :string)
+    field(:public_key, :string)
     field(:ap_id, :string)
     field(:avatar, :map)
     field(:local, :boolean, default: true)
@@ -93,7 +99,6 @@ defmodule Pleroma.User do
     field(:last_digest_emailed_at, :naive_datetime)
     field(:banner, :map, default: %{})
     field(:background, :map, default: %{})
-    field(:source_data, :map, default: %{})
     field(:note_count, :integer, default: 0)
     field(:follower_count, :integer, default: 0)
     field(:following_count, :integer, default: 0)
@@ -110,8 +115,7 @@ defmodule Pleroma.User do
     field(:is_admin, :boolean, default: false)
     field(:show_role, :boolean, default: true)
     field(:settings, :map, default: nil)
-    field(:magic_key, :string, default: nil)
-    field(:uri, :string, default: nil)
+    field(:uri, Types.Uri, default: nil)
     field(:hide_followers_count, :boolean, default: false)
     field(:hide_follows_count, :boolean, default: false)
     field(:hide_followers, :boolean, default: false)
@@ -121,7 +125,7 @@ defmodule Pleroma.User do
     field(:pinned_activities, {:array, :string}, default: [])
     field(:email_notifications, :map, default: %{"digest" => false})
     field(:mascot, :map, default: nil)
-    field(:emoji, {:array, :map}, default: [])
+    field(:emoji, :map, default: %{})
     field(:pleroma_settings_store, :map, default: %{})
     field(:fields, {:array, :map}, default: [])
     field(:raw_fields, {:array, :map}, default: [])
@@ -131,6 +135,8 @@ defmodule Pleroma.User do
     field(:skip_thread_containment, :boolean, default: false)
     field(:actor_type, :string, default: "Person")
     field(:also_known_as, {:array, :string}, default: [])
+    field(:inbox, :string)
+    field(:shared_inbox, :string)
 
     embeds_one(
       :notification_settings,
@@ -150,22 +156,26 @@ defmodule Pleroma.User do
            {outgoing_relation, outgoing_relation_target},
            {incoming_relation, incoming_relation_source}
          ]} <- @user_relationships_config do
-      # Definitions of `has_many :blocker_blocks`, `has_many :muter_mutes` etc.
+      # Definitions of `has_many` relations: :blocker_blocks, :muter_mutes, :reblog_muter_mutes,
+      #   :notification_muter_mutes, :subscribee_subscriptions
       has_many(outgoing_relation, UserRelationship,
         foreign_key: :source_id,
         where: [relationship_type: relationship_type]
       )
 
-      # Definitions of `has_many :blockee_blocks`, `has_many :mutee_mutes` etc.
+      # Definitions of `has_many` relations: :blockee_blocks, :mutee_mutes, :reblog_mutee_mutes,
+      #   :notification_mutee_mutes, :subscriber_subscriptions
       has_many(incoming_relation, UserRelationship,
         foreign_key: :target_id,
         where: [relationship_type: relationship_type]
       )
 
-      # Definitions of `has_many :blocked_users`, `has_many :muted_users` etc.
+      # Definitions of `has_many` relations: :blocked_users, :muted_users, :reblog_muted_users,
+      #   :notification_muted_users, :subscriber_users
       has_many(outgoing_relation_target, through: [outgoing_relation, :target])
 
-      # Definitions of `has_many :blocker_users`, `has_many :muter_users` etc.
+      # Definitions of `has_many` relations: :blocker_users, :muter_users, :reblog_muter_users,
+      #   :notification_muter_users, :subscribee_users
       has_many(incoming_relation_source, through: [incoming_relation, :source])
     end
 
@@ -180,12 +190,20 @@ defmodule Pleroma.User do
     # `:subscribers` is deprecated (replaced with `subscriber_users` relation)
     field(:subscribers, {:array, :string}, default: [])
 
+    embeds_one(
+      :multi_factor_authentication_settings,
+      MFA.Settings,
+      on_replace: :delete
+    )
+
     timestamps()
   end
 
   for {_relationship_type, [{_outgoing_relation, outgoing_relation_target}, _]} <-
         @user_relationships_config do
-    # Definitions of `blocked_users_relation/1`, `muted_users_relation/1`, etc.
+    # `def blocked_users_relation/2`, `def muted_users_relation/2`,
+    #   `def reblog_muted_users_relation/2`, `def notification_muted_users/2`,
+    #   `def subscriber_users/2`
     def unquote(:"#{outgoing_relation_target}_relation")(user, restrict_deactivated? \\ false) do
       target_users_query = assoc(user, unquote(outgoing_relation_target))
 
@@ -196,7 +214,8 @@ defmodule Pleroma.User do
       end
     end
 
-    # Definitions of `blocked_users/1`, `muted_users/1`, etc.
+    # `def blocked_users/2`, `def muted_users/2`, `def reblog_muted_users/2`,
+    #   `def notification_muted_users/2`, `def subscriber_users/2`
     def unquote(outgoing_relation_target)(user, restrict_deactivated? \\ false) do
       __MODULE__
       |> apply(unquote(:"#{outgoing_relation_target}_relation"), [
@@ -206,7 +225,8 @@ defmodule Pleroma.User do
       |> Repo.all()
     end
 
-    # Definitions of `blocked_users_ap_ids/1`, `muted_users_ap_ids/1`, etc.
+    # `def blocked_users_ap_ids/2`, `def muted_users_ap_ids/2`, `def reblog_muted_users_ap_ids/2`,
+    #   `def notification_muted_users_ap_ids/2`, `def subscriber_users_ap_ids/2`
     def unquote(:"#{outgoing_relation_target}_ap_ids")(user, restrict_deactivated? \\ false) do
       __MODULE__
       |> apply(unquote(:"#{outgoing_relation_target}_relation"), [
@@ -218,6 +238,24 @@ defmodule Pleroma.User do
     end
   end
 
+  @doc """
+  Dumps Flake Id to SQL-compatible format (16-byte UUID).
+  E.g. "9pQtDGXuq4p3VlcJEm" -> <<0, 0, 1, 110, 179, 218, 42, 92, 213, 41, 44, 227, 95, 213, 0, 0>>
+  """
+  def binary_id(source_id) when is_binary(source_id) do
+    with {:ok, dumped_id} <- FlakeId.Ecto.CompatType.dump(source_id) do
+      dumped_id
+    else
+      _ -> source_id
+    end
+  end
+
+  def binary_id(source_ids) when is_list(source_ids) do
+    Enum.map(source_ids, &binary_id/1)
+  end
+
+  def binary_id(%User{} = user), do: binary_id(user.id)
+
   @doc "Returns status account"
   @spec account_status(User.t()) :: account_status()
   def account_status(%User{deactivated: true}), do: :deactivated
@@ -267,8 +305,13 @@ defmodule Pleroma.User do
 
   def avatar_url(user, options \\ []) do
     case user.avatar do
-      %{"url" => [%{"href" => href} | _]} -> href
-      _ -> !options[:no_default] && "#{Web.base_url()}/images/avi.png"
+      %{"url" => [%{"href" => href} | _]} ->
+        href
+
+      _ ->
+        unless options[:no_default] do
+          Config.get([:assets, :default_user_avatar], "#{Web.base_url()}/images/avi.png")
+        end
     end
   end
 
@@ -279,37 +322,16 @@ defmodule Pleroma.User do
     end
   end
 
-  def profile_url(%User{source_data: %{"url" => url}}), do: url
-  def profile_url(%User{ap_id: ap_id}), do: ap_id
-  def profile_url(_), do: nil
-
+  # Should probably be renamed or removed
   def ap_id(%User{nickname: nickname}), do: "#{Web.base_url()}/users/#{nickname}"
 
   def ap_followers(%User{follower_address: fa}) when is_binary(fa), do: fa
   def ap_followers(%User{} = user), do: "#{ap_id(user)}/followers"
 
-  @spec ap_following(User.t()) :: Sring.t()
+  @spec ap_following(User.t()) :: String.t()
   def ap_following(%User{following_address: fa}) when is_binary(fa), do: fa
   def ap_following(%User{} = user), do: "#{ap_id(user)}/following"
 
-  def follow_state(%User{} = user, %User{} = target) do
-    case Utils.fetch_latest_follow(user, target) do
-      %{data: %{"state" => state}} -> state
-      # Ideally this would be nil, but then Cachex does not commit the value
-      _ -> false
-    end
-  end
-
-  def get_cached_follow_state(user, target) do
-    key = "follow_state:#{user.ap_id}|#{target.ap_id}"
-    Cachex.fetch!(:user_cache, key, fn _ -> {:commit, follow_state(user, target)} end)
-  end
-
-  @spec set_follow_state_cache(String.t(), String.t(), String.t()) :: {:ok | :error, boolean()}
-  def set_follow_state_cache(user_ap_id, target_ap_id, state) do
-    Cachex.put(:user_cache, "follow_state:#{user_ap_id}|#{target_ap_id}", state)
-  end
-
   @spec restrict_deactivated(Ecto.Query.t()) :: Ecto.Query.t()
   def restrict_deactivated(query) do
     from(u in query, where: u.deactivated != ^true)
@@ -334,62 +356,71 @@ defmodule Pleroma.User do
     end
   end
 
-  def remote_user_creation(params) do
+  defp fix_follower_address(%{follower_address: _, following_address: _} = params), do: params
+
+  defp fix_follower_address(%{nickname: nickname} = params),
+    do: Map.put(params, :follower_address, ap_followers(%User{nickname: nickname}))
+
+  defp fix_follower_address(params), do: params
+
+  def remote_user_changeset(struct \\ %User{local: false}, params) do
     bio_limit = Pleroma.Config.get([:instance, :user_bio_length], 5000)
     name_limit = Pleroma.Config.get([:instance, :user_name_length], 100)
 
+    name =
+      case params[:name] do
+        name when is_binary(name) and byte_size(name) > 0 -> name
+        _ -> params[:nickname]
+      end
+
     params =
       params
+      |> Map.put(:name, name)
+      |> Map.put_new(:last_refreshed_at, NaiveDateTime.utc_now())
       |> truncate_if_exists(:name, name_limit)
       |> truncate_if_exists(:bio, bio_limit)
       |> truncate_fields_param()
+      |> fix_follower_address()
 
-    changeset =
-      %User{local: false}
-      |> cast(
-        params,
-        [
-          :bio,
-          :name,
-          :ap_id,
-          :nickname,
-          :avatar,
-          :ap_enabled,
-          :source_data,
-          :banner,
-          :locked,
-          :magic_key,
-          :uri,
-          :hide_followers,
-          :hide_follows,
-          :hide_followers_count,
-          :hide_follows_count,
-          :follower_count,
-          :fields,
-          :following_count,
-          :discoverable,
-          :invisible,
-          :actor_type,
-          :also_known_as
-        ]
-      )
-      |> validate_required([:name, :ap_id])
-      |> unique_constraint(:nickname)
-      |> validate_format(:nickname, @email_regex)
-      |> validate_length(:bio, max: bio_limit)
-      |> validate_length(:name, max: name_limit)
-      |> validate_fields(true)
-
-    case params[:source_data] do
-      %{"followers" => followers, "following" => following} ->
-        changeset
-        |> put_change(:follower_address, followers)
-        |> put_change(:following_address, following)
-
-      _ ->
-        followers = ap_followers(%User{nickname: get_field(changeset, :nickname)})
-        put_change(changeset, :follower_address, followers)
-    end
+    struct
+    |> cast(
+      params,
+      [
+        :bio,
+        :name,
+        :emoji,
+        :ap_id,
+        :inbox,
+        :shared_inbox,
+        :nickname,
+        :public_key,
+        :avatar,
+        :ap_enabled,
+        :banner,
+        :locked,
+        :last_refreshed_at,
+        :uri,
+        :follower_address,
+        :following_address,
+        :hide_followers,
+        :hide_follows,
+        :hide_followers_count,
+        :hide_follows_count,
+        :follower_count,
+        :fields,
+        :following_count,
+        :discoverable,
+        :invisible,
+        :actor_type,
+        :also_known_as
+      ]
+    )
+    |> validate_required([:name, :ap_id])
+    |> unique_constraint(:nickname)
+    |> validate_format(:nickname, @email_regex)
+    |> validate_length(:bio, max: bio_limit)
+    |> validate_length(:name, max: name_limit)
+    |> validate_fields(true)
   end
 
   def update_changeset(struct, params \\ %{}) do
@@ -402,7 +433,11 @@ defmodule Pleroma.User do
       [
         :bio,
         :name,
+        :emoji,
         :avatar,
+        :public_key,
+        :inbox,
+        :shared_inbox,
         :locked,
         :no_rich_text,
         :default_scope,
@@ -428,50 +463,94 @@ defmodule Pleroma.User do
     |> validate_format(:nickname, local_nickname_regex())
     |> validate_length(:bio, max: bio_limit)
     |> validate_length(:name, min: 1, max: name_limit)
+    |> put_fields()
+    |> put_emoji()
+    |> put_change_if_present(:bio, &{:ok, parse_bio(&1, struct)})
+    |> put_change_if_present(:avatar, &put_upload(&1, :avatar))
+    |> put_change_if_present(:banner, &put_upload(&1, :banner))
+    |> put_change_if_present(:background, &put_upload(&1, :background))
+    |> put_change_if_present(
+      :pleroma_settings_store,
+      &{:ok, Map.merge(struct.pleroma_settings_store, &1)}
+    )
     |> validate_fields(false)
   end
 
-  def upgrade_changeset(struct, params \\ %{}, remote? \\ false) do
-    bio_limit = Pleroma.Config.get([:instance, :user_bio_length], 5000)
-    name_limit = Pleroma.Config.get([:instance, :user_name_length], 100)
+  defp put_fields(changeset) do
+    if raw_fields = get_change(changeset, :raw_fields) do
+      raw_fields =
+        raw_fields
+        |> Enum.filter(fn %{"name" => n} -> n != "" end)
 
-    params = Map.put(params, :last_refreshed_at, NaiveDateTime.utc_now())
+      fields =
+        raw_fields
+        |> Enum.map(fn f -> Map.update!(f, "value", &parse_fields(&1)) end)
 
-    params = if remote?, do: truncate_fields_param(params), else: params
+      changeset
+      |> put_change(:raw_fields, raw_fields)
+      |> put_change(:fields, fields)
+    else
+      changeset
+    end
+  end
 
+  defp parse_fields(value) do
+    value
+    |> Formatter.linkify(mentions_format: :full)
+    |> elem(0)
+  end
+
+  defp put_emoji(changeset) do
+    bio = get_change(changeset, :bio)
+    name = get_change(changeset, :name)
+
+    if bio || name do
+      emoji = Map.merge(Emoji.Formatter.get_emoji_map(bio), Emoji.Formatter.get_emoji_map(name))
+      put_change(changeset, :emoji, emoji)
+    else
+      changeset
+    end
+  end
+
+  defp put_change_if_present(changeset, map_field, value_function) do
+    if value = get_change(changeset, map_field) do
+      with {:ok, new_value} <- value_function.(value) do
+        put_change(changeset, map_field, new_value)
+      else
+        _ -> changeset
+      end
+    else
+      changeset
+    end
+  end
+
+  defp put_upload(value, type) do
+    with %Plug.Upload{} <- value,
+         {:ok, object} <- ActivityPub.upload(value, type: type) do
+      {:ok, object.data}
+    end
+  end
+
+  def update_as_admin_changeset(struct, params) do
     struct
-    |> cast(
-      params,
-      [
-        :bio,
-        :name,
-        :follower_address,
-        :following_address,
-        :avatar,
-        :last_refreshed_at,
-        :ap_enabled,
-        :source_data,
-        :banner,
-        :locked,
-        :magic_key,
-        :follower_count,
-        :following_count,
-        :hide_follows,
-        :fields,
-        :hide_followers,
-        :allow_following_move,
-        :discoverable,
-        :hide_followers_count,
-        :hide_follows_count,
-        :actor_type,
-        :also_known_as
-      ]
-    )
-    |> unique_constraint(:nickname)
-    |> validate_format(:nickname, local_nickname_regex())
-    |> validate_length(:bio, max: bio_limit)
-    |> validate_length(:name, max: name_limit)
-    |> validate_fields(remote?)
+    |> update_changeset(params)
+    |> cast(params, [:email])
+    |> delete_change(:also_known_as)
+    |> unique_constraint(:email)
+    |> validate_format(:email, @email_regex)
+    |> validate_inclusion(:actor_type, ["Person", "Service"])
+  end
+
+  @spec update_as_admin(User.t(), map()) :: {:ok, User.t()} | {:error, Changeset.t()}
+  def update_as_admin(user, params) do
+    params = Map.put(params, "password_confirmation", params["password"])
+    changeset = update_as_admin_changeset(user, params)
+
+    if params["password"] do
+      reset_password(user, changeset, params)
+    else
+      User.update_and_set_cache(changeset)
+    end
   end
 
   def password_update_changeset(struct, params) do
@@ -483,11 +562,15 @@ defmodule Pleroma.User do
     |> put_change(:password_reset_pending, false)
   end
 
-  @spec reset_password(User.t(), map) :: {:ok, User.t()} | {:error, Ecto.Changeset.t()}
-  def reset_password(%User{id: user_id} = user, data) do
+  @spec reset_password(User.t(), map()) :: {:ok, User.t()} | {:error, Changeset.t()}
+  def reset_password(%User{} = user, params) do
+    reset_password(user, user, params)
+  end
+
+  def reset_password(%User{id: user_id} = user, struct, params) do
     multi =
       Multi.new()
-      |> Multi.update(:user, password_update_changeset(user, data))
+      |> Multi.update(:user, password_update_changeset(struct, params))
       |> Multi.delete_all(:tokens, OAuth.Token.Query.get_by_user(user_id))
       |> Multi.delete_all(:auth, OAuth.Authorization.delete_by_user_query(user))
 
@@ -524,7 +607,7 @@ defmodule Pleroma.User do
 
     struct
     |> confirmation_changeset(need_confirmation: need_confirmation?)
-    |> cast(params, [:bio, :email, :name, :nickname, :password, :password_confirmation])
+    |> cast(params, [:bio, :email, :name, :nickname, :password, :password_confirmation, :emoji])
     |> validate_required([:name, :nickname, :password, :password_confirmation])
     |> validate_confirmation(:password)
     |> unique_constraint(:email)
@@ -617,8 +700,10 @@ defmodule Pleroma.User do
   def needs_update?(_), do: true
 
   @spec maybe_direct_follow(User.t(), User.t()) :: {:ok, User.t()} | {:error, String.t()}
+
+  # "Locked" (self-locked) users demand explicit authorization of follow requests
   def maybe_direct_follow(%User{} = follower, %User{local: true, locked: true} = followed) do
-    follow(follower, followed, "pending")
+    follow(follower, followed, :follow_pending)
   end
 
   def maybe_direct_follow(%User{} = follower, %User{local: true} = followed) do
@@ -638,14 +723,14 @@ defmodule Pleroma.User do
   def follow_all(follower, followeds) do
     followeds
     |> Enum.reject(fn followed -> blocks?(follower, followed) || blocks?(followed, follower) end)
-    |> Enum.each(&follow(follower, &1, "accept"))
+    |> Enum.each(&follow(follower, &1, :follow_accept))
 
     set_cache(follower)
   end
 
   defdelegate following(user), to: FollowingRelationship
 
-  def follow(%User{} = follower, %User{} = followed, state \\ "accept") do
+  def follow(%User{} = follower, %User{} = followed, state \\ :follow_accept) do
     deny_follow_blocked = Pleroma.Config.get([:user, :deny_follow_blocked])
 
     cond do
@@ -670,9 +755,21 @@ defmodule Pleroma.User do
     {:error, "Not subscribed!"}
   end
 
+  @spec unfollow(User.t(), User.t()) :: {:ok, User.t(), Activity.t()} | {:error, String.t()}
   def unfollow(%User{} = follower, %User{} = followed) do
+    case do_unfollow(follower, followed) do
+      {:ok, follower, followed} ->
+        {:ok, follower, Utils.fetch_latest_follow(follower, followed)}
+
+      error ->
+        error
+    end
+  end
+
+  @spec do_unfollow(User.t(), User.t()) :: {:ok, User.t(), User.t()} | {:error, String.t()}
+  defp do_unfollow(%User{} = follower, %User{} = followed) do
     case get_follow_state(follower, followed) do
-      state when state in ["accept", "pending"] ->
+      state when state in [:follow_pending, :follow_accept] ->
         FollowingRelationship.unfollow(follower, followed)
         {:ok, followed} = update_follower_count(followed)
 
@@ -681,7 +778,7 @@ defmodule Pleroma.User do
           |> update_following_count()
           |> set_cache()
 
-        {:ok, follower, Utils.fetch_latest_follow(follower, followed)}
+        {:ok, follower, followed}
 
       nil ->
         {:error, "Not subscribed!"}
@@ -690,14 +787,25 @@ defmodule Pleroma.User do
 
   defdelegate following?(follower, followed), to: FollowingRelationship
 
+  @doc "Returns follow state as Pleroma.FollowingRelationship.State value"
   def get_follow_state(%User{} = follower, %User{} = following) do
     following_relationship = FollowingRelationship.get(follower, following)
+    get_follow_state(follower, following, following_relationship)
+  end
 
+  def get_follow_state(
+        %User{} = follower,
+        %User{} = following,
+        following_relationship
+      ) do
     case {following_relationship, following.local} do
       {nil, false} ->
         case Utils.fetch_latest_follow(follower, following) do
-          %{data: %{"state" => state}} when state in ["pending", "accept"] -> state
-          _ -> nil
+          %Activity{data: %{"state" => state}} when state in ["pending", "accept"] ->
+            FollowingRelationship.state_to_enum(state)
+
+          _ ->
+            nil
         end
 
       {%{state: state}, _} ->
@@ -748,6 +856,7 @@ defmodule Pleroma.User do
   def set_cache(%User{} = user) do
     Cachex.put(:user_cache, "ap_id:#{user.ap_id}", user)
     Cachex.put(:user_cache, "nickname:#{user.nickname}", user)
+    Cachex.put(:user_cache, "friends_ap_ids:#{user.nickname}", get_user_friends_ap_ids(user))
     {:ok, user}
   end
 
@@ -763,9 +872,22 @@ defmodule Pleroma.User do
     end
   end
 
+  def get_user_friends_ap_ids(user) do
+    from(u in User.get_friends_query(user), select: u.ap_id)
+    |> Repo.all()
+  end
+
+  @spec get_cached_user_friends_ap_ids(User.t()) :: [String.t()]
+  def get_cached_user_friends_ap_ids(user) do
+    Cachex.fetch!(:user_cache, "friends_ap_ids:#{user.ap_id}", fn _ ->
+      get_user_friends_ap_ids(user)
+    end)
+  end
+
   def invalidate_cache(user) do
     Cachex.del(:user_cache, "ap_id:#{user.ap_id}")
     Cachex.del(:user_cache, "nickname:#{user.nickname}")
+    Cachex.del(:user_cache, "friends_ap_ids:#{user.ap_id}")
   end
 
   @spec get_cached_by_ap_id(String.t()) :: User.t() | nil
@@ -829,6 +951,7 @@ defmodule Pleroma.User do
     end
   end
 
+  @spec get_by_nickname(String.t()) :: User.t() | nil
   def get_by_nickname(nickname) do
     Repo.get_by(User, nickname: nickname) ||
       if Regex.match?(~r(@#{Pleroma.Web.Endpoint.host()})i, nickname) do
@@ -1086,8 +1209,9 @@ defmodule Pleroma.User do
 
   def increment_unread_conversation_count(_, user), do: {:ok, user}
 
-  @spec get_users_from_set([String.t()], boolean()) :: [User.t()]
-  def get_users_from_set(ap_ids, local_only \\ true) do
+  @spec get_users_from_set([String.t()], keyword()) :: [User.t()]
+  def get_users_from_set(ap_ids, opts \\ []) do
+    local_only = Keyword.get(opts, :local_only, true)
     criteria = %{ap_id: ap_ids, deactivated: false}
     criteria = if local_only, do: Map.put(criteria, :local, true), else: criteria
 
@@ -1096,8 +1220,12 @@ defmodule Pleroma.User do
   end
 
   @spec get_recipients_from_activity(Activity.t()) :: [User.t()]
-  def get_recipients_from_activity(%Activity{recipients: to}) do
-    User.Query.build(%{recipients_from_activity: to, local: true, deactivated: false})
+  def get_recipients_from_activity(%Activity{recipients: to, actor: actor}) do
+    to = [actor | to]
+
+    query = User.Query.build(%{recipients_from_activity: to, local: true, deactivated: false})
+
+    query
     |> Repo.all()
   end
 
@@ -1196,7 +1324,7 @@ defmodule Pleroma.User do
 
   def blocks?(%User{} = user, %User{} = target) do
     blocks_user?(user, target) ||
-      (!User.following?(user, target) && blocks_domain?(user, target))
+      (blocks_domain?(user, target) and not User.following?(user, target))
   end
 
   def blocks_user?(%User{} = user, %User{} = target) do
@@ -1225,13 +1353,15 @@ defmodule Pleroma.User do
   end
 
   @doc """
-  Returns map of outgoing (blocked, muted etc.) relations' user AP IDs by relation type.
-  E.g. `outgoing_relations_ap_ids(user, [:block])` -> `%{block: ["https://some.site/users/userapid"]}`
+  Returns map of outgoing (blocked, muted etc.) relationships' user AP IDs by relation type.
+  E.g. `outgoing_relationships_ap_ids(user, [:block])` -> `%{block: ["https://some.site/users/userapid"]}`
   """
-  @spec outgoing_relations_ap_ids(User.t(), list(atom())) :: %{atom() => list(String.t())}
-  def outgoing_relations_ap_ids(_, []), do: %{}
+  @spec outgoing_relationships_ap_ids(User.t(), list(atom())) :: %{atom() => list(String.t())}
+  def outgoing_relationships_ap_ids(_user, []), do: %{}
 
-  def outgoing_relations_ap_ids(%User{} = user, relationship_types)
+  def outgoing_relationships_ap_ids(nil, _relationship_types), do: %{}
+
+  def outgoing_relationships_ap_ids(%User{} = user, relationship_types)
       when is_list(relationship_types) do
     db_result =
       user
@@ -1250,6 +1380,30 @@ defmodule Pleroma.User do
     )
   end
 
+  def incoming_relationships_ungrouped_ap_ids(user, relationship_types, ap_ids \\ nil)
+
+  def incoming_relationships_ungrouped_ap_ids(_user, [], _ap_ids), do: []
+
+  def incoming_relationships_ungrouped_ap_ids(nil, _relationship_types, _ap_ids), do: []
+
+  def incoming_relationships_ungrouped_ap_ids(%User{} = user, relationship_types, ap_ids)
+      when is_list(relationship_types) do
+    user
+    |> assoc(:incoming_relationships)
+    |> join(:inner, [user_rel], u in assoc(user_rel, :source))
+    |> where([user_rel, u], user_rel.relationship_type in ^relationship_types)
+    |> maybe_filter_on_ap_id(ap_ids)
+    |> select([user_rel, u], u.ap_id)
+    |> distinct(true)
+    |> Repo.all()
+  end
+
+  defp maybe_filter_on_ap_id(query, ap_ids) when is_list(ap_ids) do
+    where(query, [user_rel, u], u.ap_id in ^ap_ids)
+  end
+
+  defp maybe_filter_on_ap_id(query, _ap_ids), do: query
+
   def deactivate_async(user, status \\ true) do
     BackgroundWorker.enqueue("deactivate_user", %{"user_id" => user.id, "status" => status})
   end
@@ -1267,15 +1421,13 @@ defmodule Pleroma.User do
       user
       |> get_followers()
       |> Enum.filter(& &1.local)
-      |> Enum.each(fn follower ->
-        follower |> update_following_count() |> set_cache()
-      end)
+      |> Enum.each(&set_cache(update_following_count(&1)))
 
       # Only update local user counts, remote will be update during the next pull.
       user
       |> get_friends()
       |> Enum.filter(& &1.local)
-      |> Enum.each(&update_follower_count/1)
+      |> Enum.each(&do_unfollow(user, &1))
 
       {:ok, user}
     end
@@ -1297,12 +1449,29 @@ defmodule Pleroma.User do
     BackgroundWorker.enqueue("delete_user", %{"user_id" => user.id})
   end
 
+  defp delete_and_invalidate_cache(%User{} = user) do
+    invalidate_cache(user)
+    Repo.delete(user)
+  end
+
+  defp delete_or_deactivate(%User{local: false} = user), do: delete_and_invalidate_cache(user)
+
+  defp delete_or_deactivate(%User{local: true} = user) do
+    status = account_status(user)
+
+    if status == :confirmation_pending do
+      delete_and_invalidate_cache(user)
+    else
+      user
+      |> change(%{deactivated: true, email: nil})
+      |> update_and_set_cache()
+    end
+  end
+
   def perform(:force_password_reset, user), do: force_password_reset(user)
 
   @spec perform(atom(), User.t()) :: {:ok, User.t()}
   def perform(:delete, %User{} = user) do
-    {:ok, _user} = ActivityPub.delete(user)
-
     # Remove all relationships
     user
     |> get_followers()
@@ -1319,8 +1488,8 @@ defmodule Pleroma.User do
     end)
 
     delete_user_activities(user)
-    invalidate_cache(user)
-    Repo.delete(user)
+
+    delete_or_deactivate(user)
   end
 
   def perform(:deactivate_async, user, status), do: deactivate(user, status)
@@ -1405,37 +1574,42 @@ defmodule Pleroma.User do
     })
   end
 
-  def delete_user_activities(%User{ap_id: ap_id}) do
+  def delete_user_activities(%User{ap_id: ap_id} = user) do
     ap_id
     |> Activity.Queries.by_actor()
     |> RepoStreamer.chunk_stream(50)
-    |> Stream.each(fn activities -> Enum.each(activities, &delete_activity/1) end)
+    |> Stream.each(fn activities ->
+      Enum.each(activities, fn activity -> delete_activity(activity, user) end)
+    end)
     |> Stream.run()
   end
 
-  defp delete_activity(%{data: %{"type" => "Create"}} = activity) do
-    activity
-    |> Object.normalize()
-    |> ActivityPub.delete()
+  defp delete_activity(%{data: %{"type" => "Create", "object" => object}} = activity, user) do
+    with {_, %Object{}} <- {:find_object, Object.get_by_ap_id(object)},
+         {:ok, delete_data, _} <- Builder.delete(user, object) do
+      Pipeline.common_pipeline(delete_data, local: user.local)
+    else
+      {:find_object, nil} ->
+        # We have the create activity, but not the object, it was probably pruned.
+        # Insert a tombstone and try again
+        with {:ok, tombstone_data, _} <- Builder.tombstone(user.ap_id, object),
+             {:ok, _tombstone} <- Object.create(tombstone_data) do
+          delete_activity(activity, user)
+        end
+
+      e ->
+        Logger.error("Could not delete #{object} created by #{activity.data["ap_id"]}")
+        Logger.error("Error: #{inspect(e)}")
+    end
   end
 
-  defp delete_activity(%{data: %{"type" => "Like"}} = activity) do
-    object = Object.normalize(activity)
-
-    activity.actor
-    |> get_cached_by_ap_id()
-    |> ActivityPub.unlike(object)
+  defp delete_activity(%{data: %{"type" => type}} = activity, user)
+       when type in ["Like", "Announce"] do
+    {:ok, undo, _} = Builder.undo(user, activity)
+    Pipeline.common_pipeline(undo, local: user.local)
   end
 
-  defp delete_activity(%{data: %{"type" => "Announce"}} = activity) do
-    object = Object.normalize(activity)
-
-    activity.actor
-    |> get_cached_by_ap_id()
-    |> ActivityPub.unannounce(object)
-  end
-
-  defp delete_activity(_activity), do: "Doing nothing"
+  defp delete_activity(_activity, _user), do: "Doing nothing"
 
   def html_filter_policy(%User{no_rich_text: true}) do
     Pleroma.HTML.Scrubber.TwitterText
@@ -1446,12 +1620,19 @@ defmodule Pleroma.User do
   def fetch_by_ap_id(ap_id), do: ActivityPub.make_user_from_ap_id(ap_id)
 
   def get_or_fetch_by_ap_id(ap_id) do
-    user = get_cached_by_ap_id(ap_id)
+    cached_user = get_cached_by_ap_id(ap_id)
 
-    if !is_nil(user) and !needs_update?(user) do
-      {:ok, user}
-    else
-      fetch_by_ap_id(ap_id)
+    maybe_fetched_user = needs_update?(cached_user) && fetch_by_ap_id(ap_id)
+
+    case {cached_user, maybe_fetched_user} do
+      {_, {:ok, %User{} = user}} ->
+        {:ok, user}
+
+      {%User{} = user, _} ->
+        {:ok, user}
+
+      _ ->
+        {:error, :not_found}
     end
   end
 
@@ -1502,8 +1683,7 @@ defmodule Pleroma.User do
     |> set_cache()
   end
 
-  # AP style
-  def public_key(%{source_data: %{"publicKey" => %{"publicKeyPem" => public_key_pem}}}) do
+  def public_key(%{public_key: public_key_pem}) when is_binary(public_key_pem) do
     key =
       public_key_pem
       |> :public_key.pem_decode()
@@ -1513,7 +1693,7 @@ defmodule Pleroma.User do
     {:ok, key}
   end
 
-  def public_key(_), do: {:error, "not found key"}
+  def public_key(_), do: {:error, "key not found"}
 
   def get_public_key_for_ap_id(ap_id) do
     with {:ok, %User{} = user} <- get_or_fetch_by_ap_id(ap_id),
@@ -1524,17 +1704,6 @@ defmodule Pleroma.User do
     end
   end
 
-  defp blank?(""), do: nil
-  defp blank?(n), do: n
-
-  def insert_or_update_user(data) do
-    data
-    |> Map.put(:name, blank?(data[:name]) || data[:nickname])
-    |> remote_user_creation()
-    |> Repo.insert(on_conflict: {:replace_all_except, [:id]}, conflict_target: :nickname)
-    |> set_cache()
-  end
-
   def ap_enabled?(%User{local: true}), do: true
   def ap_enabled?(%User{ap_enabled: ap_enabled}), do: ap_enabled
   def ap_enabled?(_), do: false
@@ -1660,8 +1829,12 @@ defmodule Pleroma.User do
     |> Repo.all()
   end
 
+  def muting_reblogs?(%User{} = user, %User{} = target) do
+    UserRelationship.reblog_mute_exists?(user, target)
+  end
+
   def showing_reblogs?(%User{} = user, %User{} = target) do
-    not UserRelationship.reblog_mute_exists?(user, target)
+    not muting_reblogs?(user, target)
   end
 
   @doc """
@@ -1790,7 +1963,7 @@ defmodule Pleroma.User do
   defp put_password_hash(
          %Ecto.Changeset{valid?: true, changes: %{password: password}} = changeset
        ) do
-    change(changeset, password_hash: Pbkdf2.hashpwsalt(password))
+    change(changeset, password_hash: Pbkdf2.hash_pwd_salt(password))
   end
 
   defp put_password_hash(changeset), do: changeset
@@ -1839,12 +2012,6 @@ defmodule Pleroma.User do
     |> update_and_set_cache()
   end
 
-  def update_source_data(user, source_data) do
-    user
-    |> cast(%{source_data: source_data}, [:source_data])
-    |> update_and_set_cache()
-  end
-
   def roles(%{is_moderator: is_moderator, is_admin: is_admin}) do
     %{
       admin: is_admin,
@@ -1852,21 +2019,6 @@ defmodule Pleroma.User do
     }
   end
 
-  # ``fields`` is an array of mastodon profile field, containing ``{"name": "…", "value": "…"}``.
-  # For example: [{"name": "Pronoun", "value": "she/her"}, …]
-  def fields(%{fields: nil, source_data: %{"attachment" => attachment}}) do
-    limit = Pleroma.Config.get([:instance, :max_remote_account_fields], 0)
-
-    attachment
-    |> Enum.filter(fn %{"type" => t} -> t == "PropertyValue" end)
-    |> Enum.map(fn fields -> Map.take(fields, ["name", "value"]) end)
-    |> Enum.take(limit)
-  end
-
-  def fields(%{fields: nil}), do: []
-
-  def fields(%{fields: fields}), do: fields
-
   def validate_fields(changeset, remote? \\ false) do
     limit_name = if remote?, do: :max_remote_account_fields, else: :max_account_fields
     limit = Pleroma.Config.get([:instance, limit_name], 0)
@@ -2054,9 +2206,7 @@ defmodule Pleroma.User do
   # - display name
   def sanitize_html(%User{} = user, filter) do
     fields =
-      user
-      |> User.fields()
-      |> Enum.map(fn %{"name" => name, "value" => value} ->
+      Enum.map(user.fields, fn %{"name" => name, "value" => value} ->
         %{
           "name" => name,
           "value" => HTML.filter_tags(value, Pleroma.HTML.Scrubber.LinksOnly)

lib/pleroma/user/query.ex

@@ -45,6 +45,7 @@ defmodule Pleroma.User.Query do
             is_admin: boolean(),
             is_moderator: boolean(),
             super_users: boolean(),
+            exclude_service_users: boolean(),
             followers: User.t(),
             friends: User.t(),
             recipients_from_activity: [String.t()],
@@ -54,13 +55,13 @@ defmodule Pleroma.User.Query do
             select: term(),
             limit: pos_integer()
           }
-          | %{}
+          | map()
 
   @ilike_criteria [:nickname, :name, :query]
   @equal_criteria [:email]
   @contains_criteria [:ap_id, :nickname]
 
-  @spec build(criteria()) :: Query.t()
+  @spec build(Query.t(), criteria()) :: Query.t()
   def build(query \\ base_query(), criteria) do
     prepare_query(query, criteria)
   end
@@ -88,6 +89,10 @@ defmodule Pleroma.User.Query do
     where(query, [u], ilike(field(u, ^key), ^"%#{value}%"))
   end
 
+  defp compose_query({:exclude_service_users, _}, query) do
+    where(query, [u], not like(u.ap_id, "%/relay") and not like(u.ap_id, "%/internal/fetch"))
+  end
+
   defp compose_query({key, value}, query)
        when key in @equal_criteria and not_empty_string(value) do
     where(query, [u], ^[{key, value}])
@@ -98,7 +103,7 @@ defmodule Pleroma.User.Query do
   end
 
   defp compose_query({:tags, tags}, query) when is_list(tags) and length(tags) > 0 do
-    Enum.reduce(tags, query, &prepare_tag_criteria/2)
+    where(query, [u], fragment("? && ?", u.tags, ^tags))
   end
 
   defp compose_query({:is_admin, _}, query) do
@@ -148,7 +153,7 @@ defmodule Pleroma.User.Query do
       as: :relationships,
       on: r.following_id == ^id and r.follower_id == u.id
     )
-    |> where([relationships: r], r.state == "accept")
+    |> where([relationships: r], r.state == ^:follow_accept)
   end
 
   defp compose_query({:friends, %User{id: id}}, query) do
@@ -158,24 +163,22 @@ defmodule Pleroma.User.Query do
       as: :relationships,
       on: r.following_id == u.id and r.follower_id == ^id
     )
-    |> where([relationships: r], r.state == "accept")
+    |> where([relationships: r], r.state == ^:follow_accept)
   end
 
   defp compose_query({:recipients_from_activity, to}, query) do
-    query
-    |> join(:left, [u], r in FollowingRelationship,
-      as: :relationships,
-      on: r.follower_id == u.id
+    following_query =
+      from(u in User,
+        join: f in FollowingRelationship,
+        on: u.id == f.following_id,
+        where: f.state == ^:follow_accept,
+        where: u.follower_address in ^to,
+        select: f.follower_id
+      )
+
+    from(u in query,
+      where: u.ap_id in ^to or u.id in subquery(following_query)
     )
-    |> join(:left, [relationships: r], f in User,
-      as: :following,
-      on: f.id == r.following_id
-    )
-    |> where(
-      [u, following: f, relationships: r],
-      u.ap_id in ^to or (f.follower_address in ^to and r.state == "accept")
-    )
-    |> distinct(true)
   end
 
   defp compose_query({:order_by, key}, query) do
@@ -192,10 +195,6 @@ defmodule Pleroma.User.Query do
 
   defp compose_query(_unsupported_param, query), do: query
 
-  defp prepare_tag_criteria(tag, query) do
-    or_where(query, [u], fragment("? = any(?)", ^tag, u.tags))
-  end
-
   defp location_query(query, local) do
     where(query, [u], u.local == ^local)
     |> where([u], not is_nil(u.nickname))

lib/pleroma/user/welcome_message.ex

@@ -10,8 +10,8 @@ defmodule Pleroma.User.WelcomeMessage do
     with %User{} = sender_user <- welcome_user(),
          message when is_binary(message) <- welcome_message() do
       CommonAPI.post(sender_user, %{
-        "visibility" => "direct",
-        "status" => "@#{user.nickname}\n#{message}"
+        visibility: "direct",
+        status: "@#{user.nickname}\n#{message}"
       })
     else
       _ -> {:ok, nil}

lib/pleroma/user_relationship.ex

@@ -8,6 +8,8 @@ defmodule Pleroma.UserRelationship do
   import Ecto.Changeset
   import Ecto.Query
 
+  alias Ecto.Changeset
+  alias Pleroma.FollowingRelationship
   alias Pleroma.Repo
   alias Pleroma.User
   alias Pleroma.UserRelationship
@@ -15,25 +17,32 @@ defmodule Pleroma.UserRelationship do
   schema "user_relationships" do
     belongs_to(:source, User, type: FlakeId.Ecto.CompatType)
     belongs_to(:target, User, type: FlakeId.Ecto.CompatType)
-    field(:relationship_type, UserRelationshipTypeEnum)
+    field(:relationship_type, Pleroma.UserRelationship.Type)
 
     timestamps(updated_at: false)
   end
 
-  for relationship_type <- Keyword.keys(UserRelationshipTypeEnum.__enum_map__()) do
-    # Definitions of `create_block/2`, `create_mute/2` etc.
+  for relationship_type <- Keyword.keys(Pleroma.UserRelationship.Type.__enum_map__()) do
+    # `def create_block/2`, `def create_mute/2`, `def create_reblog_mute/2`,
+    #   `def create_notification_mute/2`, `def create_inverse_subscription/2`
     def unquote(:"create_#{relationship_type}")(source, target),
       do: create(unquote(relationship_type), source, target)
 
-    # Definitions of `delete_block/2`, `delete_mute/2` etc.
+    # `def delete_block/2`, `def delete_mute/2`, `def delete_reblog_mute/2`,
+    #   `def delete_notification_mute/2`, `def delete_inverse_subscription/2`
     def unquote(:"delete_#{relationship_type}")(source, target),
       do: delete(unquote(relationship_type), source, target)
 
-    # Definitions of `block_exists?/2`, `mute_exists?/2` etc.
+    # `def block_exists?/2`, `def mute_exists?/2`, `def reblog_mute_exists?/2`,
+    #   `def notification_mute_exists?/2`, `def inverse_subscription_exists?/2`
     def unquote(:"#{relationship_type}_exists?")(source, target),
       do: exists?(unquote(relationship_type), source, target)
   end
 
+  def user_relationship_types, do: Keyword.keys(user_relationship_mappings())
+
+  def user_relationship_mappings, do: Pleroma.UserRelationship.Type.__enum_map__()
+
   def changeset(%UserRelationship{} = user_relationship, params \\ %{}) do
     user_relationship
     |> cast(params, [:relationship_type, :source_id, :target_id])
@@ -72,16 +81,122 @@ defmodule Pleroma.UserRelationship do
     end
   end
 
-  defp validate_not_self_relationship(%Ecto.Changeset{} = changeset) do
-    changeset
-    |> validate_change(:target_id, fn _, target_id ->
-      if target_id == get_field(changeset, :source_id) do
-        [target_id: "can't be equal to source_id"]
-      else
-        []
+  def dictionary(
+        source_users,
+        target_users,
+        source_to_target_rel_types \\ nil,
+        target_to_source_rel_types \\ nil
+      )
+
+  def dictionary(
+        _source_users,
+        _target_users,
+        [] = _source_to_target_rel_types,
+        [] = _target_to_source_rel_types
+      ) do
+    []
+  end
+
+  def dictionary(
+        source_users,
+        target_users,
+        source_to_target_rel_types,
+        target_to_source_rel_types
+      )
+      when is_list(source_users) and is_list(target_users) do
+    source_user_ids = User.binary_id(source_users)
+    target_user_ids = User.binary_id(target_users)
+
+    get_rel_type_codes = fn rel_type -> user_relationship_mappings()[rel_type] end
+
+    source_to_target_rel_types =
+      Enum.map(source_to_target_rel_types || user_relationship_types(), &get_rel_type_codes.(&1))
+
+    target_to_source_rel_types =
+      Enum.map(target_to_source_rel_types || user_relationship_types(), &get_rel_type_codes.(&1))
+
+    __MODULE__
+    |> where(
+      fragment(
+        "(source_id = ANY(?) AND target_id = ANY(?) AND relationship_type = ANY(?)) OR \
+        (source_id = ANY(?) AND target_id = ANY(?) AND relationship_type = ANY(?))",
+        ^source_user_ids,
+        ^target_user_ids,
+        ^source_to_target_rel_types,
+        ^target_user_ids,
+        ^source_user_ids,
+        ^target_to_source_rel_types
+      )
+    )
+    |> select([ur], [ur.relationship_type, ur.source_id, ur.target_id])
+    |> Repo.all()
+  end
+
+  def exists?(dictionary, rel_type, source, target, func) do
+    cond do
+      is_nil(source) or is_nil(target) ->
+        false
+
+      dictionary ->
+        [rel_type, source.id, target.id] in dictionary
+
+      true ->
+        func.(source, target)
+    end
+  end
+
+  @doc ":relationships option for StatusView / AccountView / NotificationView"
+  def view_relationships_option(reading_user, actors, opts \\ [])
+
+  def view_relationships_option(nil = _reading_user, _actors, _opts) do
+    %{user_relationships: [], following_relationships: []}
+  end
+
+  def view_relationships_option(%User{} = reading_user, actors, opts) do
+    {source_to_target_rel_types, target_to_source_rel_types} =
+      case opts[:subset] do
+        :source_mutes ->
+          # Used for statuses rendering (FE needs `muted` flag for each status when statuses load)
+          {[:mute], []}
+
+        nil ->
+          {[:block, :mute, :notification_mute, :reblog_mute], [:block, :inverse_subscription]}
+
+        unknown ->
+          raise "Unsupported :subset option value: #{inspect(unknown)}"
       end
-    end)
-    |> validate_change(:source_id, fn _, source_id ->
+
+    user_relationships =
+      UserRelationship.dictionary(
+        [reading_user],
+        actors,
+        source_to_target_rel_types,
+        target_to_source_rel_types
+      )
+
+    following_relationships =
+      case opts[:subset] do
+        :source_mutes ->
+          []
+
+        nil ->
+          FollowingRelationship.all_between_user_sets([reading_user], actors)
+
+        unknown ->
+          raise "Unsupported :subset option value: #{inspect(unknown)}"
+      end
+
+    %{user_relationships: user_relationships, following_relationships: following_relationships}
+  end
+
+  defp validate_not_self_relationship(%Changeset{} = changeset) do
+    changeset
+    |> validate_source_id_target_id_inequality()
+    |> validate_target_id_source_id_inequality()
+  end
+
+  defp validate_source_id_target_id_inequality(%Changeset{} = changeset) do
+    validate_change(changeset, :source_id, fn _, source_id ->
       if source_id == get_field(changeset, :target_id) do
         [source_id: "can't be equal to target_id"]
       else
@@ -89,4 +204,14 @@ defmodule Pleroma.UserRelationship do
       end
     end)
   end
+
+  defp validate_target_id_source_id_inequality(%Changeset{} = changeset) do
+    validate_change(changeset, :target_id, fn _, target_id ->
+      if target_id == get_field(changeset, :source_id) do
+        [target_id: "can't be equal to source_id"]
+      else
+        []
+      end
+    end)
+  end
 end

lib/pleroma/web/activity_pub/activity_pub.ex

@@ -118,13 +118,29 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
 
   def increase_poll_votes_if_vote(%{
         "object" => %{"inReplyTo" => reply_ap_id, "name" => name},
-        "type" => "Create"
+        "type" => "Create",
+        "actor" => actor
       }) do
-    Object.increase_vote_count(reply_ap_id, name)
+    Object.increase_vote_count(reply_ap_id, name, actor)
   end
 
   def increase_poll_votes_if_vote(_create_data), do: :noop
 
+  @spec persist(map(), keyword()) :: {:ok, Activity.t() | Object.t()}
+  def persist(object, meta) do
+    with local <- Keyword.fetch!(meta, :local),
+         {recipients, _, _} <- get_recipients(object),
+         {:ok, activity} <-
+           Repo.insert(%Activity{
+             data: object,
+             local: local,
+             recipients: recipients,
+             actor: object["actor"]
+           }) do
+      {:ok, activity, meta}
+    end
+  end
+
   @spec insert(map(), boolean(), boolean(), boolean()) :: {:ok, Activity.t()} | {:error, any()}
   def insert(map, local \\ true, fake \\ false, bypass_actor_check \\ false) when is_map(map) do
     with nil <- Activity.normalize(map),
@@ -154,12 +170,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
 
       BackgroundWorker.enqueue("fetch_data_for_activity", %{"activity_id" => activity.id})
 
-      Notification.create_notifications(activity)
-
-      conversation = create_or_bump_conversation(activity, map["actor"])
-      participations = get_participations(conversation)
-      stream_out(activity)
-      stream_out_participations(participations)
       {:ok, activity}
     else
       %Activity{} = activity ->
@@ -182,6 +192,15 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     end
   end
 
+  def notify_and_stream(activity) do
+    Notification.create_notifications(activity)
+
+    conversation = create_or_bump_conversation(activity, activity.actor)
+    participations = get_participations(conversation)
+    stream_out(activity)
+    stream_out_participations(participations)
+  end
+
   defp create_or_bump_conversation(activity, actor) do
     with {:ok, conversation} <- Conversation.create_or_bump_for(activity),
          %User{} = user <- User.get_cached_by_ap_id(actor),
@@ -258,6 +277,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
          _ <- increase_poll_votes_if_vote(create_data),
          {:quick_insert, false, activity} <- {:quick_insert, quick_insert?, activity},
          {:ok, _actor} <- increase_note_count_if_public(actor, activity),
+         _ <- notify_and_stream(activity),
          :ok <- maybe_federate(activity) do
       {:ok, activity}
     else
@@ -285,6 +305,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
              additional
            ),
          {:ok, activity} <- insert(listen_data, local),
+         _ <- notify_and_stream(activity),
          :ok <- maybe_federate(activity) do
       {:ok, activity}
     end
@@ -309,6 +330,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
            %{"to" => to, "type" => type, "actor" => actor.ap_id, "object" => object}
            |> Utils.maybe_put("id", activity_id),
          {:ok, activity} <- insert(data, local),
+         _ <- notify_and_stream(activity),
          :ok <- maybe_federate(activity) do
       {:ok, activity}
     end
@@ -328,169 +350,12 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
          },
          data <- Utils.maybe_put(data, "id", activity_id),
          {:ok, activity} <- insert(data, local),
+         _ <- notify_and_stream(activity),
          :ok <- maybe_federate(activity) do
       {:ok, activity}
     end
   end
 
-  @spec react_with_emoji(User.t(), Object.t(), String.t(), keyword()) ::
-          {:ok, Activity.t(), Object.t()} | {:error, any()}
-  def react_with_emoji(user, object, emoji, options \\ []) do
-    with {:ok, result} <-
-           Repo.transaction(fn -> do_react_with_emoji(user, object, emoji, options) end) do
-      result
-    end
-  end
-
-  defp do_react_with_emoji(user, object, emoji, options) do
-    with local <- Keyword.get(options, :local, true),
-         activity_id <- Keyword.get(options, :activity_id, nil),
-         true <- Pleroma.Emoji.is_unicode_emoji?(emoji),
-         reaction_data <- make_emoji_reaction_data(user, object, emoji, activity_id),
-         {:ok, activity} <- insert(reaction_data, local),
-         {:ok, object} <- add_emoji_reaction_to_object(activity, object),
-         :ok <- maybe_federate(activity) do
-      {:ok, activity, object}
-    else
-      false -> {:error, false}
-      {:error, error} -> Repo.rollback(error)
-    end
-  end
-
-  @spec unreact_with_emoji(User.t(), String.t(), keyword()) ::
-          {:ok, Activity.t(), Object.t()} | {:error, any()}
-  def unreact_with_emoji(user, reaction_id, options \\ []) do
-    with {:ok, result} <-
-           Repo.transaction(fn -> do_unreact_with_emoji(user, reaction_id, options) end) do
-      result
-    end
-  end
-
-  defp do_unreact_with_emoji(user, reaction_id, options) do
-    with local <- Keyword.get(options, :local, true),
-         activity_id <- Keyword.get(options, :activity_id, nil),
-         user_ap_id <- user.ap_id,
-         %Activity{actor: ^user_ap_id} = reaction_activity <- Activity.get_by_ap_id(reaction_id),
-         object <- Object.normalize(reaction_activity),
-         unreact_data <- make_undo_data(user, reaction_activity, activity_id),
-         {:ok, activity} <- insert(unreact_data, local),
-         {:ok, object} <- remove_emoji_reaction_from_object(reaction_activity, object),
-         :ok <- maybe_federate(activity) do
-      {:ok, activity, object}
-    else
-      {:error, error} -> Repo.rollback(error)
-    end
-  end
-
-  # TODO: This is weird, maybe we shouldn't check here if we can make the activity.
-  @spec like(User.t(), Object.t(), String.t() | nil, boolean()) ::
-          {:ok, Activity.t(), Object.t()} | {:error, any()}
-  def like(user, object, activity_id \\ nil, local \\ true) do
-    with {:ok, result} <- Repo.transaction(fn -> do_like(user, object, activity_id, local) end) do
-      result
-    end
-  end
-
-  defp do_like(
-         %User{ap_id: ap_id} = user,
-         %Object{data: %{"id" => _}} = object,
-         activity_id,
-         local
-       ) do
-    with nil <- get_existing_like(ap_id, object),
-         like_data <- make_like_data(user, object, activity_id),
-         {:ok, activity} <- insert(like_data, local),
-         {:ok, object} <- add_like_to_object(activity, object),
-         :ok <- maybe_federate(activity) do
-      {:ok, activity, object}
-    else
-      %Activity{} = activity ->
-        {:ok, activity, object}
-
-      {:error, error} ->
-        Repo.rollback(error)
-    end
-  end
-
-  @spec unlike(User.t(), Object.t(), String.t() | nil, boolean()) ::
-          {:ok, Activity.t(), Activity.t(), Object.t()} | {:ok, Object.t()} | {:error, any()}
-  def unlike(%User{} = actor, %Object{} = object, activity_id \\ nil, local \\ true) do
-    with {:ok, result} <-
-           Repo.transaction(fn -> do_unlike(actor, object, activity_id, local) end) do
-      result
-    end
-  end
-
-  defp do_unlike(actor, object, activity_id, local) do
-    with %Activity{} = like_activity <- get_existing_like(actor.ap_id, object),
-         unlike_data <- make_unlike_data(actor, like_activity, activity_id),
-         {:ok, unlike_activity} <- insert(unlike_data, local),
-         {:ok, _activity} <- Repo.delete(like_activity),
-         {:ok, object} <- remove_like_from_object(like_activity, object),
-         :ok <- maybe_federate(unlike_activity) do
-      {:ok, unlike_activity, like_activity, object}
-    else
-      nil -> {:ok, object}
-      {:error, error} -> Repo.rollback(error)
-    end
-  end
-
-  @spec announce(User.t(), Object.t(), String.t() | nil, boolean(), boolean()) ::
-          {:ok, Activity.t(), Object.t()} | {:error, any()}
-  def announce(
-        %User{ap_id: _} = user,
-        %Object{data: %{"id" => _}} = object,
-        activity_id \\ nil,
-        local \\ true,
-        public \\ true
-      ) do
-    with {:ok, result} <-
-           Repo.transaction(fn -> do_announce(user, object, activity_id, local, public) end) do
-      result
-    end
-  end
-
-  defp do_announce(user, object, activity_id, local, public) do
-    with true <- is_announceable?(object, user, public),
-         announce_data <- make_announce_data(user, object, activity_id, public),
-         {:ok, activity} <- insert(announce_data, local),
-         {:ok, object} <- add_announce_to_object(activity, object),
-         :ok <- maybe_federate(activity) do
-      {:ok, activity, object}
-    else
-      false -> {:error, false}
-      {:error, error} -> Repo.rollback(error)
-    end
-  end
-
-  @spec unannounce(User.t(), Object.t(), String.t() | nil, boolean()) ::
-          {:ok, Activity.t(), Object.t()} | {:ok, Object.t()} | {:error, any()}
-  def unannounce(
-        %User{} = actor,
-        %Object{} = object,
-        activity_id \\ nil,
-        local \\ true
-      ) do
-    with {:ok, result} <-
-           Repo.transaction(fn -> do_unannounce(actor, object, activity_id, local) end) do
-      result
-    end
-  end
-
-  defp do_unannounce(actor, object, activity_id, local) do
-    with %Activity{} = announce_activity <- get_existing_announce(actor.ap_id, object),
-         unannounce_data <- make_unannounce_data(actor, announce_activity, activity_id),
-         {:ok, unannounce_activity} <- insert(unannounce_data, local),
-         :ok <- maybe_federate(unannounce_activity),
-         {:ok, _activity} <- Repo.delete(announce_activity),
-         {:ok, object} <- remove_announce_from_object(announce_activity, object) do
-      {:ok, unannounce_activity, object}
-    else
-      nil -> {:ok, object}
-      {:error, error} -> Repo.rollback(error)
-    end
-  end
-
   @spec follow(User.t(), User.t(), String.t() | nil, boolean()) ::
           {:ok, Activity.t()} | {:error, any()}
   def follow(follower, followed, activity_id \\ nil, local \\ true) do
@@ -503,8 +368,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
   defp do_follow(follower, followed, activity_id, local) do
     with data <- make_follow_data(follower, followed, activity_id),
          {:ok, activity} <- insert(data, local),
-         :ok <- maybe_federate(activity),
-         _ <- User.set_follow_state_cache(follower.ap_id, followed.ap_id, activity.data["state"]) do
+         _ <- notify_and_stream(activity),
+         :ok <- maybe_federate(activity) do
       {:ok, activity}
     else
       {:error, error} -> Repo.rollback(error)
@@ -525,6 +390,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
          {:ok, follow_activity} <- update_follow_state(follow_activity, "cancelled"),
          unfollow_data <- make_unfollow_data(follower, followed, follow_activity, activity_id),
          {:ok, activity} <- insert(unfollow_data, local),
+         _ <- notify_and_stream(activity),
          :ok <- maybe_federate(activity) do
       {:ok, activity}
     else
@@ -533,57 +399,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     end
   end
 
-  @spec delete(User.t() | Object.t(), keyword()) :: {:ok, User.t() | Object.t()} | {:error, any()}
-  def delete(entity, options \\ []) do
-    with {:ok, result} <- Repo.transaction(fn -> do_delete(entity, options) end) do
-      result
-    end
-  end
-
-  defp do_delete(%User{ap_id: ap_id, follower_address: follower_address} = user, _) do
-    with data <- %{
-           "to" => [follower_address],
-           "type" => "Delete",
-           "actor" => ap_id,
-           "object" => %{"type" => "Person", "id" => ap_id}
-         },
-         {:ok, activity} <- insert(data, true, true, true),
-         :ok <- maybe_federate(activity) do
-      {:ok, user}
-    end
-  end
-
-  defp do_delete(%Object{data: %{"id" => id, "actor" => actor}} = object, options) do
-    local = Keyword.get(options, :local, true)
-    activity_id = Keyword.get(options, :activity_id, nil)
-    actor = Keyword.get(options, :actor, actor)
-
-    user = User.get_cached_by_ap_id(actor)
-    to = (object.data["to"] || []) ++ (object.data["cc"] || [])
-
-    with create_activity <- Activity.get_create_by_object_ap_id(id),
-         data <-
-           %{
-             "type" => "Delete",
-             "actor" => actor,
-             "object" => id,
-             "to" => to,
-             "deleted_activity_id" => create_activity && create_activity.id
-           }
-           |> maybe_put("id", activity_id),
-         {:ok, activity} <- insert(data, local, false),
-         {:ok, object, _create_activity} <- Object.delete(object),
-         stream_out_participations(object, user),
-         _ <- decrease_replies_count_if_reply(object),
-         {:ok, _actor} <- decrease_note_count_if_public(user, object),
-         :ok <- maybe_federate(activity) do
-      {:ok, activity}
-    else
-      {:error, error} ->
-        Repo.rollback(error)
-    end
-  end
-
   @spec block(User.t(), User.t(), String.t() | nil, boolean()) ::
           {:ok, Activity.t()} | {:error, any()}
   def block(blocker, blocked, activity_id \\ nil, local \\ true) do
@@ -594,7 +409,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
   end
 
   defp do_block(blocker, blocked, activity_id, local) do
-    outgoing_blocks = Config.get([:activitypub, :outgoing_blocks])
     unfollow_blocked = Config.get([:activitypub, :unfollow_blocked])
 
     if unfollow_blocked do
@@ -602,9 +416,9 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
       if follow_activity, do: unfollow(blocker, blocked, nil, local)
     end
 
-    with true <- outgoing_blocks,
-         block_data <- make_block_data(blocker, blocked, activity_id),
+    with block_data <- make_block_data(blocker, blocked, activity_id),
          {:ok, activity} <- insert(block_data, local),
+         _ <- notify_and_stream(activity),
          :ok <- maybe_federate(activity) do
       {:ok, activity}
     else
@@ -612,27 +426,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     end
   end
 
-  @spec unblock(User.t(), User.t(), String.t() | nil, boolean()) ::
-          {:ok, Activity.t()} | {:error, any()} | nil
-  def unblock(blocker, blocked, activity_id \\ nil, local \\ true) do
-    with {:ok, result} <-
-           Repo.transaction(fn -> do_unblock(blocker, blocked, activity_id, local) end) do
-      result
-    end
-  end
-
-  defp do_unblock(blocker, blocked, activity_id, local) do
-    with %Activity{} = block_activity <- fetch_latest_block(blocker, blocked),
-         unblock_data <- make_unblock_data(blocker, blocked, block_activity, activity_id),
-         {:ok, activity} <- insert(unblock_data, local),
-         :ok <- maybe_federate(activity) do
-      {:ok, activity}
-    else
-      nil -> nil
-      {:error, error} -> Repo.rollback(error)
-    end
-  end
-
   @spec flag(map()) :: {:ok, Activity.t()} | {:error, any()}
   def flag(
         %{
@@ -659,6 +452,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     with flag_data <- make_flag_data(params, additional),
          {:ok, activity} <- insert(flag_data, local),
          {:ok, stripped_activity} <- strip_report_status_data(activity),
+         _ <- notify_and_stream(activity),
          :ok <- maybe_federate(stripped_activity) do
       User.all_superusers()
       |> Enum.filter(fn user -> not is_nil(user.email) end)
@@ -682,7 +476,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     }
 
     with true <- origin.ap_id in target.also_known_as,
-         {:ok, activity} <- insert(params, local) do
+         {:ok, activity} <- insert(params, local),
+         _ <- notify_and_stream(activity) do
       maybe_federate(activity)
 
       BackgroundWorker.enqueue("move_following", %{
@@ -697,7 +492,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     end
   end
 
-  defp fetch_activities_for_context_query(context, opts) do
+  def fetch_activities_for_context_query(context, opts) do
     public = [Constants.as_public()]
 
     recipients =
@@ -743,14 +538,27 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     |> Repo.one()
   end
 
-  @spec fetch_public_activities(map(), Pagination.type()) :: [Activity.t()]
-  def fetch_public_activities(opts \\ %{}, pagination \\ :keyset) do
+  @spec fetch_public_or_unlisted_activities(map(), Pagination.type()) :: [Activity.t()]
+  def fetch_public_or_unlisted_activities(opts \\ %{}, pagination \\ :keyset) do
     opts = Map.drop(opts, ["user"])
 
-    [Constants.as_public()]
-    |> fetch_activities_query(opts)
-    |> restrict_unlisted()
-    |> Pagination.fetch_paginated(opts, pagination)
+    query = fetch_activities_query([Constants.as_public()], opts)
+
+    query =
+      if opts["restrict_unlisted"] do
+        restrict_unlisted(query)
+      else
+        query
+      end
+
+    Pagination.fetch_paginated(query, opts, pagination)
+  end
+
+  @spec fetch_public_activities(map(), Pagination.type()) :: [Activity.t()]
+  def fetch_public_activities(opts \\ %{}, pagination \\ :keyset) do
+    opts
+    |> Map.put("restrict_unlisted", true)
+    |> fetch_public_or_unlisted_activities(pagination)
   end
 
   @valid_visibilities ~w[direct unlisted public private]
@@ -829,7 +637,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
   end
 
   defp exclude_visibility(query, %{"exclude_visibilities" => visibility})
-       when visibility not in @valid_visibilities do
+       when visibility not in [nil | @valid_visibilities] do
     Logger.error("Could not exclude visibility to #{visibility}")
     query
   end
@@ -1036,7 +844,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     raise "Can't use the child object without preloading!"
   end
 
-  defp restrict_media(query, %{"only_media" => val}) when val == "true" or val == "1" do
+  defp restrict_media(query, %{"only_media" => val}) when val in [true, "true", "1"] do
     from(
       [_activity, object] in query,
       where: fragment("not (?)->'attachment' = (?)", object.data, ^[])
@@ -1045,16 +853,51 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
 
   defp restrict_media(query, _), do: query
 
-  defp restrict_replies(query, %{"exclude_replies" => val}) when val == "true" or val == "1" do
+  defp restrict_replies(query, %{"exclude_replies" => val}) when val in [true, "true", "1"] do
     from(
       [_activity, object] in query,
       where: fragment("?->>'inReplyTo' is null", object.data)
     )
   end
 
+  defp restrict_replies(query, %{
+         "reply_filtering_user" => user,
+         "reply_visibility" => "self"
+       }) do
+    from(
+      [activity, object] in query,
+      where:
+        fragment(
+          "?->>'inReplyTo' is null OR ? = ANY(?)",
+          object.data,
+          ^user.ap_id,
+          activity.recipients
+        )
+    )
+  end
+
+  defp restrict_replies(query, %{
+         "reply_filtering_user" => user,
+         "reply_visibility" => "following"
+       }) do
+    from(
+      [activity, object] in query,
+      where:
+        fragment(
+          "?->>'inReplyTo' is null OR ? && array_remove(?, ?) OR ? = ?",
+          object.data,
+          ^[user.ap_id | User.get_cached_user_friends_ap_ids(user)],
+          activity.recipients,
+          activity.actor,
+          activity.actor,
+          ^user.ap_id
+        )
+    )
+  end
+
   defp restrict_replies(query, _), do: query
 
-  defp restrict_reblogs(query, %{"exclude_reblogs" => val}) when val == "true" or val == "1" do
+  defp restrict_reblogs(query, %{"exclude_reblogs" => val}) when val in [true, "true", "1"] do
     from(activity in query, where: fragment("?->>'type' != 'Announce'", activity.data))
   end
 
@@ -1133,7 +976,12 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     )
   end
 
-  defp restrict_pinned(query, %{"pinned" => "true", "pinned_activity_ids" => ids}) do
+  # TODO: when all endpoints migrated to OpenAPI compare `pinned` with `true` (boolean) only,
+  # the same for `restrict_media/2`, `restrict_replies/2`, 'restrict_reblogs/2'
+  # and `restrict_muted/2`
+
+  defp restrict_pinned(query, %{"pinned" => pinned, "pinned_activity_ids" => ids})
+       when pinned in [true, "true", "1"] do
     from(activity in query, where: activity.id in ^ids)
   end
 
@@ -1230,17 +1078,17 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
 
   defp fetch_activities_query_ap_ids_ops(opts) do
     source_user = opts["muting_user"]
-    ap_id_relations = if source_user, do: [:mute, :reblog_mute], else: []
+    ap_id_relationships = if source_user, do: [:mute, :reblog_mute], else: []
 
-    ap_id_relations =
-      ap_id_relations ++
+    ap_id_relationships =
+      ap_id_relationships ++
         if opts["blocking_user"] && opts["blocking_user"] == source_user do
           [:block]
         else
           []
         end
 
-    preloaded_ap_ids = User.outgoing_relations_ap_ids(source_user, ap_id_relations)
+    preloaded_ap_ids = User.outgoing_relationships_ap_ids(source_user, ap_id_relationships)
 
     restrict_blocked_opts = Map.merge(%{"blocked_users_ap_ids" => preloaded_ap_ids[:block]}, opts)
     restrict_muted_opts = Map.merge(%{"muted_users_ap_ids" => preloaded_ap_ids[:mute]}, opts)
@@ -1266,6 +1114,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     |> maybe_set_thread_muted_field(opts)
     |> maybe_order(opts)
     |> restrict_recipients(recipients, opts["user"])
+    |> restrict_replies(opts)
     |> restrict_tag(opts)
     |> restrict_tag_reject(opts)
     |> restrict_tag_all(opts)
@@ -1280,7 +1129,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     |> restrict_media(opts)
     |> restrict_visibility(opts)
     |> restrict_thread_visibility(opts, config)
-    |> restrict_replies(opts)
     |> restrict_reblogs(opts)
     |> restrict_pinned(opts)
     |> restrict_muted_reblogs(restrict_muted_reblogs_opts)
@@ -1310,7 +1158,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     |> Activity.with_joined_object()
     |> Object.with_joined_activity()
     |> select([_like, object, activity], %{activity | object: object})
-    |> order_by([like, _, _], desc: like.id)
+    |> order_by([like, _, _], desc_nulls_last: like.id)
     |> Pagination.fetch_paginated(
       Map.merge(params, %{"skip_order" => true}),
       pagination,
@@ -1370,6 +1218,18 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     end
   end
 
+  @spec get_actor_url(any()) :: binary() | nil
+  defp get_actor_url(url) when is_binary(url), do: url
+  defp get_actor_url(%{"href" => href}) when is_binary(href), do: href
+
+  defp get_actor_url(url) when is_list(url) do
+    url
+    |> List.first()
+    |> get_actor_url()
+  end
+
+  defp get_actor_url(_url), do: nil
+
   defp object_to_user_data(data) do
     avatar =
       data["icon"]["url"] &&
@@ -1391,18 +1251,44 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
       |> Enum.filter(fn %{"type" => t} -> t == "PropertyValue" end)
       |> Enum.map(fn fields -> Map.take(fields, ["name", "value"]) end)
 
+    emojis =
+      data
+      |> Map.get("tag", [])
+      |> Enum.filter(fn
+        %{"type" => "Emoji"} -> true
+        _ -> false
+      end)
+      |> Enum.reduce(%{}, fn %{"icon" => %{"url" => url}, "name" => name}, acc ->
+        Map.put(acc, String.trim(name, ":"), url)
+      end)
+
     locked = data["manuallyApprovesFollowers"] || false
     data = Transmogrifier.maybe_fix_user_object(data)
     discoverable = data["discoverable"] || false
     invisible = data["invisible"] || false
     actor_type = data["type"] || "Person"
 
+    public_key =
+      if is_map(data["publicKey"]) && is_binary(data["publicKey"]["publicKeyPem"]) do
+        data["publicKey"]["publicKeyPem"]
+      else
+        nil
+      end
+
+    shared_inbox =
+      if is_map(data["endpoints"]) && is_binary(data["endpoints"]["sharedInbox"]) do
+        data["endpoints"]["sharedInbox"]
+      else
+        nil
+      end
+
     user_data = %{
       ap_id: data["id"],
+      uri: get_actor_url(data["url"]),
       ap_enabled: true,
-      source_data: data,
       banner: banner,
       fields: fields,
+      emoji: emojis,
       locked: locked,
       discoverable: discoverable,
       invisible: invisible,
@@ -1412,7 +1298,10 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
       following_address: data["following"],
       bio: data["summary"],
       actor_type: actor_type,
-      also_known_as: Map.get(data, "alsoKnownAs", [])
+      also_known_as: Map.get(data, "alsoKnownAs", []),
+      public_key: public_key,
+      inbox: data["inbox"],
+      shared_inbox: shared_inbox
     }
 
     # nickname can be nil because of virtual actors
@@ -1453,21 +1342,34 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
   defp normalize_counter(counter) when is_integer(counter), do: counter
   defp normalize_counter(_), do: 0
 
-  defp maybe_update_follow_information(data) do
+  def maybe_update_follow_information(user_data) do
     with {:enabled, true} <- {:enabled, Config.get([:instance, :external_user_synchronization])},
-         {:ok, info} <- fetch_follow_information_for_user(data) do
-      info = Map.merge(data[:info] || %{}, info)
-      Map.put(data, :info, info)
+         {_, true} <- {:user_type_check, user_data[:type] in ["Person", "Service"]},
+         {_, true} <-
+           {:collections_available,
+            !!(user_data[:following_address] && user_data[:follower_address])},
+         {:ok, info} <-
+           fetch_follow_information_for_user(user_data) do
+      info = Map.merge(user_data[:info] || %{}, info)
+
+      user_data
+      |> Map.put(:info, info)
     else
+      {:user_type_check, false} ->
+        user_data
+
+      {:collections_available, false} ->
+        user_data
+
       {:enabled, false} ->
-        data
+        user_data
 
       e ->
         Logger.error(
-          "Follower/Following counter update for #{data.ap_id} failed.\n" <> inspect(e)
+          "Follower/Following counter update for #{user_data.ap_id} failed.\n" <> inspect(e)
         )
 
-        data
+        user_data
     end
   end
 
@@ -1514,11 +1416,22 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
   end
 
   def make_user_from_ap_id(ap_id) do
-    if _user = User.get_cached_by_ap_id(ap_id) do
+    user = User.get_cached_by_ap_id(ap_id)
+
+    if user && !User.ap_enabled?(user) do
       Transmogrifier.upgrade_user_from_ap_id(ap_id)
     else
       with {:ok, data} <- fetch_and_prepare_user_from_ap_id(ap_id) do
-        User.insert_or_update_user(data)
+        if user do
+          user
+          |> User.remote_user_changeset(data)
+          |> User.update_and_set_cache()
+        else
+          data
+          |> User.remote_user_changeset()
+          |> Repo.insert()
+          |> User.set_cache()
+        end
       else
         e -> {:error, e}
       end

lib/pleroma/web/activity_pub/activity_pub_controller.ex

@@ -12,13 +12,16 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
   alias Pleroma.Plugs.EnsureAuthenticatedPlug
   alias Pleroma.User
   alias Pleroma.Web.ActivityPub.ActivityPub
+  alias Pleroma.Web.ActivityPub.Builder
   alias Pleroma.Web.ActivityPub.InternalFetchActor
   alias Pleroma.Web.ActivityPub.ObjectView
+  alias Pleroma.Web.ActivityPub.Pipeline
   alias Pleroma.Web.ActivityPub.Relay
   alias Pleroma.Web.ActivityPub.Transmogrifier
   alias Pleroma.Web.ActivityPub.UserView
   alias Pleroma.Web.ActivityPub.Utils
   alias Pleroma.Web.ActivityPub.Visibility
+  alias Pleroma.Web.Endpoint
   alias Pleroma.Web.FederatingPlug
   alias Pleroma.Web.Federator
 
@@ -32,12 +35,13 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
 
   plug(
     EnsureAuthenticatedPlug,
-    [unless_func: &FederatingPlug.federating?/0] when action not in @federating_only_actions
+    [unless_func: &FederatingPlug.federating?/1] when action not in @federating_only_actions
   )
 
+  # Note: :following and :followers must be served even without authentication (as via :api)
   plug(
     EnsureAuthenticatedPlug
-    when action in [:read_inbox, :update_outbox, :whoami, :upload_media, :following, :followers]
+    when action in [:read_inbox, :update_outbox, :whoami, :upload_media]
   )
 
   plug(
@@ -72,8 +76,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
     end
   end
 
-  def object(conn, %{"uuid" => uuid}) do
-    with ap_id <- o_status_url(conn, :object, uuid),
+  def object(conn, _) do
+    with ap_id <- Endpoint.url() <> conn.request_path,
          %Object{} = object <- Object.get_cached_by_ap_id(ap_id),
          {_, true} <- {:public?, Visibility.is_public?(object)} do
       conn
@@ -98,8 +102,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
     conn
   end
 
-  def activity(conn, %{"uuid" => uuid}) do
-    with ap_id <- o_status_url(conn, :activity, uuid),
+  def activity(conn, _params) do
+    with ap_id <- Endpoint.url() <> conn.request_path,
          %Activity{} = activity <- Activity.normalize(ap_id),
          {_, true} <- {:public?, Visibility.is_public?(activity)} do
       conn
@@ -393,7 +397,10 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
     |> json(err)
   end
 
-  defp handle_user_activity(%User{} = user, %{"type" => "Create"} = params) do
+  defp handle_user_activity(
+         %User{} = user,
+         %{"type" => "Create", "object" => %{"type" => "Note"}} = params
+       ) do
     object =
       params["object"]
       |> Map.merge(Map.take(params, ["to", "cc"]))
@@ -412,7 +419,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
   defp handle_user_activity(%User{} = user, %{"type" => "Delete"} = params) do
     with %Object{} = object <- Object.normalize(params["object"]),
          true <- user.is_moderator || user.ap_id == object.data["actor"],
-         {:ok, delete} <- ActivityPub.delete(object) do
+         {:ok, delete_data, _} <- Builder.delete(user, object.data["id"]),
+         {:ok, delete, _} <- Pipeline.common_pipeline(delete_data, local: true) do
       {:ok, delete}
     else
       _ -> {:error, dgettext("errors", "Can't delete object")}
@@ -421,7 +429,10 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
 
   defp handle_user_activity(%User{} = user, %{"type" => "Like"} = params) do
     with %Object{} = object <- Object.normalize(params["object"]),
-         {:ok, activity, _object} <- ActivityPub.like(user, object) do
+         {_, {:ok, like_object, meta}} <- {:build_object, Builder.like(user, object)},
+         {_, {:ok, %Activity{} = activity, _meta}} <-
+           {:common_pipeline,
+            Pipeline.common_pipeline(like_object, Keyword.put(meta, :local, true))} do
       {:ok, activity}
     else
       _ -> {:error, dgettext("errors", "Can't like object")}

lib/pleroma/web/activity_pub/builder.ex

@@ -0,0 +1,146 @@
+defmodule Pleroma.Web.ActivityPub.Builder do
+  @moduledoc """
+  This module builds the objects. Meant to be used for creating local objects.
+
+  This module encodes our addressing policies and general shape of our objects.
+  """
+
+  alias Pleroma.Object
+  alias Pleroma.User
+  alias Pleroma.Web.ActivityPub.Relay
+  alias Pleroma.Web.ActivityPub.Utils
+  alias Pleroma.Web.ActivityPub.Visibility
+
+  require Pleroma.Constants
+
+  @spec emoji_react(User.t(), Object.t(), String.t()) :: {:ok, map(), keyword()}
+  def emoji_react(actor, object, emoji) do
+    with {:ok, data, meta} <- object_action(actor, object) do
+      data =
+        data
+        |> Map.put("content", emoji)
+        |> Map.put("type", "EmojiReact")
+
+      {:ok, data, meta}
+    end
+  end
+
+  @spec undo(User.t(), Activity.t()) :: {:ok, map(), keyword()}
+  def undo(actor, object) do
+    {:ok,
+     %{
+       "id" => Utils.generate_activity_id(),
+       "actor" => actor.ap_id,
+       "type" => "Undo",
+       "object" => object.data["id"],
+       "to" => object.data["to"] || [],
+       "cc" => object.data["cc"] || []
+     }, []}
+  end
+
+  @spec delete(User.t(), String.t()) :: {:ok, map(), keyword()}
+  def delete(actor, object_id) do
+    object = Object.normalize(object_id, false)
+
+    user = !object && User.get_cached_by_ap_id(object_id)
+
+    to =
+      case {object, user} do
+        {%Object{}, _} ->
+          # We are deleting an object, address everyone who was originally mentioned
+          (object.data["to"] || []) ++ (object.data["cc"] || [])
+
+        {_, %User{follower_address: follower_address}} ->
+          # We are deleting a user, address the followers of that user
+          [follower_address]
+      end
+
+    {:ok,
+     %{
+       "id" => Utils.generate_activity_id(),
+       "actor" => actor.ap_id,
+       "object" => object_id,
+       "to" => to,
+       "type" => "Delete"
+     }, []}
+  end
+
+  @spec tombstone(String.t(), String.t()) :: {:ok, map(), keyword()}
+  def tombstone(actor, id) do
+    {:ok,
+     %{
+       "id" => id,
+       "actor" => actor,
+       "type" => "Tombstone"
+     }, []}
+  end
+
+  @spec like(User.t(), Object.t()) :: {:ok, map(), keyword()}
+  def like(actor, object) do
+    with {:ok, data, meta} <- object_action(actor, object) do
+      data =
+        data
+        |> Map.put("type", "Like")
+
+      {:ok, data, meta}
+    end
+  end
+
+  @spec announce(User.t(), Object.t(), keyword()) :: {:ok, map(), keyword()}
+  def announce(actor, object, options \\ []) do
+    public? = Keyword.get(options, :public, false)
+
+    to =
+      cond do
+        actor.ap_id == Relay.relay_ap_id() ->
+          [actor.follower_address]
+
+        public? ->
+          [actor.follower_address, object.data["actor"], Pleroma.Constants.as_public()]
+
+        true ->
+          [actor.follower_address, object.data["actor"]]
+      end
+
+    {:ok,
+     %{
+       "id" => Utils.generate_activity_id(),
+       "actor" => actor.ap_id,
+       "object" => object.data["id"],
+       "to" => to,
+       "context" => object.data["context"],
+       "type" => "Announce",
+       "published" => Utils.make_date()
+     }, []}
+  end
+
+  @spec object_action(User.t(), Object.t()) :: {:ok, map(), keyword()}
+  defp object_action(actor, object) do
+    object_actor = User.get_cached_by_ap_id(object.data["actor"])
+
+    # Address the actor of the object, and our actor's follower collection if the post is public.
+    to =
+      if Visibility.is_public?(object) do
+        [actor.follower_address, object.data["actor"]]
+      else
+        [object.data["actor"]]
+      end
+
+    # CC everyone who's been addressed in the object, except ourself and the object actor's
+    # follower collection
+    cc =
+      (object.data["to"] ++ (object.data["cc"] || []))
+      |> List.delete(actor.ap_id)
+      |> List.delete(object_actor.follower_address)
+
+    {:ok,
+     %{
+       "id" => Utils.generate_activity_id(),
+       "actor" => actor.ap_id,
+       "object" => object.data["id"],
+       "to" => to,
+       "cc" => cc,
+       "context" => object.data["context"]
+     }, []}
+  end
+end

lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2019 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.ActivityPub.MRF.AntiFollowbotPolicy do

lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex

@@ -12,17 +12,23 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do
 
   require Logger
 
-  @hackney_options [
-    pool: :media,
-    recv_timeout: 10_000
+  @options [
+    pool: :media
   ]
 
   def perform(:prefetch, url) do
     Logger.debug("Prefetching #{inspect(url)}")
 
+    opts =
+      if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Hackney do
+        Keyword.put(@options, :recv_timeout, 10_000)
+      else
+        @options
+      end
+
     url
     |> MediaProxy.url()
-    |> HTTP.get([], adapter: @hackney_options)
+    |> HTTP.get([], adapter: opts)
   end
 
   def perform(:preload, %{"object" => %{"attachment" => attachments}} = _message) do

lib/pleroma/web/activity_pub/mrf/no_placeholder_text_policy.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2019 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.ActivityPub.MRF.NoPlaceholderTextPolicy do

lib/pleroma/web/activity_pub/mrf/object_age_policy.ex

@@ -11,7 +11,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.ObjectAgePolicy do
   @moduledoc "Filter activities depending on their age"
   @behaviour Pleroma.Web.ActivityPub.MRF
 
-  defp check_date(%{"published" => published} = message) do
+  defp check_date(%{"object" => %{"published" => published}} = message) do
     with %DateTime{} = now <- DateTime.utc_now(),
          {:ok, %DateTime{} = then, _} <- DateTime.from_iso8601(published),
          max_ttl <- Config.get([:mrf_object_age, :threshold]),
@@ -96,5 +96,11 @@ defmodule Pleroma.Web.ActivityPub.MRF.ObjectAgePolicy do
   def filter(message), do: {:ok, message}
 
   @impl true
-  def describe, do: {:ok, %{}}
+  def describe do
+    mrf_object_age =
+      Pleroma.Config.get(:mrf_object_age)
+      |> Enum.into(%{})
+
+    {:ok, %{mrf_object_age: mrf_object_age}}
+  end
 end

lib/pleroma/web/activity_pub/mrf/simple_policy.ex

@@ -148,6 +148,21 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do
 
   defp check_banner_removal(_actor_info, object), do: {:ok, object}
 
+  @impl true
+  def filter(%{"type" => "Delete", "actor" => actor} = object) do
+    %{host: actor_host} = URI.parse(actor)
+
+    reject_deletes =
+      Pleroma.Config.get([:mrf_simple, :reject_deletes])
+      |> MRF.subdomains_regex()
+
+    if MRF.subdomain_match?(reject_deletes, actor_host) do
+      {:reject, nil}
+    else
+      {:ok, object}
+    end
+  end
+
   @impl true
   def filter(%{"actor" => actor} = object) do
     actor_info = URI.parse(actor)

lib/pleroma/web/activity_pub/mrf/steal_emoji_policy.ex

@@ -0,0 +1,97 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ActivityPub.MRF.StealEmojiPolicy do
+  require Logger
+
+  alias Pleroma.Config
+
+  @moduledoc "Detect new emojis by their shortcode and steals them"
+  @behaviour Pleroma.Web.ActivityPub.MRF
+
+  defp remote_host?(host), do: host != Config.get([Pleroma.Web.Endpoint, :url, :host])
+
+  defp accept_host?(host), do: host in Config.get([:mrf_steal_emoji, :hosts], [])
+
+  defp steal_emoji({shortcode, url}) do
+    url = Pleroma.Web.MediaProxy.url(url)
+    {:ok, response} = Pleroma.HTTP.get(url)
+    size_limit = Config.get([:mrf_steal_emoji, :size_limit], 50_000)
+
+    if byte_size(response.body) <= size_limit do
+      emoji_dir_path =
+        Config.get(
+          [:mrf_steal_emoji, :path],
+          Path.join(Config.get([:instance, :static_dir]), "emoji/stolen")
+        )
+
+      extension =
+        url
+        |> URI.parse()
+        |> Map.get(:path)
+        |> Path.basename()
+        |> Path.extname()
+
+      file_path = Path.join([emoji_dir_path, shortcode <> (extension || ".png")])
+
+      try do
+        :ok = File.write(file_path, response.body)
+
+        shortcode
+      rescue
+        e ->
+          Logger.warn("MRF.StealEmojiPolicy: Failed to write to #{file_path}: #{inspect(e)}")
+          nil
+      end
+    else
+      Logger.debug(
+        "MRF.StealEmojiPolicy: :#{shortcode}: at #{url} (#{byte_size(response.body)} B) over size limit (#{
+          size_limit
+        } B)"
+      )
+
+      nil
+    end
+  rescue
+    e ->
+      Logger.warn("MRF.StealEmojiPolicy: Failed to fetch #{url}: #{inspect(e)}")
+      nil
+  end
+
+  @impl true
+  def filter(%{"object" => %{"emoji" => foreign_emojis, "actor" => actor}} = message) do
+    host = URI.parse(actor).host
+
+    if remote_host?(host) and accept_host?(host) do
+      installed_emoji = Pleroma.Emoji.get_all() |> Enum.map(fn {k, _} -> k end)
+
+      new_emojis =
+        foreign_emojis
+        |> Enum.filter(fn {shortcode, _url} -> shortcode not in installed_emoji end)
+        |> Enum.filter(fn {shortcode, _url} ->
+          reject_emoji? =
+            Config.get([:mrf_steal_emoji, :rejected_shortcodes], [])
+            |> Enum.find(false, fn regex -> String.match?(shortcode, regex) end)
+
+          !reject_emoji?
+        end)
+        |> Enum.map(&steal_emoji(&1))
+        |> Enum.filter(& &1)
+
+      if !Enum.empty?(new_emojis) do
+        Logger.info("Stole new emojis: #{inspect(new_emojis)}")
+        Pleroma.Emoji.reload()
+      end
+    end
+
+    {:ok, message}
+  end
+
+  def filter(message), do: {:ok, message}
+
+  @impl true
+  def describe do
+    {:ok, %{}}
+  end
+end

lib/pleroma/web/activity_pub/object_validator.ex

@@ -0,0 +1,94 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ActivityPub.ObjectValidator do
+  @moduledoc """
+  This module is responsible for validating an object (which can be an activity)
+  and checking if it is both well formed and also compatible with our view of
+  the system.
+  """
+
+  alias Pleroma.Object
+  alias Pleroma.User
+  alias Pleroma.Web.ActivityPub.ObjectValidators.AnnounceValidator
+  alias Pleroma.Web.ActivityPub.ObjectValidators.DeleteValidator
+  alias Pleroma.Web.ActivityPub.ObjectValidators.EmojiReactValidator
+  alias Pleroma.Web.ActivityPub.ObjectValidators.LikeValidator
+  alias Pleroma.Web.ActivityPub.ObjectValidators.Types
+  alias Pleroma.Web.ActivityPub.ObjectValidators.UndoValidator
+
+  @spec validate(map(), keyword()) :: {:ok, map(), keyword()} | {:error, any()}
+  def validate(object, meta)
+
+  def validate(%{"type" => "Undo"} = object, meta) do
+    with {:ok, object} <-
+           object
+           |> UndoValidator.cast_and_validate()
+           |> Ecto.Changeset.apply_action(:insert) do
+      object = stringify_keys(object)
+      {:ok, object, meta}
+    end
+  end
+
+  def validate(%{"type" => "Delete"} = object, meta) do
+    with cng <- DeleteValidator.cast_and_validate(object),
+         do_not_federate <- DeleteValidator.do_not_federate?(cng),
+         {:ok, object} <- Ecto.Changeset.apply_action(cng, :insert) do
+      object = stringify_keys(object)
+      meta = Keyword.put(meta, :do_not_federate, do_not_federate)
+      {:ok, object, meta}
+    end
+  end
+
+  def validate(%{"type" => "Like"} = object, meta) do
+    with {:ok, object} <-
+           object |> LikeValidator.cast_and_validate() |> Ecto.Changeset.apply_action(:insert) do
+      object = stringify_keys(object |> Map.from_struct())
+      {:ok, object, meta}
+    end
+  end
+
+  def validate(%{"type" => "EmojiReact"} = object, meta) do
+    with {:ok, object} <-
+           object
+           |> EmojiReactValidator.cast_and_validate()
+           |> Ecto.Changeset.apply_action(:insert) do
+      object = stringify_keys(object |> Map.from_struct())
+      {:ok, object, meta}
+    end
+  end
+
+  def validate(%{"type" => "Announce"} = object, meta) do
+    with {:ok, object} <-
+           object
+           |> AnnounceValidator.cast_and_validate()
+           |> Ecto.Changeset.apply_action(:insert) do
+      object = stringify_keys(object |> Map.from_struct())
+      {:ok, object, meta}
+    end
+  end
+
+  def stringify_keys(%{__struct__: _} = object) do
+    object
+    |> Map.from_struct()
+    |> stringify_keys
+  end
+
+  def stringify_keys(object) do
+    object
+    |> Map.new(fn {key, val} -> {to_string(key), val} end)
+  end
+
+  def fetch_actor(object) do
+    with {:ok, actor} <- Types.ObjectID.cast(object["actor"]) do
+      User.get_or_fetch_by_ap_id(actor)
+    end
+  end
+
+  def fetch_actor_and_object(object) do
+    fetch_actor(object)
+    Object.normalize(object["object"], true)
+    :ok
+  end
+end

lib/pleroma/web/activity_pub/object_validators/announce_validator.ex

@@ -0,0 +1,101 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ActivityPub.ObjectValidators.AnnounceValidator do
+  use Ecto.Schema
+
+  alias Pleroma.Object
+  alias Pleroma.User
+  alias Pleroma.Web.ActivityPub.ObjectValidators.Types
+  alias Pleroma.Web.ActivityPub.Utils
+  alias Pleroma.Web.ActivityPub.Visibility
+
+  import Ecto.Changeset
+  import Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations
+
+  require Pleroma.Constants
+
+  @primary_key false
+
+  embedded_schema do
+    field(:id, Types.ObjectID, primary_key: true)
+    field(:type, :string)
+    field(:object, Types.ObjectID)
+    field(:actor, Types.ObjectID)
+    field(:context, :string, autogenerate: {Utils, :generate_context_id, []})
+    field(:to, Types.Recipients, default: [])
+    field(:cc, Types.Recipients, default: [])
+    field(:published, Types.DateTime)
+  end
+
+  def cast_and_validate(data) do
+    data
+    |> cast_data()
+    |> validate_data()
+  end
+
+  def cast_data(data) do
+    %__MODULE__{}
+    |> changeset(data)
+  end
+
+  def changeset(struct, data) do
+    struct
+    |> cast(data, __schema__(:fields))
+    |> fix_after_cast()
+  end
+
+  def fix_after_cast(cng) do
+    cng
+  end
+
+  def validate_data(data_cng) do
+    data_cng
+    |> validate_inclusion(:type, ["Announce"])
+    |> validate_required([:id, :type, :object, :actor, :to, :cc])
+    |> validate_actor_presence()
+    |> validate_object_presence()
+    |> validate_existing_announce()
+    |> validate_announcable()
+  end
+
+  def validate_announcable(cng) do
+    with actor when is_binary(actor) <- get_field(cng, :actor),
+         object when is_binary(object) <- get_field(cng, :object),
+         %User{} = actor <- User.get_cached_by_ap_id(actor),
+         %Object{} = object <- Object.get_cached_by_ap_id(object),
+         false <- Visibility.is_public?(object) do
+      same_actor = object.data["actor"] == actor.ap_id
+      is_public = Pleroma.Constants.as_public() in (get_field(cng, :to) ++ get_field(cng, :cc))
+
+      cond do
+        same_actor && is_public ->
+          cng
+          |> add_error(:actor, "can not announce this object publicly")
+
+        !same_actor ->
+          cng
+          |> add_error(:actor, "can not announce this object")
+
+        true ->
+          cng
+      end
+    else
+      _ -> cng
+    end
+  end
+
+  def validate_existing_announce(cng) do
+    actor = get_field(cng, :actor)
+    object = get_field(cng, :object)
+
+    if actor && object && Utils.get_existing_announce(actor, %{data: %{"id" => object}}) do
+      cng
+      |> add_error(:actor, "already announced this object")
+      |> add_error(:object, "already announced by this actor")
+    else
+      cng
+    end
+  end
+end

lib/pleroma/web/activity_pub/object_validators/common_validations.ex

@@ -0,0 +1,80 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations do
+  import Ecto.Changeset
+
+  alias Pleroma.Activity
+  alias Pleroma.Object
+  alias Pleroma.User
+
+  def validate_recipients_presence(cng, fields \\ [:to, :cc]) do
+    non_empty =
+      fields
+      |> Enum.map(fn field -> get_field(cng, field) end)
+      |> Enum.any?(fn
+        [] -> false
+        _ -> true
+      end)
+
+    if non_empty do
+      cng
+    else
+      fields
+      |> Enum.reduce(cng, fn field, cng ->
+        cng
+        |> add_error(field, "no recipients in any field")
+      end)
+    end
+  end
+
+  def validate_actor_presence(cng, options \\ []) do
+    field_name = Keyword.get(options, :field_name, :actor)
+
+    cng
+    |> validate_change(field_name, fn field_name, actor ->
+      if User.get_cached_by_ap_id(actor) do
+        []
+      else
+        [{field_name, "can't find user"}]
+      end
+    end)
+  end
+
+  def validate_object_presence(cng, options \\ []) do
+    field_name = Keyword.get(options, :field_name, :object)
+    allowed_types = Keyword.get(options, :allowed_types, false)
+
+    cng
+    |> validate_change(field_name, fn field_name, object_id ->
+      object = Object.get_cached_by_ap_id(object_id) || Activity.get_by_ap_id(object_id)
+
+      cond do
+        !object ->
+          [{field_name, "can't find object"}]
+
+        object && allowed_types && object.data["type"] not in allowed_types ->
+          [{field_name, "object not in allowed types"}]
+
+        true ->
+          []
+      end
+    end)
+  end
+
+  def validate_object_or_user_presence(cng, options \\ []) do
+    field_name = Keyword.get(options, :field_name, :object)
+    options = Keyword.put(options, :field_name, field_name)
+
+    actor_cng =
+      cng
+      |> validate_actor_presence(options)
+
+    object_cng =
+      cng
+      |> validate_object_presence(options)
+
+    if actor_cng.valid?, do: actor_cng, else: object_cng
+  end
+end

lib/pleroma/web/activity_pub/object_validators/create_validator.ex

@@ -0,0 +1,30 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateNoteValidator do
+  use Ecto.Schema
+
+  alias Pleroma.Web.ActivityPub.ObjectValidators.NoteValidator
+  alias Pleroma.Web.ActivityPub.ObjectValidators.Types
+
+  import Ecto.Changeset
+
+  @primary_key false
+
+  embedded_schema do
+    field(:id, Types.ObjectID, primary_key: true)
+    field(:actor, Types.ObjectID)
+    field(:type, :string)
+    field(:to, {:array, :string})
+    field(:cc, {:array, :string})
+    field(:bto, {:array, :string}, default: [])
+    field(:bcc, {:array, :string}, default: [])
+
+    embeds_one(:object, NoteValidator)
+  end
+
+  def cast_data(data) do
+    cast(%__MODULE__{}, data, __schema__(:fields))
+  end
+end