Merge remote-tracking branch 'pleroma/develop' into status-notification-type

Signed-off-by: marcin mikołajczak <git@mkljczk.pl>

lib/mix/tasks/pleroma/config.ex

@@ -304,13 +304,8 @@ defmodule Mix.Tasks.Pleroma.Config do
     System.cmd("mix", ["format", path])
   end
 
-  if Code.ensure_loaded?(Config.Reader) do
-    defp config_header, do: "import Config\r\n\r\n"
-    defp read_file(config_file), do: Config.Reader.read_imports!(config_file)
-  else
-    defp config_header, do: "use Mix.Config\r\n\r\n"
-    defp read_file(config_file), do: Mix.Config.eval!(config_file)
-  end
+  defp config_header, do: "import Config\r\n\r\n"
+  defp read_file(config_file), do: Config.Reader.read_imports!(config_file)
 
   defp write_and_delete(config, file, delete?) do
     config

lib/mix/tasks/pleroma/database.ex

@@ -154,9 +154,8 @@ defmodule Mix.Tasks.Pleroma.Database do
     |> join(:inner, [a], o in Object,
       on:
         fragment(
-          "(?->>'id') = COALESCE((?)->'object'->> 'id', (?)->>'object')",
+          "(?->>'id') = associated_object_id((?))",
           o.data,
-          a.data,
           a.data
         )
     )

lib/mix/tasks/pleroma/instance.ex

@@ -34,7 +34,8 @@ defmodule Mix.Tasks.Pleroma.Instance do
           static_dir: :string,
           listen_ip: :string,
           listen_port: :string,
-          strip_uploads: :string,
+          strip_uploads_location: :string,
+          read_uploads_description: :string,
           anonymize_uploads: :string,
           dedupe_uploads: :string
         ],
@@ -161,7 +162,7 @@ defmodule Mix.Tasks.Pleroma.Instance do
         )
         |> Path.expand()
 
-      {strip_uploads_message, strip_uploads_default} =
+      {strip_uploads_location_message, strip_uploads_location_default} =
         if Pleroma.Utils.command_available?("exiftool") do
           {"Do you want to strip location (GPS) data from uploaded images? This requires exiftool, it was detected as installed. (y/n)",
            "y"}
@@ -170,12 +171,29 @@ defmodule Mix.Tasks.Pleroma.Instance do
            "n"}
         end
 
-      strip_uploads =
+      strip_uploads_location =
         get_option(
           options,
-          :strip_uploads,
-          strip_uploads_message,
-          strip_uploads_default
+          :strip_uploads_location,
+          strip_uploads_location_message,
+          strip_uploads_location_default
+        ) === "y"
+
+      {read_uploads_description_message, read_uploads_description_default} =
+        if Pleroma.Utils.command_available?("exiftool") do
+          {"Do you want to read data from uploaded files so clients can use it to prefill fields like image description? This requires exiftool, it was detected as installed. (y/n)",
+           "y"}
+        else
+          {"Do you want to read data from uploaded files so clients can use it to prefill fields like image description? This requires exiftool, it was detected as not installed, please install it if you answer yes. (y/n)",
+           "n"}
+        end
+
+      read_uploads_description =
+        get_option(
+          options,
+          :read_uploads_description,
+          read_uploads_description_message,
+          read_uploads_description_default
         ) === "y"
 
       anonymize_uploads =
@@ -229,7 +247,8 @@ defmodule Mix.Tasks.Pleroma.Instance do
           listen_port: listen_port,
           upload_filters:
             upload_filters(%{
-              strip: strip_uploads,
+              strip_location: strip_uploads_location,
+              read_description: read_uploads_description,
               anonymize: anonymize_uploads,
               dedupe: dedupe_uploads
             })
@@ -297,12 +316,19 @@ defmodule Mix.Tasks.Pleroma.Instance do
 
   defp upload_filters(filters) when is_map(filters) do
     enabled_filters =
-      if filters.strip do
-        [Pleroma.Upload.Filter.Exiftool]
+      if filters.strip_location do
+        [Pleroma.Upload.Filter.Exiftool.StripLocation]
       else
         []
       end
 
+    enabled_filters =
+      if filters.read_description do
+        enabled_filters ++ [Pleroma.Upload.Filter.Exiftool.ReadDescription]
+      else
+        enabled_filters
+      end
+
     enabled_filters =
       if filters.anonymize do
         enabled_filters ++ [Pleroma.Upload.Filter.AnonymizeFilename]

lib/mix/tasks/pleroma/openapi_spec.ex

@@ -6,7 +6,70 @@ defmodule Mix.Tasks.Pleroma.OpenapiSpec do
   def run([path]) do
     # Load Pleroma application to get version info
     Application.load(:pleroma)
-    spec = Pleroma.Web.ApiSpec.spec(server_specific: false) |> Jason.encode!()
-    File.write(path, spec)
+
+    spec_json = Pleroma.Web.ApiSpec.spec(server_specific: false) |> Jason.encode!()
+    # to get rid of the structs
+    spec_regened = spec_json |> Jason.decode!()
+
+    check_specs!(spec_regened)
+
+    File.write(path, spec_json)
+  end
+
+  defp check_specs!(spec) do
+    with :ok <- check_specs(spec) do
+      :ok
+    else
+      {_, errors} ->
+        IO.puts(IO.ANSI.format([:red, :bright, "Spec check failed, errors:"]))
+        Enum.map(errors, &IO.puts/1)
+
+        raise "Spec check failed"
+    end
+  end
+
+  def check_specs(spec) do
+    errors =
+      spec["paths"]
+      |> Enum.flat_map(fn {path, %{} = endpoints} ->
+        Enum.map(
+          endpoints,
+          fn {method, endpoint} ->
+            with :ok <- check_endpoint(spec, endpoint) do
+              :ok
+            else
+              error ->
+                "#{endpoint["operationId"]} (#{method} #{path}): #{error}"
+            end
+          end
+        )
+        |> Enum.reject(fn res -> res == :ok end)
+      end)
+
+    if errors == [] do
+      :ok
+    else
+      {:error, errors}
+    end
+  end
+
+  defp check_endpoint(spec, endpoint) do
+    valid_tags = available_tags(spec)
+
+    with {_, [_ | _] = tags} <- {:tags, endpoint["tags"]},
+         {_, []} <- {:unavailable, Enum.reject(tags, &(&1 in valid_tags))} do
+      :ok
+    else
+      {:tags, _} ->
+        "No tags specified"
+
+      {:unavailable, tags} ->
+        "Tags #{inspect(tags)} not available. Please add it in \"x-tagGroups\" in Pleroma.Web.ApiSpec"
+    end
+  end
+
+  defp available_tags(spec) do
+    spec["x-tagGroups"]
+    |> Enum.flat_map(fn %{"tags" => tags} -> tags end)
   end
 end

lib/mix/tasks/pleroma/user.ex

@@ -112,9 +112,10 @@ defmodule Mix.Tasks.Pleroma.User do
          {:ok, token} <- Pleroma.PasswordResetToken.create_token(user) do
       shell_info("Generated password reset token for #{user.nickname}")
 
-      IO.puts("URL: #{Pleroma.Web.Router.Helpers.reset_password_url(Pleroma.Web.Endpoint,
-      :reset,
-      token.token)}")
+      url =
+        Pleroma.Web.Router.Helpers.reset_password_url(Pleroma.Web.Endpoint, :reset, token.token)
+
+      IO.puts("URL: #{url}")
     else
       _ ->
         shell_error("No local user #{nickname}")
@@ -421,6 +422,38 @@ defmodule Mix.Tasks.Pleroma.User do
     |> Stream.run()
   end
 
+  def run(["fix_follow_state", local_user, remote_user]) do
+    start_pleroma()
+
+    with {:local, %User{} = local} <- {:local, User.get_by_nickname(local_user)},
+         {:remote, %User{} = remote} <- {:remote, User.get_by_nickname(remote_user)},
+         {:follow_data, %{data: %{"state" => request_state}}} <-
+           {:follow_data, Pleroma.Web.ActivityPub.Utils.fetch_latest_follow(local, remote)} do
+      calculated_state = User.following?(local, remote)
+
+      shell_info(
+        "Request state is #{request_state}, vs calculated state of following=#{calculated_state}"
+      )
+
+      if calculated_state == false && request_state == "accept" do
+        shell_info("Discrepancy found, fixing")
+        Pleroma.Web.CommonAPI.reject_follow_request(local, remote)
+        shell_info("Relationship fixed")
+      else
+        shell_info("No discrepancy found")
+      end
+    else
+      {:local, _} ->
+        shell_error("No local user #{local_user}")
+
+      {:remote, _} ->
+        shell_error("No remote user #{remote_user}")
+
+      {:follow_data, _} ->
+        shell_error("No follow data for #{local_user} and #{remote_user}")
+    end
+  end
+
   defp set_moderator(user, value) do
     {:ok, user} =
       user

lib/pleroma/activity.ex

@@ -53,7 +53,7 @@ defmodule Pleroma.Activity do
     #
     # ```
     # |> join(:inner, [activity], o in Object,
-    #      on: fragment("(?->>'id') = COALESCE((?)->'object'->> 'id', (?)->>'object')",
+    #      on: fragment("(?->>'id') = associated_object_id((?))",
     #        o.data, activity.data, activity.data))
     # |> preload([activity, object], [object: object])
     # ```
@@ -69,9 +69,8 @@ defmodule Pleroma.Activity do
     join(query, join_type, [activity], o in Object,
       on:
         fragment(
-          "(?->>'id') = COALESCE(?->'object'->>'id', ?->>'object')",
+          "(?->>'id') = associated_object_id(?)",
           o.data,
-          activity.data,
           activity.data
         ),
       as: :object
@@ -362,9 +361,11 @@ defmodule Pleroma.Activity do
   end
 
   def restrict_deactivated_users(query) do
-    deactivated_users_query = from(u in User.Query.build(%{deactivated: true}), select: u.ap_id)
-
-    from(activity in query, where: activity.actor not in subquery(deactivated_users_query))
+    query
+    |> join(:inner, [activity], user in User,
+      as: :user,
+      on: activity.actor == user.ap_id and user.is_active == true
+    )
   end
 
   defdelegate search(user, query, options \\ []), to: Pleroma.Activity.Search

lib/pleroma/activity/html.ex

@@ -8,6 +8,40 @@ defmodule Pleroma.Activity.HTML do
 
   @cachex Pleroma.Config.get([:cachex, :provider], Cachex)
 
+  # We store a list of cache keys related to an activity in a
+  # separate cache, scrubber_management_cache. It has the same
+  # size as scrubber_cache (see application.ex). Every time we add
+  # a cache to scrubber_cache, we update scrubber_management_cache.
+  #
+  # The most recent write of a certain key in the management cache
+  # is the same as the most recent write of any record related to that
+  # key in the main cache.
+  # Assuming LRW ( https://hexdocs.pm/cachex/Cachex.Policy.LRW.html ),
+  # this means when the management cache is evicted by cachex, all
+  # related records in the main cache will also have been evicted.
+
+  defp get_cache_keys_for(activity_id) do
+    with {:ok, list} when is_list(list) <- @cachex.get(:scrubber_management_cache, activity_id) do
+      list
+    else
+      _ -> []
+    end
+  end
+
+  defp add_cache_key_for(activity_id, additional_key) do
+    current = get_cache_keys_for(activity_id)
+
+    unless additional_key in current do
+      @cachex.put(:scrubber_management_cache, activity_id, [additional_key | current])
+    end
+  end
+
+  def invalidate_cache_for(activity_id) do
+    keys = get_cache_keys_for(activity_id)
+    Enum.map(keys, &@cachex.del(:scrubber_cache, &1))
+    @cachex.del(:scrubber_management_cache, activity_id)
+  end
+
   def get_cached_scrubbed_html_for_activity(
         content,
         scrubbers,
@@ -19,6 +53,8 @@ defmodule Pleroma.Activity.HTML do
 
     @cachex.fetch!(:scrubber_cache, key, fn _key ->
       object = Object.normalize(activity, fetch: false)
+
+      add_cache_key_for(activity.id, key)
       HTML.ensure_scrubbed_html(content, scrubbers, object.data["fake"] || false, callback)
     end)
   end

lib/pleroma/activity/ir/topics.ex

@@ -13,6 +13,14 @@ defmodule Pleroma.Activity.Ir.Topics do
     |> List.flatten()
   end
 
+  defp generate_topics(%{data: %{"type" => "ChatMessage"}}, %{data: %{"type" => "Delete"}}) do
+    ["user", "user:pleroma_chat"]
+  end
+
+  defp generate_topics(%{data: %{"type" => "ChatMessage"}}, %{data: %{"type" => "Create"}}) do
+    []
+  end
+
   defp generate_topics(%{data: %{"type" => "Answer"}}, _) do
     []
   end
@@ -21,7 +29,7 @@ defmodule Pleroma.Activity.Ir.Topics do
     ["user", "list"] ++ visibility_tags(object, activity)
   end
 
-  defp visibility_tags(object, activity) do
+  defp visibility_tags(object, %{data: %{"type" => type}} = activity) when type != "Announce" do
     case Visibility.get_visibility(activity) do
       "public" ->
         if activity.local do
@@ -31,6 +39,10 @@ defmodule Pleroma.Activity.Ir.Topics do
         end
         |> item_creation_tags(object, activity)
 
+      "local" ->
+        ["public:local"]
+        |> item_creation_tags(object, activity)
+
       "direct" ->
         ["direct"]
 
@@ -39,6 +51,10 @@ defmodule Pleroma.Activity.Ir.Topics do
     end
   end
 
+  defp visibility_tags(_object, _activity) do
+    []
+  end
+
   defp item_creation_tags(tags, object, %{data: %{"type" => "Create"}} = activity) do
     tags ++
       remote_topics(activity) ++ hashtags_to_topics(object) ++ attachment_topics(object, activity)
@@ -63,7 +79,18 @@ defmodule Pleroma.Activity.Ir.Topics do
 
   defp attachment_topics(%{data: %{"attachment" => []}}, _act), do: []
 
-  defp attachment_topics(_object, %{local: true}), do: ["public:media", "public:local:media"]
+  defp attachment_topics(_object, %{local: true} = activity) do
+    case Visibility.get_visibility(activity) do
+      "public" ->
+        ["public:media", "public:local:media"]
+
+      "local" ->
+        ["public:local:media"]
+
+      _ ->
+        []
+    end
+  end
 
   defp attachment_topics(_object, %{actor: actor}) when is_binary(actor),
     do: ["public:media", "public:remote:media:" <> URI.parse(actor).host]

lib/pleroma/activity/queries.ex

@@ -52,8 +52,7 @@ defmodule Pleroma.Activity.Queries do
       activity in query,
       where:
         fragment(
-          "coalesce((?)->'object'->>'id', (?)->>'object') = ANY(?)",
-          activity.data,
+          "associated_object_id((?)) = ANY(?)",
           activity.data,
           ^object_ids
         )
@@ -64,8 +63,7 @@ defmodule Pleroma.Activity.Queries do
     from(activity in query,
       where:
         fragment(
-          "coalesce((?)->'object'->>'id', (?)->>'object') = ?",
-          activity.data,
+          "associated_object_id((?)) = ?",
           activity.data,
           ^object_id
         )

lib/pleroma/activity/search.ex

@@ -30,7 +30,7 @@ defmodule Pleroma.Activity.Search do
       Activity
       |> Activity.with_preloaded_object()
       |> Activity.restrict_deactivated_users()
-      |> restrict_public()
+      |> restrict_public(user)
       |> query_with(index_type, search_query, search_function)
       |> maybe_restrict_local(user)
       |> maybe_restrict_author(author)
@@ -57,7 +57,19 @@ defmodule Pleroma.Activity.Search do
 
   def maybe_restrict_blocked(query, _), do: query
 
-  defp restrict_public(q) do
+  defp restrict_public(q, user) when not is_nil(user) do
+    intended_recipients = [
+      Pleroma.Constants.as_public(),
+      Pleroma.Web.ActivityPub.Utils.as_local_public()
+    ]
+
+    from([a, o] in q,
+      where: fragment("?->>'type' = 'Create'", a.data),
+      where: fragment("? && ?", ^intended_recipients, a.recipients)
+    )
+  end
+
+  defp restrict_public(q, _user) do
     from([a, o] in q,
       where: fragment("?->>'type' = 'Create'", a.data),
       where: ^Pleroma.Constants.as_public() in a.recipients

lib/pleroma/announcement.ex

@@ -0,0 +1,160 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Announcement do
+  use Ecto.Schema
+
+  import Ecto.Changeset, only: [cast: 3, validate_required: 2]
+  import Ecto.Query
+
+  alias Pleroma.AnnouncementReadRelationship
+  alias Pleroma.Repo
+
+  @type t :: %__MODULE__{}
+  @primary_key {:id, FlakeId.Ecto.CompatType, autogenerate: true}
+
+  schema "announcements" do
+    field(:data, :map)
+    field(:starts_at, :utc_datetime)
+    field(:ends_at, :utc_datetime)
+    field(:rendered, :map)
+
+    timestamps(type: :utc_datetime)
+  end
+
+  def change(struct, params \\ %{}) do
+    struct
+    |> cast(validate_params(struct, params), [:data, :starts_at, :ends_at, :rendered])
+    |> validate_required([:data])
+  end
+
+  defp validate_params(struct, params) do
+    base_data =
+      %{
+        "content" => "",
+        "all_day" => false
+      }
+      |> Map.merge((struct && struct.data) || %{})
+
+    merged_data =
+      Map.merge(base_data, params.data)
+      |> Map.take(["content", "all_day"])
+
+    params
+    |> Map.merge(%{data: merged_data})
+    |> add_rendered_properties()
+  end
+
+  def add_rendered_properties(params) do
+    {content_html, _, _} =
+      Pleroma.Web.CommonAPI.Utils.format_input(params.data["content"], "text/plain",
+        mentions_format: :full
+      )
+
+    rendered = %{
+      "content" => content_html
+    }
+
+    params
+    |> Map.put(:rendered, rendered)
+  end
+
+  def add(params) do
+    changeset = change(%__MODULE__{}, params)
+
+    Repo.insert(changeset)
+  end
+
+  def update(announcement, params) do
+    changeset = change(announcement, params)
+
+    Repo.update(changeset)
+  end
+
+  def list_all do
+    __MODULE__
+    |> Repo.all()
+  end
+
+  def list_paginated(%{limit: limited_number, offset: offset_number}) do
+    __MODULE__
+    |> limit(^limited_number)
+    |> offset(^offset_number)
+    |> Repo.all()
+  end
+
+  def get_by_id(id) do
+    Repo.get_by(__MODULE__, id: id)
+  end
+
+  def delete_by_id(id) do
+    with announcement when not is_nil(announcement) <- get_by_id(id),
+         {:ok, _} <- Repo.delete(announcement) do
+      :ok
+    else
+      _ ->
+        :error
+    end
+  end
+
+  def read_by?(announcement, user) do
+    AnnouncementReadRelationship.exists?(user, announcement)
+  end
+
+  def mark_read_by(announcement, user) do
+    AnnouncementReadRelationship.mark_read(user, announcement)
+  end
+
+  def render_json(announcement, opts \\ []) do
+    extra_params =
+      case Keyword.fetch(opts, :for) do
+        {:ok, user} when not is_nil(user) ->
+          %{read: read_by?(announcement, user)}
+
+        _ ->
+          %{}
+      end
+
+    admin_extra_params =
+      case Keyword.fetch(opts, :admin) do
+        {:ok, true} ->
+          %{pleroma: %{raw_content: announcement.data["content"]}}
+
+        _ ->
+          %{}
+      end
+
+    base = %{
+      id: announcement.id,
+      content: announcement.rendered["content"],
+      starts_at: announcement.starts_at,
+      ends_at: announcement.ends_at,
+      all_day: announcement.data["all_day"],
+      published_at: announcement.inserted_at,
+      updated_at: announcement.updated_at,
+      mentions: [],
+      statuses: [],
+      tags: [],
+      emojis: [],
+      reactions: []
+    }
+
+    base
+    |> Map.merge(extra_params)
+    |> Map.merge(admin_extra_params)
+  end
+
+  # "visible" means:
+  # starts_at < time < ends_at
+  def list_all_visible_when(time) do
+    __MODULE__
+    |> where([a], is_nil(a.starts_at) or a.starts_at < ^time)
+    |> where([a], is_nil(a.ends_at) or a.ends_at > ^time)
+    |> Repo.all()
+  end
+
+  def list_all_visible do
+    list_all_visible_when(DateTime.now("Etc/UTC") |> elem(1))
+  end
+end

lib/pleroma/announcement_read_relationship.ex

@@ -0,0 +1,55 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.AnnouncementReadRelationship do
+  use Ecto.Schema
+
+  import Ecto.Changeset
+
+  alias FlakeId.Ecto.CompatType
+  alias Pleroma.Announcement
+  alias Pleroma.Repo
+  alias Pleroma.User
+
+  @type t :: %__MODULE__{}
+
+  schema "announcement_read_relationships" do
+    belongs_to(:user, User, type: CompatType)
+    belongs_to(:announcement, Announcement, type: CompatType)
+
+    timestamps(updated_at: false)
+  end
+
+  def mark_read(user, announcement) do
+    %__MODULE__{}
+    |> cast(%{user_id: user.id, announcement_id: announcement.id}, [:user_id, :announcement_id])
+    |> validate_required([:user_id, :announcement_id])
+    |> foreign_key_constraint(:user_id)
+    |> foreign_key_constraint(:announcement_id)
+    |> unique_constraint([:user_id, :announcement_id])
+    |> Repo.insert()
+  end
+
+  def mark_unread(user, announcement) do
+    with relationship <- get(user, announcement),
+         {:exists, true} <- {:exists, not is_nil(relationship)},
+         {:ok, _} <- Repo.delete(relationship) do
+      :ok
+    else
+      {:exists, false} ->
+        :ok
+
+      _ ->
+        :error
+    end
+  end
+
+  def get(user, announcement) do
+    Repo.get_by(__MODULE__, user_id: user.id, announcement_id: announcement.id)
+  end
+
+  def exists?(user, announcement) do
+    not is_nil(get(user, announcement))
+  end
+end

lib/pleroma/application.ex

@@ -94,7 +94,8 @@ defmodule Pleroma.Application do
         Pleroma.Repo,
         Config.TransferTask,
         Pleroma.Emoji,
-        Pleroma.Web.Plugs.RateLimiter.Supervisor
+        Pleroma.Web.Plugs.RateLimiter.Supervisor,
+        {Task.Supervisor, name: Pleroma.TaskSupervisor}
       ] ++
         cachex_children() ++
         http_children(adapter, @mix_env) ++
@@ -112,7 +113,17 @@ defmodule Pleroma.Application do
 
     # See http://elixir-lang.org/docs/stable/elixir/Supervisor.html
     # for other strategies and supported options
-    opts = [strategy: :one_for_one, name: Pleroma.Supervisor]
+    # If we have a lot of caches, default max_restarts can cause test
+    # resets to fail.
+    # Go for the default 3 unless we're in test
+    max_restarts =
+      if @mix_env == :test do
+        100
+      else
+        3
+      end
+
+    opts = [strategy: :one_for_one, name: Pleroma.Supervisor, max_restarts: max_restarts]
     result = Supervisor.start_link(children, opts)
 
     set_postgres_server_version()
@@ -189,6 +200,7 @@ defmodule Pleroma.Application do
       build_cachex("object", default_ttl: 25_000, ttl_interval: 1000, limit: 2500),
       build_cachex("rich_media", default_ttl: :timer.minutes(120), limit: 5000),
       build_cachex("scrubber", limit: 2500),
+      build_cachex("scrubber_management", limit: 2500),
       build_cachex("idempotency", expiration: idempotency_expiration(), limit: 2500),
       build_cachex("web_resp", limit: 2500),
       build_cachex("emoji_packs", expiration: emoji_packs_expiration(), limit: 10),
@@ -197,7 +209,8 @@ defmodule Pleroma.Application do
       build_cachex("chat_message_id_idempotency_key",
         expiration: chat_message_id_idempotency_key_expiration(),
         limit: 500_000
-      )
+      ),
+      build_cachex("rel_me", limit: 2500)
     ]
   end
 
@@ -238,7 +251,8 @@ defmodule Pleroma.Application do
 
   defp background_migrators do
     [
-      Pleroma.Migrators.HashtagsTableMigrator
+      Pleroma.Migrators.HashtagsTableMigrator,
+      Pleroma.Migrators.ContextObjectsDeletionMigrator
     ]
   end
 

lib/pleroma/application_requirements.ex

@@ -164,7 +164,8 @@ defmodule Pleroma.ApplicationRequirements do
 
   defp check_system_commands!(:ok) do
     filter_commands_statuses = [
-      check_filter(Pleroma.Upload.Filter.Exiftool, "exiftool"),
+      check_filter(Pleroma.Upload.Filter.Exiftool.StripLocation, "exiftool"),
+      check_filter(Pleroma.Upload.Filter.Exiftool.ReadDescription, "exiftool"),
       check_filter(Pleroma.Upload.Filter.Mogrify, "mogrify"),
       check_filter(Pleroma.Upload.Filter.Mogrifun, "mogrify"),
       check_filter(Pleroma.Upload.Filter.AnalyzeMetadata, "mogrify"),

lib/pleroma/bbs/handler.ex

@@ -42,8 +42,45 @@ defmodule Pleroma.BBS.Handler do
 
   def puts_activity(activity) do
     status = Pleroma.Web.MastodonAPI.StatusView.render("show.json", %{activity: activity})
+
     IO.puts("-- #{status.id} by #{status.account.display_name} (#{status.account.acct})")
-    IO.puts(HTML.strip_tags(status.content))
+
+    status.content
+    |> String.split("<br/>")
+    |> Enum.map(&HTML.strip_tags/1)
+    |> Enum.map(&HtmlEntities.decode/1)
+    |> Enum.map(&IO.puts/1)
+  end
+
+  def puts_notification(activity, user) do
+    notification =
+      Pleroma.Web.MastodonAPI.NotificationView.render("show.json", %{
+        notification: activity,
+        for: user
+      })
+
+    IO.puts(
+      "== (#{notification.type}) #{notification.status.id} by #{notification.account.display_name} (#{notification.account.acct})"
+    )
+
+    notification.status.content
+    |> String.split("<br/>")
+    |> Enum.map(&HTML.strip_tags/1)
+    |> Enum.map(&HtmlEntities.decode/1)
+    |> (fn x ->
+          case x do
+            [content] ->
+              "> " <> content
+
+            [head | _tail] ->
+              # "> " <> hd <> "..."
+              head
+              |> String.slice(1, 80)
+              |> (fn x -> "> " <> x <> "..." end).()
+          end
+        end).()
+    |> IO.puts()
+
     IO.puts("")
   end
 
@@ -53,6 +90,11 @@ defmodule Pleroma.BBS.Handler do
     IO.puts("home - Show the home timeline")
     IO.puts("p <text> - Post the given text")
     IO.puts("r <id> <text> - Reply to the post with the given id")
+    IO.puts("t <id> - Show a thread from the given id")
+    IO.puts("n - Show notifications")
+    IO.puts("n read - Mark all notifactions as read")
+    IO.puts("f <id> - Favourites the post with the given id")
+    IO.puts("R <id> - Repeat the post with the given id")
     IO.puts("quit - Quit")
 
     state
@@ -73,11 +115,53 @@ defmodule Pleroma.BBS.Handler do
     state
   end
 
+  def handle_command(%{user: user} = state, "t " <> activity_id) do
+    with %Activity{} = activity <- Activity.get_by_id(activity_id) do
+      activities =
+        ActivityPub.fetch_activities_for_context(activity.data["context"], %{
+          blocking_user: user,
+          user: user,
+          exclude_id: activity.id
+        })
+
+      case activities do
+        [] ->
+          activity_id
+          |> Activity.get_by_id()
+          |> puts_activity()
+
+        _ ->
+          activities
+          |> Enum.reverse()
+          |> Enum.each(&puts_activity/1)
+      end
+    else
+      _e -> IO.puts("Could not show this thread...")
+    end
+
+    state
+  end
+
+  def handle_command(%{user: user} = state, "n read") do
+    Pleroma.Notification.clear(user)
+    IO.puts("All notifications were marked as read")
+
+    state
+  end
+
+  def handle_command(%{user: user} = state, "n") do
+    user
+    |> Pleroma.Web.MastodonAPI.MastodonAPI.get_notifications(%{})
+    |> Enum.each(&puts_notification(&1, user))
+
+    state
+  end
+
   def handle_command(%{user: user} = state, "p " <> text) do
     text = String.trim(text)
 
-    with {:ok, _activity} <- CommonAPI.post(user, %{status: text}) do
-      IO.puts("Posted!")
+    with {:ok, activity} <- CommonAPI.post(user, %{status: text}) do
+      IO.puts("Posted! ID: #{activity.id}")
     else
       _e -> IO.puts("Could not post...")
     end
@@ -85,6 +169,19 @@ defmodule Pleroma.BBS.Handler do
     state
   end
 
+  def handle_command(%{user: user} = state, "f " <> id) do
+    id = String.trim(id)
+
+    with %Activity{} = activity <- Activity.get_by_id(id),
+         {:ok, _activity} <- CommonAPI.favorite(user, activity) do
+      IO.puts("Favourited!")
+    else
+      _e -> IO.puts("Could not Favourite...")
+    end
+
+    state
+  end
+
   def handle_command(state, "home") do
     user = state.user
 
@@ -123,7 +220,7 @@ defmodule Pleroma.BBS.Handler do
 
         loop(%{state | counter: state.counter + 1})
 
-      {:error, :interrupted} ->
+      {:input, ^input, {:error, :interrupted}} ->
         IO.puts("Caught Ctrl+C...")
         loop(%{state | counter: state.counter + 1})
 

lib/pleroma/config/deprecation_warnings.ex

@@ -20,6 +20,43 @@ defmodule Pleroma.Config.DeprecationWarnings do
      "\n* `config :pleroma, :instance, mrf_transparency_exclusions` is now `config :pleroma, :mrf, transparency_exclusions`"}
   ]
 
+  def check_exiftool_filter do
+    filters = Config.get([Pleroma.Upload]) |> Keyword.get(:filters, [])
+
+    if Pleroma.Upload.Filter.Exiftool in filters do
+      Logger.warn("""
+      !!!DEPRECATION WARNING!!!
+      Your config is using Exiftool as a filter instead of Exiftool.StripLocation. This should work for now, but you are advised to change to the new configuration to prevent possible issues later:
+
+      ```
+      config :pleroma, Pleroma.Upload,
+        filters: [Pleroma.Upload.Filter.Exiftool]
+      ```
+
+      Is now
+
+
+      ```
+      config :pleroma, Pleroma.Upload,
+        filters: [Pleroma.Upload.Filter.Exiftool.StripLocation]
+      ```
+      """)
+
+      new_config =
+        filters
+        |> Enum.map(fn
+          Pleroma.Upload.Filter.Exiftool -> Pleroma.Upload.Filter.Exiftool.StripLocation
+          filter -> filter
+        end)
+
+      Config.put([Pleroma.Upload, :filters], new_config)
+
+      :error
+    else
+      :ok
+    end
+  end
+
   def check_simple_policy_tuples do
     has_strings =
       Config.get([:mrf_simple])
@@ -180,7 +217,8 @@ defmodule Pleroma.Config.DeprecationWarnings do
       check_old_chat_shoutbox(),
       check_quarantined_instances_tuples(),
       check_transparency_exclusions_tuples(),
-      check_simple_policy_tuples()
+      check_simple_policy_tuples(),
+      check_exiftool_filter()
     ]
     |> Enum.reduce(:ok, fn
       :ok, :ok -> :ok
@@ -273,7 +311,7 @@ defmodule Pleroma.Config.DeprecationWarnings do
 
     warning_preface = """
     !!!DEPRECATION WARNING!!!
-    Your config is using old setting name `timeout` instead of `recv_timeout` in pool settings. Setting should work for now, but you are advised to change format to scheme with port to prevent possible issues later.
+    Your config is using old setting name `timeout` instead of `recv_timeout` in pool settings. The setting will not take effect until updated.
     """
 
     updated_config =

lib/pleroma/config/loader.ex

@@ -19,21 +19,10 @@ defmodule Pleroma.Config.Loader do
     :tesla
   ]
 
-  if Code.ensure_loaded?(Config.Reader) do
-    @reader Config.Reader
-
-    def read(path), do: @reader.read!(path)
-  else
-    # support for Elixir less than 1.9
-    @reader Mix.Config
-    def read(path) do
-      path
-      |> @reader.eval!()
-      |> elem(0)
-    end
-  end
+  @reader Config.Reader
 
   @spec read(Path.t()) :: keyword()
+  def read(path), do: @reader.read!(path)
 
   @spec merge(keyword(), keyword()) :: keyword()
   def merge(c1, c2), do: @reader.merge(c1, c2)

lib/pleroma/config/transfer_task.ex

@@ -47,7 +47,7 @@ defmodule Pleroma.Config.TransferTask do
       {logger, other} =
         (Repo.all(ConfigDB) ++ deleted_settings)
         |> Enum.map(&merge_with_default/1)
-        |> Enum.split_with(fn {group, _, _, _} -> group in [:logger, :quack] end)
+        |> Enum.split_with(fn {group, _, _, _} -> group in [:logger] end)
 
       logger
       |> Enum.sort()
@@ -104,11 +104,6 @@ defmodule Pleroma.Config.TransferTask do
   end
 
   # change logger configuration in runtime, without restart
-  defp configure({:quack, key, _, merged}) do
-    Logger.configure_backend(Quack.Logger, [{key, merged}])
-    :ok = update_env(:quack, key, merged)
-  end
-
   defp configure({_, :backends, _, merged}) do
     # removing current backends
     Enum.each(Application.get_env(:logger, :backends), &Logger.remove_backend/1)

lib/pleroma/config_db.ex

@@ -163,7 +163,6 @@ defmodule Pleroma.ConfigDB do
   defp only_full_update?(%ConfigDB{group: group, key: key}) do
     full_key_update = [
       {:pleroma, :ecto_repos},
-      {:quack, :meta},
       {:mime, :types},
       {:cors_plug, [:max_age, :methods, :expose, :headers]},
       {:swarm, :node_blacklist},
@@ -386,7 +385,7 @@ defmodule Pleroma.ConfigDB do
 
   @spec module_name?(String.t()) :: boolean()
   def module_name?(string) do
-    Regex.match?(~r/^(Pleroma|Phoenix|Tesla|Quack|Ueberauth|Swoosh)\./, string) or
+    Regex.match?(~r/^(Pleroma|Phoenix|Tesla|Ueberauth|Swoosh)\./, string) or
       string in ["Oban", "Ueberauth", "ExSyslogger", "ConcurrentLimiter"]
   end
 end

lib/pleroma/constants.ex

@@ -27,4 +27,46 @@ defmodule Pleroma.Constants do
     do:
       ~w(index.html robots.txt static static-fe finmoji emoji packs sounds images instance sw.js sw-pleroma.js favicon.png schemas doc embed.js embed.css)
   )
+
+  const(status_updatable_fields,
+    do: [
+      "source",
+      "tag",
+      "updated",
+      "emoji",
+      "content",
+      "summary",
+      "sensitive",
+      "attachment",
+      "generator"
+    ]
+  )
+
+  const(updatable_object_types,
+    do: [
+      "Note",
+      "Question",
+      "Audio",
+      "Video",
+      "Event",
+      "Article",
+      "Page"
+    ]
+  )
+
+  const(actor_types,
+    do: [
+      "Application",
+      "Group",
+      "Organization",
+      "Person",
+      "Service"
+    ]
+  )
+
+  # basic regex, just there to weed out potential mistakes
+  # https://datatracker.ietf.org/doc/html/rfc2045#section-5.1
+  const(mime_regex,
+    do: ~r/^[^[:cntrl:] ()<>@,;:\\"\/\[\]?=]+\/[^[:cntrl:] ()<>@,;:\\"\/\[\]?=]+(; .*)?$/
+  )
 end

lib/pleroma/data_migration.ex

@@ -42,4 +42,5 @@ defmodule Pleroma.DataMigration do
   end
 
   def populate_hashtags_table, do: get_by_name("populate_hashtags_table")
+  def delete_context_objects, do: get_by_name("delete_context_objects")
 end

lib/pleroma/docs/translator.ex

@@ -0,0 +1,10 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Docs.Translator do
+  require Pleroma.Docs.Translator.Compiler
+  require Pleroma.Web.Gettext
+
+  @before_compile Pleroma.Docs.Translator.Compiler
+end

lib/pleroma/docs/translator/compiler.ex

@@ -0,0 +1,119 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Docs.Translator.Compiler do
+  @external_resource "config/description.exs"
+  @raw_config Pleroma.Config.Loader.read("config/description.exs")
+  @raw_descriptions @raw_config[:pleroma][:config_description]
+
+  defmacro __before_compile__(_env) do
+    strings =
+      __MODULE__.descriptions()
+      |> __MODULE__.extract_strings()
+
+    quote do
+      def placeholder do
+        unquote do
+          Enum.map(
+            strings,
+            fn {path, type, string} ->
+              ctxt = msgctxt_for(path, type)
+
+              quote do
+                Pleroma.Web.Gettext.dpgettext_noop(
+                  "config_descriptions",
+                  unquote(ctxt),
+                  unquote(string)
+                )
+              end
+            end
+          )
+        end
+      end
+    end
+  end
+
+  def descriptions do
+    Pleroma.Web.ActivityPub.MRF.config_descriptions()
+    |> Enum.reduce(@raw_descriptions, fn description, acc -> [description | acc] end)
+    |> Pleroma.Docs.Generator.convert_to_strings()
+  end
+
+  def extract_strings(descriptions) do
+    descriptions
+    |> Enum.reduce(%{strings: [], path: []}, &process_item/2)
+    |> Map.get(:strings)
+  end
+
+  defp process_item(entity, acc) do
+    current_level =
+      acc
+      |> process_desc(entity)
+      |> process_label(entity)
+
+    process_children(entity, current_level)
+  end
+
+  defp process_desc(acc, %{description: desc} = item) do
+    %{
+      strings: [{acc.path ++ [key_for(item)], "description", desc} | acc.strings],
+      path: acc.path
+    }
+  end
+
+  defp process_desc(acc, _) do
+    acc
+  end
+
+  defp process_label(acc, %{label: label} = item) do
+    %{
+      strings: [{acc.path ++ [key_for(item)], "label", label} | acc.strings],
+      path: acc.path
+    }
+  end
+
+  defp process_label(acc, _) do
+    acc
+  end
+
+  defp process_children(%{children: children} = item, acc) do
+    current_level = Map.put(acc, :path, acc.path ++ [key_for(item)])
+
+    children
+    |> Enum.reduce(current_level, &process_item/2)
+    |> Map.put(:path, acc.path)
+  end
+
+  defp process_children(_, acc) do
+    acc
+  end
+
+  def msgctxt_for(path, type) do
+    "config #{type} at #{Enum.join(path, " > ")}"
+  end
+
+  defp convert_group({_, group}) do
+    group
+  end
+
+  defp convert_group(group) do
+    group
+  end
+
+  def key_for(%{group: group, key: key}) do
+    "#{convert_group(group)}-#{key}"
+  end
+
+  def key_for(%{group: group}) do
+    convert_group(group)
+  end
+
+  def key_for(%{key: key}) do
+    key
+  end
+
+  def key_for(_) do
+    nil
+  end
+end

lib/pleroma/ecto_type/activity_pub/object_validators/mime.ex

@@ -0,0 +1,25 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.EctoType.ActivityPub.ObjectValidators.MIME do
+  use Ecto.Type
+
+  require Pleroma.Constants
+
+  def type, do: :string
+
+  def cast(mime) when is_binary(mime) do
+    if mime =~ Pleroma.Constants.mime_regex() do
+      {:ok, mime}
+    else
+      {:ok, "application/octet-stream"}
+    end
+  end
+
+  def cast(_), do: :error
+
+  def dump(data), do: {:ok, data}
+
+  def load(data), do: {:ok, data}
+end

lib/pleroma/emoji-test.txt

@@ -1,13 +1,13 @@
 # emoji-test.txt
-# Date: 2021-08-26, 17:22:23 GMT
-# © 2021 Unicode®, Inc.
+# Date: 2022-08-12, 20:24:39 GMT
+# © 2022 Unicode®, Inc.
 # Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries.
-# For terms of use, see http://www.unicode.org/terms_of_use.html
+# For terms of use, see https://www.unicode.org/terms_of_use.html
 #
 # Emoji Keyboard/Display Test Data for UTS #51
-# Version: 14.0
+# Version: 15.0
 #
-# For documentation and usage, see http://www.unicode.org/reports/tr51
+# For documentation and usage, see https://www.unicode.org/reports/tr51
 #
 # This file provides data for testing which emoji forms should be in keyboards and which should also be displayed/processed.
 # Format: code points; status # emoji name
@@ -92,6 +92,7 @@
 1F62C                                                  ; fully-qualified     # 😬 E1.0 grimacing face
 1F62E 200D 1F4A8                                       ; fully-qualified     # 😮‍💨 E13.1 face exhaling
 1F925                                                  ; fully-qualified     # 🤥 E3.0 lying face
+1FAE8                                                  ; fully-qualified     # 🫨 E15.0 shaking face
 
 # subgroup: face-sleepy
 1F60C                                                  ; fully-qualified     # 😌 E0.6 relieved face
@@ -155,7 +156,7 @@
 
 # subgroup: face-negative
 1F624                                                  ; fully-qualified     # 😤 E0.6 face with steam from nose
-1F621                                                  ; fully-qualified     # 😡 E0.6 pouting face
+1F621                                                  ; fully-qualified     # 😡 E0.6 enraged face
 1F620                                                  ; fully-qualified     # 😠 E0.6 angry face
 1F92C                                                  ; fully-qualified     # 🤬 E5.0 face with symbols on mouth
 1F608                                                  ; fully-qualified     # 😈 E1.0 smiling face with horns
@@ -190,8 +191,7 @@
 1F649                                                  ; fully-qualified     # 🙉 E0.6 hear-no-evil monkey
 1F64A                                                  ; fully-qualified     # 🙊 E0.6 speak-no-evil monkey
 
-# subgroup: emotion
-1F48B                                                  ; fully-qualified     # 💋 E0.6 kiss mark
+# subgroup: heart
 1F48C                                                  ; fully-qualified     # 💌 E0.6 love letter
 1F498                                                  ; fully-qualified     # 💘 E0.6 heart with arrow
 1F49D                                                  ; fully-qualified     # 💝 E0.6 heart with ribbon
@@ -210,14 +210,20 @@
 2764 200D 1FA79                                        ; unqualified         # ❤‍🩹 E13.1 mending heart
 2764 FE0F                                              ; fully-qualified     # ❤️ E0.6 red heart
 2764                                                   ; unqualified         # ❤ E0.6 red heart
+1FA77                                                  ; fully-qualified     # 🩷 E15.0 pink heart
 1F9E1                                                  ; fully-qualified     # 🧡 E5.0 orange heart
 1F49B                                                  ; fully-qualified     # 💛 E0.6 yellow heart
 1F49A                                                  ; fully-qualified     # 💚 E0.6 green heart
 1F499                                                  ; fully-qualified     # 💙 E0.6 blue heart
+1FA75                                                  ; fully-qualified     # 🩵 E15.0 light blue heart
 1F49C                                                  ; fully-qualified     # 💜 E0.6 purple heart
 1F90E                                                  ; fully-qualified     # 🤎 E12.0 brown heart
 1F5A4                                                  ; fully-qualified     # 🖤 E3.0 black heart
+1FA76                                                  ; fully-qualified     # 🩶 E15.0 grey heart
 1F90D                                                  ; fully-qualified     # 🤍 E12.0 white heart
+
+# subgroup: emotion
+1F48B                                                  ; fully-qualified     # 💋 E0.6 kiss mark
 1F4AF                                                  ; fully-qualified     # 💯 E0.6 hundred points
 1F4A2                                                  ; fully-qualified     # 💢 E0.6 anger symbol
 1F4A5                                                  ; fully-qualified     # 💥 E0.6 collision
@@ -226,21 +232,20 @@
 1F4A8                                                  ; fully-qualified     # 💨 E0.6 dashing away
 1F573 FE0F                                             ; fully-qualified     # 🕳️ E0.7 hole
 1F573                                                  ; unqualified         # 🕳 E0.7 hole
-1F4A3                                                  ; fully-qualified     # 💣 E0.6 bomb
 1F4AC                                                  ; fully-qualified     # 💬 E0.6 speech balloon
 1F441 FE0F 200D 1F5E8 FE0F                             ; fully-qualified     # 👁️‍🗨️ E2.0 eye in speech bubble
 1F441 200D 1F5E8 FE0F                                  ; unqualified         # 👁‍🗨️ E2.0 eye in speech bubble
-1F441 FE0F 200D 1F5E8                                  ; unqualified         # 👁️‍🗨 E2.0 eye in speech bubble
+1F441 FE0F 200D 1F5E8                                  ; minimally-qualified # 👁️‍🗨 E2.0 eye in speech bubble
 1F441 200D 1F5E8                                       ; unqualified         # 👁‍🗨 E2.0 eye in speech bubble
 1F5E8 FE0F                                             ; fully-qualified     # 🗨️ E2.0 left speech bubble
 1F5E8                                                  ; unqualified         # 🗨 E2.0 left speech bubble
 1F5EF FE0F                                             ; fully-qualified     # 🗯️ E0.7 right anger bubble
 1F5EF                                                  ; unqualified         # 🗯 E0.7 right anger bubble
 1F4AD                                                  ; fully-qualified     # 💭 E1.0 thought balloon
-1F4A4                                                  ; fully-qualified     # 💤 E0.6 zzz
+1F4A4                                                  ; fully-qualified     # 💤 E0.6 ZZZ
 
-# Smileys & Emotion subtotal:		177
-# Smileys & Emotion subtotal:		177	w/o modifiers
+# Smileys & Emotion subtotal:		180
+# Smileys & Emotion subtotal:		180	w/o modifiers
 
 # group: People & Body
 
@@ -300,6 +305,18 @@
 1FAF4 1F3FD                                            ; fully-qualified     # 🫴🏽 E14.0 palm up hand: medium skin tone
 1FAF4 1F3FE                                            ; fully-qualified     # 🫴🏾 E14.0 palm up hand: medium-dark skin tone
 1FAF4 1F3FF                                            ; fully-qualified     # 🫴🏿 E14.0 palm up hand: dark skin tone
+1FAF7                                                  ; fully-qualified     # 🫷 E15.0 leftwards pushing hand
+1FAF7 1F3FB                                            ; fully-qualified     # 🫷🏻 E15.0 leftwards pushing hand: light skin tone
+1FAF7 1F3FC                                            ; fully-qualified     # 🫷🏼 E15.0 leftwards pushing hand: medium-light skin tone
+1FAF7 1F3FD                                            ; fully-qualified     # 🫷🏽 E15.0 leftwards pushing hand: medium skin tone
+1FAF7 1F3FE                                            ; fully-qualified     # 🫷🏾 E15.0 leftwards pushing hand: medium-dark skin tone
+1FAF7 1F3FF                                            ; fully-qualified     # 🫷🏿 E15.0 leftwards pushing hand: dark skin tone
+1FAF8                                                  ; fully-qualified     # 🫸 E15.0 rightwards pushing hand
+1FAF8 1F3FB                                            ; fully-qualified     # 🫸🏻 E15.0 rightwards pushing hand: light skin tone
+1FAF8 1F3FC                                            ; fully-qualified     # 🫸🏼 E15.0 rightwards pushing hand: medium-light skin tone
+1FAF8 1F3FD                                            ; fully-qualified     # 🫸🏽 E15.0 rightwards pushing hand: medium skin tone
+1FAF8 1F3FE                                            ; fully-qualified     # 🫸🏾 E15.0 rightwards pushing hand: medium-dark skin tone
+1FAF8 1F3FF                                            ; fully-qualified     # 🫸🏿 E15.0 rightwards pushing hand: dark skin tone
 
 # subgroup: hand-fingers-partial
 1F44C                                                  ; fully-qualified     # 👌 E0.6 OK hand
@@ -473,11 +490,11 @@
 1F932 1F3FE                                            ; fully-qualified     # 🤲🏾 E5.0 palms up together: medium-dark skin tone
 1F932 1F3FF                                            ; fully-qualified     # 🤲🏿 E5.0 palms up together: dark skin tone
 1F91D                                                  ; fully-qualified     # 🤝 E3.0 handshake
-1F91D 1F3FB                                            ; fully-qualified     # 🤝🏻 E3.0 handshake: light skin tone
-1F91D 1F3FC                                            ; fully-qualified     # 🤝🏼 E3.0 handshake: medium-light skin tone
-1F91D 1F3FD                                            ; fully-qualified     # 🤝🏽 E3.0 handshake: medium skin tone
-1F91D 1F3FE                                            ; fully-qualified     # 🤝🏾 E3.0 handshake: medium-dark skin tone
-1F91D 1F3FF                                            ; fully-qualified     # 🤝🏿 E3.0 handshake: dark skin tone
+1F91D 1F3FB                                            ; fully-qualified     # 🤝🏻 E14.0 handshake: light skin tone
+1F91D 1F3FC                                            ; fully-qualified     # 🤝🏼 E14.0 handshake: medium-light skin tone
+1F91D 1F3FD                                            ; fully-qualified     # 🤝🏽 E14.0 handshake: medium skin tone
+1F91D 1F3FE                                            ; fully-qualified     # 🤝🏾 E14.0 handshake: medium-dark skin tone
+1F91D 1F3FF                                            ; fully-qualified     # 🤝🏿 E14.0 handshake: dark skin tone
 1FAF1 1F3FB 200D 1FAF2 1F3FC                           ; fully-qualified     # 🫱🏻‍🫲🏼 E14.0 handshake: light skin tone, medium-light skin tone
 1FAF1 1F3FB 200D 1FAF2 1F3FD                           ; fully-qualified     # 🫱🏻‍🫲🏽 E14.0 handshake: light skin tone, medium skin tone
 1FAF1 1F3FB 200D 1FAF2 1F3FE                           ; fully-qualified     # 🫱🏻‍🫲🏾 E14.0 handshake: light skin tone, medium-dark skin tone
@@ -1455,7 +1472,7 @@
 1F575 1F3FF                                            ; fully-qualified     # 🕵🏿 E2.0 detective: dark skin tone
 1F575 FE0F 200D 2642 FE0F                              ; fully-qualified     # 🕵️‍♂️ E4.0 man detective
 1F575 200D 2642 FE0F                                   ; unqualified         # 🕵‍♂️ E4.0 man detective
-1F575 FE0F 200D 2642                                   ; unqualified         # 🕵️‍♂ E4.0 man detective
+1F575 FE0F 200D 2642                                   ; minimally-qualified # 🕵️‍♂ E4.0 man detective
 1F575 200D 2642                                        ; unqualified         # 🕵‍♂ E4.0 man detective
 1F575 1F3FB 200D 2642 FE0F                             ; fully-qualified     # 🕵🏻‍♂️ E4.0 man detective: light skin tone
 1F575 1F3FB 200D 2642                                  ; minimally-qualified # 🕵🏻‍♂ E4.0 man detective: light skin tone
@@ -1469,7 +1486,7 @@
 1F575 1F3FF 200D 2642                                  ; minimally-qualified # 🕵🏿‍♂ E4.0 man detective: dark skin tone
 1F575 FE0F 200D 2640 FE0F                              ; fully-qualified     # 🕵️‍♀️ E4.0 woman detective
 1F575 200D 2640 FE0F                                   ; unqualified         # 🕵‍♀️ E4.0 woman detective
-1F575 FE0F 200D 2640                                   ; unqualified         # 🕵️‍♀ E4.0 woman detective
+1F575 FE0F 200D 2640                                   ; minimally-qualified # 🕵️‍♀ E4.0 woman detective
 1F575 200D 2640                                        ; unqualified         # 🕵‍♀ E4.0 woman detective
 1F575 1F3FB 200D 2640 FE0F                             ; fully-qualified     # 🕵🏻‍♀️ E4.0 woman detective: light skin tone
 1F575 1F3FB 200D 2640                                  ; minimally-qualified # 🕵🏻‍♀ E4.0 woman detective: light skin tone
@@ -2302,7 +2319,7 @@
 1F3CC 1F3FF                                            ; fully-qualified     # 🏌🏿 E4.0 person golfing: dark skin tone
 1F3CC FE0F 200D 2642 FE0F                              ; fully-qualified     # 🏌️‍♂️ E4.0 man golfing
 1F3CC 200D 2642 FE0F                                   ; unqualified         # 🏌‍♂️ E4.0 man golfing
-1F3CC FE0F 200D 2642                                   ; unqualified         # 🏌️‍♂ E4.0 man golfing
+1F3CC FE0F 200D 2642                                   ; minimally-qualified # 🏌️‍♂ E4.0 man golfing
 1F3CC 200D 2642                                        ; unqualified         # 🏌‍♂ E4.0 man golfing
 1F3CC 1F3FB 200D 2642 FE0F                             ; fully-qualified     # 🏌🏻‍♂️ E4.0 man golfing: light skin tone
 1F3CC 1F3FB 200D 2642                                  ; minimally-qualified # 🏌🏻‍♂ E4.0 man golfing: light skin tone
@@ -2316,7 +2333,7 @@
 1F3CC 1F3FF 200D 2642                                  ; minimally-qualified # 🏌🏿‍♂ E4.0 man golfing: dark skin tone
 1F3CC FE0F 200D 2640 FE0F                              ; fully-qualified     # 🏌️‍♀️ E4.0 woman golfing
 1F3CC 200D 2640 FE0F                                   ; unqualified         # 🏌‍♀️ E4.0 woman golfing
-1F3CC FE0F 200D 2640                                   ; unqualified         # 🏌️‍♀ E4.0 woman golfing
+1F3CC FE0F 200D 2640                                   ; minimally-qualified # 🏌️‍♀ E4.0 woman golfing
 1F3CC 200D 2640                                        ; unqualified         # 🏌‍♀ E4.0 woman golfing
 1F3CC 1F3FB 200D 2640 FE0F                             ; fully-qualified     # 🏌🏻‍♀️ E4.0 woman golfing: light skin tone
 1F3CC 1F3FB 200D 2640                                  ; minimally-qualified # 🏌🏻‍♀ E4.0 woman golfing: light skin tone
@@ -2427,7 +2444,7 @@
 26F9 1F3FF                                             ; fully-qualified     # ⛹🏿 E2.0 person bouncing ball: dark skin tone
 26F9 FE0F 200D 2642 FE0F                               ; fully-qualified     # ⛹️‍♂️ E4.0 man bouncing ball
 26F9 200D 2642 FE0F                                    ; unqualified         # ⛹‍♂️ E4.0 man bouncing ball
-26F9 FE0F 200D 2642                                    ; unqualified         # ⛹️‍♂ E4.0 man bouncing ball
+26F9 FE0F 200D 2642                                    ; minimally-qualified # ⛹️‍♂ E4.0 man bouncing ball
 26F9 200D 2642                                         ; unqualified         # ⛹‍♂ E4.0 man bouncing ball
 26F9 1F3FB 200D 2642 FE0F                              ; fully-qualified     # ⛹🏻‍♂️ E4.0 man bouncing ball: light skin tone
 26F9 1F3FB 200D 2642                                   ; minimally-qualified # ⛹🏻‍♂ E4.0 man bouncing ball: light skin tone
@@ -2441,7 +2458,7 @@
 26F9 1F3FF 200D 2642                                   ; minimally-qualified # ⛹🏿‍♂ E4.0 man bouncing ball: dark skin tone
 26F9 FE0F 200D 2640 FE0F                               ; fully-qualified     # ⛹️‍♀️ E4.0 woman bouncing ball
 26F9 200D 2640 FE0F                                    ; unqualified         # ⛹‍♀️ E4.0 woman bouncing ball
-26F9 FE0F 200D 2640                                    ; unqualified         # ⛹️‍♀ E4.0 woman bouncing ball
+26F9 FE0F 200D 2640                                    ; minimally-qualified # ⛹️‍♀ E4.0 woman bouncing ball
 26F9 200D 2640                                         ; unqualified         # ⛹‍♀ E4.0 woman bouncing ball
 26F9 1F3FB 200D 2640 FE0F                              ; fully-qualified     # ⛹🏻‍♀️ E4.0 woman bouncing ball: light skin tone
 26F9 1F3FB 200D 2640                                   ; minimally-qualified # ⛹🏻‍♀ E4.0 woman bouncing ball: light skin tone
@@ -2462,7 +2479,7 @@
 1F3CB 1F3FF                                            ; fully-qualified     # 🏋🏿 E2.0 person lifting weights: dark skin tone
 1F3CB FE0F 200D 2642 FE0F                              ; fully-qualified     # 🏋️‍♂️ E4.0 man lifting weights
 1F3CB 200D 2642 FE0F                                   ; unqualified         # 🏋‍♂️ E4.0 man lifting weights
-1F3CB FE0F 200D 2642                                   ; unqualified         # 🏋️‍♂ E4.0 man lifting weights
+1F3CB FE0F 200D 2642                                   ; minimally-qualified # 🏋️‍♂ E4.0 man lifting weights
 1F3CB 200D 2642                                        ; unqualified         # 🏋‍♂ E4.0 man lifting weights
 1F3CB 1F3FB 200D 2642 FE0F                             ; fully-qualified     # 🏋🏻‍♂️ E4.0 man lifting weights: light skin tone
 1F3CB 1F3FB 200D 2642                                  ; minimally-qualified # 🏋🏻‍♂ E4.0 man lifting weights: light skin tone
@@ -2476,7 +2493,7 @@
 1F3CB 1F3FF 200D 2642                                  ; minimally-qualified # 🏋🏿‍♂ E4.0 man lifting weights: dark skin tone
 1F3CB FE0F 200D 2640 FE0F                              ; fully-qualified     # 🏋️‍♀️ E4.0 woman lifting weights
 1F3CB 200D 2640 FE0F                                   ; unqualified         # 🏋‍♀️ E4.0 woman lifting weights
-1F3CB FE0F 200D 2640                                   ; unqualified         # 🏋️‍♀ E4.0 woman lifting weights
+1F3CB FE0F 200D 2640                                   ; minimally-qualified # 🏋️‍♀ E4.0 woman lifting weights
 1F3CB 200D 2640                                        ; unqualified         # 🏋‍♀ E4.0 woman lifting weights
 1F3CB 1F3FB 200D 2640 FE0F                             ; fully-qualified     # 🏋🏻‍♀️ E4.0 woman lifting weights: light skin tone
 1F3CB 1F3FB 200D 2640                                  ; minimally-qualified # 🏋🏻‍♀ E4.0 woman lifting weights: light skin tone
@@ -3262,8 +3279,8 @@
 1FAC2                                                  ; fully-qualified     # 🫂 E13.0 people hugging
 1F463                                                  ; fully-qualified     # 👣 E0.6 footprints
 
-# People & Body subtotal:		2986
-# People & Body subtotal:		506	w/o modifiers
+# People & Body subtotal:		2998
+# People & Body subtotal:		508	w/o modifiers
 
 # group: Component
 
@@ -3306,6 +3323,8 @@
 1F405                                                  ; fully-qualified     # 🐅 E1.0 tiger
 1F406                                                  ; fully-qualified     # 🐆 E1.0 leopard
 1F434                                                  ; fully-qualified     # 🐴 E0.6 horse face
+1FACE                                                  ; fully-qualified     # 🫎 E15.0 moose
+1FACF                                                  ; fully-qualified     # 🫏 E15.0 donkey
 1F40E                                                  ; fully-qualified     # 🐎 E0.6 horse
 1F984                                                  ; fully-qualified     # 🦄 E1.0 unicorn
 1F993                                                  ; fully-qualified     # 🦓 E5.0 zebra
@@ -3373,6 +3392,9 @@
 1F9A9                                                  ; fully-qualified     # 🦩 E12.0 flamingo
 1F99A                                                  ; fully-qualified     # 🦚 E11.0 peacock
 1F99C                                                  ; fully-qualified     # 🦜 E11.0 parrot
+1FABD                                                  ; fully-qualified     # 🪽 E15.0 wing
+1F426 200D 2B1B                                        ; fully-qualified     # 🐦‍⬛ E15.0 black bird
+1FABF                                                  ; fully-qualified     # 🪿 E15.0 goose
 
 # subgroup: animal-amphibian
 1F438                                                  ; fully-qualified     # 🐸 E0.6 frog
@@ -3399,6 +3421,7 @@
 1F419                                                  ; fully-qualified     # 🐙 E0.6 octopus
 1F41A                                                  ; fully-qualified     # 🐚 E0.6 spiral shell
 1FAB8                                                  ; fully-qualified     # 🪸 E14.0 coral
+1FABC                                                  ; fully-qualified     # 🪼 E15.0 jellyfish
 
 # subgroup: animal-bug
 1F40C                                                  ; fully-qualified     # 🐌 E0.6 snail
@@ -3433,6 +3456,7 @@
 1F33B                                                  ; fully-qualified     # 🌻 E0.6 sunflower
 1F33C                                                  ; fully-qualified     # 🌼 E0.6 blossom
 1F337                                                  ; fully-qualified     # 🌷 E0.6 tulip
+1FABB                                                  ; fully-qualified     # 🪻 E15.0 hyacinth
 
 # subgroup: plant-other
 1F331                                                  ; fully-qualified     # 🌱 E0.6 seedling
@@ -3451,9 +3475,10 @@
 1F343                                                  ; fully-qualified     # 🍃 E0.6 leaf fluttering in wind
 1FAB9                                                  ; fully-qualified     # 🪹 E14.0 empty nest
 1FABA                                                  ; fully-qualified     # 🪺 E14.0 nest with eggs
+1F344                                                  ; fully-qualified     # 🍄 E0.6 mushroom
 
-# Animals & Nature subtotal:		151
-# Animals & Nature subtotal:		151	w/o modifiers
+# Animals & Nature subtotal:		159
+# Animals & Nature subtotal:		159	w/o modifiers
 
 # group: Food & Drink
 
@@ -3492,10 +3517,11 @@
 1F966                                                  ; fully-qualified     # 🥦 E5.0 broccoli
 1F9C4                                                  ; fully-qualified     # 🧄 E12.0 garlic
 1F9C5                                                  ; fully-qualified     # 🧅 E12.0 onion
-1F344                                                  ; fully-qualified     # 🍄 E0.6 mushroom
 1F95C                                                  ; fully-qualified     # 🥜 E3.0 peanuts
 1FAD8                                                  ; fully-qualified     # 🫘 E14.0 beans
 1F330                                                  ; fully-qualified     # 🌰 E0.6 chestnut
+1FADA                                                  ; fully-qualified     # 🫚 E15.0 ginger root
+1FADB                                                  ; fully-qualified     # 🫛 E15.0 pea pod
 
 # subgroup: food-prepared
 1F35E                                                  ; fully-qualified     # 🍞 E0.6 bread
@@ -3607,8 +3633,8 @@
 1FAD9                                                  ; fully-qualified     # 🫙 E14.0 jar
 1F3FA                                                  ; fully-qualified     # 🏺 E1.0 amphora
 
-# Food & Drink subtotal:		134
-# Food & Drink subtotal:		134	w/o modifiers
+# Food & Drink subtotal:		135
+# Food & Drink subtotal:		135	w/o modifiers
 
 # group: Travel & Places
 
@@ -3974,11 +4000,10 @@
 1F3AF                                                  ; fully-qualified     # 🎯 E0.6 bullseye
 1FA80                                                  ; fully-qualified     # 🪀 E12.0 yo-yo
 1FA81                                                  ; fully-qualified     # 🪁 E12.0 kite
+1F52B                                                  ; fully-qualified     # 🔫 E0.6 water pistol
 1F3B1                                                  ; fully-qualified     # 🎱 E0.6 pool 8 ball
 1F52E                                                  ; fully-qualified     # 🔮 E0.6 crystal ball
 1FA84                                                  ; fully-qualified     # 🪄 E13.0 magic wand
-1F9FF                                                  ; fully-qualified     # 🧿 E11.0 nazar amulet
-1FAAC                                                  ; fully-qualified     # 🪬 E14.0 hamsa
 1F3AE                                                  ; fully-qualified     # 🎮 E0.6 video game
 1F579 FE0F                                             ; fully-qualified     # 🕹️ E0.7 joystick
 1F579                                                  ; unqualified         # 🕹 E0.7 joystick
@@ -4013,8 +4038,8 @@
 1F9F6                                                  ; fully-qualified     # 🧶 E11.0 yarn
 1FAA2                                                  ; fully-qualified     # 🪢 E13.0 knot
 
-# Activities subtotal:		97
-# Activities subtotal:		97	w/o modifiers
+# Activities subtotal:		96
+# Activities subtotal:		96	w/o modifiers
 
 # group: Objects
 
@@ -4040,6 +4065,7 @@
 1FA73                                                  ; fully-qualified     # 🩳 E12.0 shorts
 1F459                                                  ; fully-qualified     # 👙 E0.6 bikini
 1F45A                                                  ; fully-qualified     # 👚 E0.6 woman’s clothes
+1FAAD                                                  ; fully-qualified     # 🪭 E15.0 folding hand fan
 1F45B                                                  ; fully-qualified     # 👛 E0.6 purse
 1F45C                                                  ; fully-qualified     # 👜 E0.6 handbag
 1F45D                                                  ; fully-qualified     # 👝 E0.6 clutch bag
@@ -4055,6 +4081,7 @@
 1F461                                                  ; fully-qualified     # 👡 E0.6 woman’s sandal
 1FA70                                                  ; fully-qualified     # 🩰 E12.0 ballet shoes
 1F462                                                  ; fully-qualified     # 👢 E0.6 woman’s boot
+1FAAE                                                  ; fully-qualified     # 🪮 E15.0 hair pick
 1F451                                                  ; fully-qualified     # 👑 E0.6 crown
 1F452                                                  ; fully-qualified     # 👒 E0.6 woman’s hat
 1F3A9                                                  ; fully-qualified     # 🎩 E0.6 top hat
@@ -4103,6 +4130,8 @@
 1FA95                                                  ; fully-qualified     # 🪕 E12.0 banjo
 1F941                                                  ; fully-qualified     # 🥁 E3.0 drum
 1FA98                                                  ; fully-qualified     # 🪘 E13.0 long drum
+1FA87                                                  ; fully-qualified     # 🪇 E15.0 maracas
+1FA88                                                  ; fully-qualified     # 🪈 E15.0 flute
 
 # subgroup: phone
 1F4F1                                                  ; fully-qualified     # 📱 E0.6 mobile phone
@@ -4275,7 +4304,7 @@
 1F5E1                                                  ; unqualified         # 🗡 E0.7 dagger
 2694 FE0F                                              ; fully-qualified     # ⚔️ E1.0 crossed swords
 2694                                                   ; unqualified         # ⚔ E1.0 crossed swords
-1F52B                                                  ; fully-qualified     # 🔫 E0.6 water pistol
+1F4A3                                                  ; fully-qualified     # 💣 E0.6 bomb
 1FA83                                                  ; fully-qualified     # 🪃 E13.0 boomerang
 1F3F9                                                  ; fully-qualified     # 🏹 E1.0 bow and arrow
 1F6E1 FE0F                                             ; fully-qualified     # 🛡️ E0.7 shield
@@ -4354,12 +4383,14 @@
 1FAA6                                                  ; fully-qualified     # 🪦 E13.0 headstone
 26B1 FE0F                                              ; fully-qualified     # ⚱️ E1.0 funeral urn
 26B1                                                   ; unqualified         # ⚱ E1.0 funeral urn
+1F9FF                                                  ; fully-qualified     # 🧿 E11.0 nazar amulet
+1FAAC                                                  ; fully-qualified     # 🪬 E14.0 hamsa
 1F5FF                                                  ; fully-qualified     # 🗿 E0.6 moai
 1FAA7                                                  ; fully-qualified     # 🪧 E13.0 placard
 1FAAA                                                  ; fully-qualified     # 🪪 E14.0 identification card
 
-# Objects subtotal:		304
-# Objects subtotal:		304	w/o modifiers
+# Objects subtotal:		310
+# Objects subtotal:		310	w/o modifiers
 
 # group: Symbols
 
@@ -4455,6 +4486,7 @@
 262E                                                   ; unqualified         # ☮ E1.0 peace symbol
 1F54E                                                  ; fully-qualified     # 🕎 E1.0 menorah
 1F52F                                                  ; fully-qualified     # 🔯 E0.6 dotted six-pointed star
+1FAAF                                                  ; fully-qualified     # 🪯 E15.0 khanda
 
 # subgroup: zodiac
 2648                                                   ; fully-qualified     # ♈ E0.6 Aries
@@ -4503,6 +4535,7 @@
 1F505                                                  ; fully-qualified     # 🔅 E1.0 dim button
 1F506                                                  ; fully-qualified     # 🔆 E1.0 bright button
 1F4F6                                                  ; fully-qualified     # 📶 E0.6 antenna bars
+1F6DC                                                  ; fully-qualified     # 🛜 E15.0 wireless
 1F4F3                                                  ; fully-qualified     # 📳 E0.6 vibration mode
 1F4F4                                                  ; fully-qualified     # 📴 E0.6 mobile phone off
 
@@ -4693,8 +4726,8 @@
 1F533                                                  ; fully-qualified     # 🔳 E0.6 white square button
 1F532                                                  ; fully-qualified     # 🔲 E0.6 black square button
 
-# Symbols subtotal:		302
-# Symbols subtotal:		302	w/o modifiers
+# Symbols subtotal:		304
+# Symbols subtotal:		304	w/o modifiers
 
 # group: Flags
 
@@ -4709,7 +4742,7 @@
 1F3F3 200D 1F308                                       ; unqualified         # 🏳‍🌈 E4.0 rainbow flag
 1F3F3 FE0F 200D 26A7 FE0F                              ; fully-qualified     # 🏳️‍⚧️ E13.0 transgender flag
 1F3F3 200D 26A7 FE0F                                   ; unqualified         # 🏳‍⚧️ E13.0 transgender flag
-1F3F3 FE0F 200D 26A7                                   ; unqualified         # 🏳️‍⚧ E13.0 transgender flag
+1F3F3 FE0F 200D 26A7                                   ; minimally-qualified # 🏳️‍⚧ E13.0 transgender flag
 1F3F3 200D 26A7                                        ; unqualified         # 🏳‍⚧ E13.0 transgender flag
 1F3F4 200D 2620 FE0F                                   ; fully-qualified     # 🏴‍☠️ E11.0 pirate flag
 1F3F4 200D 2620                                        ; minimally-qualified # 🏴‍☠ E11.0 pirate flag
@@ -4983,9 +5016,9 @@
 # Flags subtotal:		275	w/o modifiers
 
 # Status Counts
-# fully-qualified : 3624
-# minimally-qualified : 817
-# unqualified : 252
+# fully-qualified : 3655
+# minimally-qualified : 827
+# unqualified : 242
 # component : 9
 
 #EOF

lib/pleroma/emoji.ex

@@ -9,6 +9,7 @@ defmodule Pleroma.Emoji do
   """
   use GenServer
 
+  alias Pleroma.Emoji.Combinations
   alias Pleroma.Emoji.Loader
 
   require Logger
@@ -137,4 +138,17 @@ defmodule Pleroma.Emoji do
   end
 
   def is_unicode_emoji?(_), do: false
+
+  emoji_qualification_map =
+    emojis
+    |> Enum.filter(&String.contains?(&1, "\uFE0F"))
+    |> Combinations.variate_emoji_qualification()
+
+  for {qualified, unqualified_list} <- emoji_qualification_map do
+    for unqualified <- unqualified_list do
+      def fully_qualify_emoji(unquote(unqualified)), do: unquote(qualified)
+    end
+  end
+
+  def fully_qualify_emoji(emoji), do: emoji
 end

lib/pleroma/emoji/combinations.ex

@@ -0,0 +1,45 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Emoji.Combinations do
+  # FE0F is the emoji variation sequence. It is used for fully-qualifying
+  # emoji, and that includes emoji combinations.
+  # This code generates combinations per emoji: for each FE0F, all possible
+  # combinations of the character being removed or staying will be generated.
+  # This is made as an attempt to find all partially-qualified and unqualified
+  # versions of a fully-qualified emoji.
+  # I have found *no cases* for which this would be a problem, after browsing
+  # the entire emoji list in emoji-test.txt. This is safe, and, sadly, most
+  # likely sane too.
+
+  defp qualification_combinations(codepoints) do
+    qualification_combinations([[]], codepoints)
+  end
+
+  defp qualification_combinations(acc, []), do: acc
+
+  defp qualification_combinations(acc, ["\uFE0F" | tail]) do
+    acc
+    |> Enum.flat_map(fn x -> [x, x ++ ["\uFE0F"]] end)
+    |> qualification_combinations(tail)
+  end
+
+  defp qualification_combinations(acc, [codepoint | tail]) do
+    acc
+    |> Enum.map(&Kernel.++(&1, [codepoint]))
+    |> qualification_combinations(tail)
+  end
+
+  def variate_emoji_qualification(emoji) when is_binary(emoji) do
+    emoji
+    |> String.codepoints()
+    |> qualification_combinations()
+    |> Enum.map(&List.to_string/1)
+  end
+
+  def variate_emoji_qualification(emoji) when is_list(emoji) do
+    emoji
+    |> Enum.map(fn emoji -> {emoji, variate_emoji_qualification(emoji)} end)
+  end
+end

lib/pleroma/following_relationship.ex

@@ -194,12 +194,13 @@ defmodule Pleroma.FollowingRelationship do
     |> join(:inner, [r], f in assoc(r, :follower))
     |> where(following_id: ^origin.id)
     |> where([r, f], f.allow_following_move == true)
+    |> where([r, f], f.local == true)
     |> limit(50)
     |> preload([:follower])
     |> Repo.all()
     |> Enum.map(fn following_relationship ->
-      Repo.delete(following_relationship)
       Pleroma.Web.CommonAPI.follow(following_relationship.follower, target)
+      Pleroma.Web.CommonAPI.unfollow(following_relationship.follower, origin)
     end)
     |> case do
       [] ->

lib/pleroma/http.ex

@@ -106,5 +106,12 @@ defmodule Pleroma.HTTP do
     [Tesla.Middleware.FollowRedirects, Pleroma.Tesla.Middleware.ConnectionPool]
   end
 
-  defp adapter_middlewares(_), do: []
+  defp adapter_middlewares(_) do
+    if Pleroma.Config.get(:env) == :test do
+      # Emulate redirects in test env, which are handled by adapters in other environments
+      [Tesla.Middleware.FollowRedirects]
+    else
+      []
+    end
+  end
 end

lib/pleroma/http/adapter_helper/hackney.ex

@@ -24,10 +24,6 @@ defmodule Pleroma.HTTP.AdapterHelper.Hackney do
     |> Pleroma.HTTP.AdapterHelper.maybe_add_proxy(proxy)
   end
 
-  defp add_scheme_opts(opts, %URI{scheme: "https"}) do
-    Keyword.put(opts, :ssl_options, versions: [:"tlsv1.2", :"tlsv1.1", :tlsv1])
-  end
-
   defp add_scheme_opts(opts, _), do: opts
 
   defp maybe_add_with_body(opts) do

lib/pleroma/migrators/context_objects_deletion_migrator.ex

@@ -0,0 +1,139 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Migrators.ContextObjectsDeletionMigrator do
+  defmodule State do
+    use Pleroma.Migrators.Support.BaseMigratorState
+
+    @impl Pleroma.Migrators.Support.BaseMigratorState
+    defdelegate data_migration(), to: Pleroma.DataMigration, as: :delete_context_objects
+  end
+
+  use Pleroma.Migrators.Support.BaseMigrator
+
+  alias Pleroma.Migrators.Support.BaseMigrator
+  alias Pleroma.Object
+
+  @doc "This migration removes objects created exclusively for contexts, containing only an `id` field."
+
+  @impl BaseMigrator
+  def feature_config_path, do: [:features, :delete_context_objects]
+
+  @impl BaseMigrator
+  def fault_rate_allowance, do: Config.get([:delete_context_objects, :fault_rate_allowance], 0)
+
+  @impl BaseMigrator
+  def perform do
+    data_migration_id = data_migration_id()
+    max_processed_id = get_stat(:max_processed_id, 0)
+
+    Logger.info("Deleting context objects from `objects` (from oid: #{max_processed_id})...")
+
+    query()
+    |> where([object], object.id > ^max_processed_id)
+    |> Repo.chunk_stream(100, :batches, timeout: :infinity)
+    |> Stream.each(fn objects ->
+      object_ids = Enum.map(objects, & &1.id)
+
+      results = Enum.map(object_ids, &delete_context_object(&1))
+
+      failed_ids =
+        results
+        |> Enum.filter(&(elem(&1, 0) == :error))
+        |> Enum.map(&elem(&1, 1))
+
+      chunk_affected_count =
+        results
+        |> Enum.filter(&(elem(&1, 0) == :ok))
+        |> length()
+
+      for failed_id <- failed_ids do
+        _ =
+          Repo.query(
+            "INSERT INTO data_migration_failed_ids(data_migration_id, record_id) " <>
+              "VALUES ($1, $2) ON CONFLICT DO NOTHING;",
+            [data_migration_id, failed_id]
+          )
+      end
+
+      _ =
+        Repo.query(
+          "DELETE FROM data_migration_failed_ids " <>
+            "WHERE data_migration_id = $1 AND record_id = ANY($2)",
+          [data_migration_id, object_ids -- failed_ids]
+        )
+
+      max_object_id = Enum.at(object_ids, -1)
+
+      put_stat(:max_processed_id, max_object_id)
+      increment_stat(:iteration_processed_count, length(object_ids))
+      increment_stat(:processed_count, length(object_ids))
+      increment_stat(:failed_count, length(failed_ids))
+      increment_stat(:affected_count, chunk_affected_count)
+      put_stat(:records_per_second, records_per_second())
+      persist_state()
+
+      # A quick and dirty approach to controlling the load this background migration imposes
+      sleep_interval = Config.get([:delete_context_objects, :sleep_interval_ms], 0)
+      Process.sleep(sleep_interval)
+    end)
+    |> Stream.run()
+  end
+
+  @impl BaseMigrator
+  def query do
+    # Context objects have no activity type, and only one field, `id`.
+    # Only those context objects are without types.
+    from(
+      object in Object,
+      where: fragment("(?)->'type' IS NULL", object.data),
+      select: %{
+        id: object.id
+      }
+    )
+  end
+
+  @spec delete_context_object(integer()) :: {:ok | :error, integer()}
+  defp delete_context_object(id) do
+    result =
+      %Object{id: id}
+      |> Repo.delete()
+      |> elem(0)
+
+    {result, id}
+  end
+
+  @impl BaseMigrator
+  def retry_failed do
+    data_migration_id = data_migration_id()
+
+    failed_objects_query()
+    |> Repo.chunk_stream(100, :one)
+    |> Stream.each(fn object ->
+      with {res, _} when res != :error <- delete_context_object(object.id) do
+        _ =
+          Repo.query(
+            "DELETE FROM data_migration_failed_ids " <>
+              "WHERE data_migration_id = $1 AND record_id = $2",
+            [data_migration_id, object.id]
+          )
+      end
+    end)
+    |> Stream.run()
+
+    put_stat(:failed_count, failures_count())
+    persist_state()
+
+    force_continue()
+  end
+
+  defp failed_objects_query do
+    from(o in Object)
+    |> join(:inner, [o], dmf in fragment("SELECT * FROM data_migration_failed_ids"),
+      on: dmf.record_id == o.id
+    )
+    |> where([_o, dmf], dmf.data_migration_id == ^data_migration_id())
+    |> order_by([o], asc: o.id)
+  end
+end

lib/pleroma/migrators/hashtags_table_migrator.ex

@@ -183,7 +183,7 @@ defmodule Pleroma.Migrators.HashtagsTableMigrator do
     DELETE FROM hashtags_objects WHERE object_id IN
       (SELECT DISTINCT objects.id FROM objects
         JOIN hashtags_objects ON hashtags_objects.object_id = objects.id LEFT JOIN activities
-          ON COALESCE(activities.data->'object'->>'id', activities.data->>'object') =
+          ON associated_object_id(activities) =
             (objects.data->>'id')
           AND activities.data->>'type' = 'Create'
         WHERE activities.id IS NULL);

lib/pleroma/notification.ex

@@ -118,9 +118,8 @@ defmodule Pleroma.Notification do
     |> join(:left, [n, a], object in Object,
       on:
         fragment(
-          "(?->>'id') = COALESCE(?->'object'->>'id', ?->>'object')",
+          "(?->>'id') = associated_object_id(?)",
           object.data,
-          a.data,
           a.data
         )
     )
@@ -180,6 +179,7 @@ defmodule Pleroma.Notification do
         from([_n, a, o] in query,
           where:
             fragment("not(?->>'content' ~* ?)", o.data, ^regex) or
+              fragment("?->>'content' is null", o.data) or
               fragment("?->>'actor' = ?", o.data, ^user.ap_id)
         )
     end
@@ -194,13 +194,11 @@ defmodule Pleroma.Notification do
       |> join(:left, [n, a], mutated_activity in Pleroma.Activity,
         on:
           fragment(
-            "COALESCE((?->'object')->>'id', ?->>'object')",
-            a.data,
+            "associated_object_id(?)",
             a.data
           ) ==
             fragment(
-              "COALESCE((?->'object')->>'id', ?->>'object')",
-              mutated_activity.data,
+              "associated_object_id(?)",
               mutated_activity.data
             ) and
             fragment("(?->>'type' = 'Like' or ?->>'type' = 'Announce')", a.data, a.data) and
@@ -342,14 +340,6 @@ defmodule Pleroma.Notification do
     |> Repo.delete_all()
   end
 
-  def destroy_multiple_from_types(%{id: user_id}, types) do
-    from(n in Notification,
-      where: n.user_id == ^user_id,
-      where: n.type in ^types
-    )
-    |> Repo.delete_all()
-  end
-
   def dismiss(%Pleroma.Activity{} = activity) do
     Notification
     |> where([n], n.activity_id == ^activity.id)
@@ -386,7 +376,7 @@ defmodule Pleroma.Notification do
   end
 
   def create_notifications(%Activity{data: %{"type" => type}} = activity, options)
-      when type in ["Follow", "Like", "Announce", "Move", "EmojiReact", "Flag"] do
+      when type in ["Follow", "Like", "Announce", "Move", "EmojiReact", "Flag", "Update"] do
     do_create_notifications(activity, options)
   end
 
@@ -447,6 +437,9 @@ defmodule Pleroma.Notification do
         activity
         |> type_from_activity_object()
 
+      "Update" ->
+        "update"
+
       t ->
         raise "No notification type for activity type #{t}"
     end
@@ -521,7 +514,16 @@ defmodule Pleroma.Notification do
   def get_notified_from_activity(activity, local_only \\ true)
 
   def get_notified_from_activity(%Activity{data: %{"type" => type}} = activity, local_only)
-      when type in ["Create", "Like", "Announce", "Follow", "Move", "EmojiReact", "Flag"] do
+      when type in [
+             "Create",
+             "Like",
+             "Announce",
+             "Follow",
+             "Move",
+             "EmojiReact",
+             "Flag",
+             "Update"
+           ] do
     potential_receiver_ap_ids = get_potential_receiver_ap_ids(activity)
 
     potential_receivers =
@@ -579,7 +581,24 @@ defmodule Pleroma.Notification do
   end
 
   def get_potential_receiver_ap_ids(%{data: %{"type" => "Flag", "actor" => actor}}) do
-    (User.all_superusers() |> Enum.map(fn user -> user.ap_id end)) -- [actor]
+    (User.all_users_with_privilege(:reports_manage_reports)
+     |> Enum.map(fn user -> user.ap_id end)) --
+      [actor]
+  end
+
+  # Update activity: notify all who repeated this
+  def get_potential_receiver_ap_ids(%{data: %{"type" => "Update", "actor" => actor}} = activity) do
+    with %Object{data: %{"id" => object_id}} <- Object.normalize(activity, fetch: false) do
+      repeaters =
+        Activity.Queries.by_type("Announce")
+        |> Activity.Queries.by_object_id(object_id)
+        |> Activity.with_joined_user_actor()
+        |> where([a, u], u.local)
+        |> select([a, u], u.ap_id)
+        |> Repo.all()
+
+      repeaters -- [actor]
+    end
   end
 
   def get_potential_receiver_ap_ids(activity) do
@@ -689,7 +708,7 @@ defmodule Pleroma.Notification do
     cond do
       opts[:type] == "poll" -> false
       user.ap_id == actor -> false
-      !User.following?(follower, user) -> true
+      !User.following?(user, follower) -> true
       true -> false
     end
   end

lib/pleroma/object.ex

@@ -40,8 +40,7 @@ defmodule Pleroma.Object do
     join(query, join_type, [{object, object_position}], a in Activity,
       on:
         fragment(
-          "COALESCE(?->'object'->>'id', ?->>'object') = (? ->> 'id') AND (?->>'type' = ?) ",
-          a.data,
+          "associated_object_id(?) = (? ->> 'id') AND (?->>'type' = ?) ",
           a.data,
           object.data,
           a.data,
@@ -145,7 +144,7 @@ defmodule Pleroma.Object do
     Logger.debug("Backtrace: #{inspect(Process.info(:erlang.self(), :current_stacktrace))}")
   end
 
-  def normalize(_, options \\ [fetch: false])
+  def normalize(_, options \\ [fetch: false, id_only: false])
 
   # If we pass an Activity to Object.normalize(), we can try to use the preloaded object.
   # Use this whenever possible, especially when walking graphs in an O(N) loop!
@@ -173,10 +172,15 @@ defmodule Pleroma.Object do
   def normalize(%{"id" => ap_id}, options), do: normalize(ap_id, options)
 
   def normalize(ap_id, options) when is_binary(ap_id) do
-    if Keyword.get(options, :fetch) do
-      Fetcher.fetch_object_from_id!(ap_id, options)
-    else
-      get_cached_by_ap_id(ap_id)
+    cond do
+      Keyword.get(options, :id_only) ->
+        ap_id
+
+      Keyword.get(options, :fetch) ->
+        Fetcher.fetch_object_from_id!(ap_id, options)
+
+      true ->
+        get_cached_by_ap_id(ap_id)
     end
   end
 
@@ -208,10 +212,6 @@ defmodule Pleroma.Object do
     end
   end
 
-  def context_mapping(context) do
-    Object.change(%Object{}, %{data: %{"id" => context}})
-  end
-
   def make_tombstone(%Object{data: %{"id" => id, "type" => type}}, deleted \\ DateTime.utc_now()) do
     %ObjectTombstone{
       id: id,

lib/pleroma/object/fetcher.ex

@@ -4,6 +4,7 @@
 
 defmodule Pleroma.Object.Fetcher do
   alias Pleroma.HTTP
+  alias Pleroma.Instances
   alias Pleroma.Maps
   alias Pleroma.Object
   alias Pleroma.Object.Containment
@@ -26,8 +27,42 @@ defmodule Pleroma.Object.Fetcher do
   end
 
   defp maybe_reinject_internal_fields(%{data: %{} = old_data}, new_data) do
+    has_history? = fn
+      %{"formerRepresentations" => %{"orderedItems" => list}} when is_list(list) -> true
+      _ -> false
+    end
+
     internal_fields = Map.take(old_data, Pleroma.Constants.object_internal_fields())
 
+    remote_history_exists? = has_history?.(new_data)
+
+    # If the remote history exists, we treat that as the only source of truth.
+    new_data =
+      if has_history?.(old_data) and not remote_history_exists? do
+        Map.put(new_data, "formerRepresentations", old_data["formerRepresentations"])
+      else
+        new_data
+      end
+
+    # If the remote does not have history information, we need to manage it ourselves
+    new_data =
+      if not remote_history_exists? do
+        changed? =
+          Pleroma.Constants.status_updatable_fields()
+          |> Enum.any?(fn field -> Map.get(old_data, field) != Map.get(new_data, field) end)
+
+        %{updated_object: updated_object} =
+          new_data
+          |> Object.Updater.maybe_update_history(old_data,
+            updated: changed?,
+            use_history_in_new_object?: false
+          )
+
+        updated_object
+      else
+        new_data
+      end
+
     Map.merge(new_data, internal_fields)
   end
 
@@ -200,6 +235,10 @@ defmodule Pleroma.Object.Fetcher do
          {:ok, body} <- get_object(id),
          {:ok, data} <- safe_json_decode(body),
          :ok <- Containment.contain_origin_from_id(id, data) do
+      if not Instances.reachable?(id) do
+        Instances.set_reachable(id)
+      end
+
       {:ok, data}
     else
       {:scheme, _} ->

lib/pleroma/object/updater.ex

@@ -0,0 +1,240 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Object.Updater do
+  require Pleroma.Constants
+
+  def update_content_fields(orig_object_data, updated_object) do
+    Pleroma.Constants.status_updatable_fields()
+    |> Enum.reduce(
+      %{data: orig_object_data, updated: false},
+      fn field, %{data: data, updated: updated} ->
+        updated =
+          updated or
+            (field != "updated" and
+               Map.get(updated_object, field) != Map.get(orig_object_data, field))
+
+        data =
+          if Map.has_key?(updated_object, field) do
+            Map.put(data, field, updated_object[field])
+          else
+            Map.drop(data, [field])
+          end
+
+        %{data: data, updated: updated}
+      end
+    )
+  end
+
+  def maybe_history(object) do
+    with history <- Map.get(object, "formerRepresentations"),
+         true <- is_map(history),
+         "OrderedCollection" <- Map.get(history, "type"),
+         true <- is_list(Map.get(history, "orderedItems")),
+         true <- is_integer(Map.get(history, "totalItems")) do
+      history
+    else
+      _ -> nil
+    end
+  end
+
+  def history_for(object) do
+    with history when not is_nil(history) <- maybe_history(object) do
+      history
+    else
+      _ -> history_skeleton()
+    end
+  end
+
+  defp history_skeleton do
+    %{
+      "type" => "OrderedCollection",
+      "totalItems" => 0,
+      "orderedItems" => []
+    }
+  end
+
+  def maybe_update_history(
+        updated_object,
+        orig_object_data,
+        opts
+      ) do
+    updated = opts[:updated]
+    use_history_in_new_object? = opts[:use_history_in_new_object?]
+
+    if not updated do
+      %{updated_object: updated_object, used_history_in_new_object?: false}
+    else
+      # Put edit history
+      # Note that we may have got the edit history by first fetching the object
+      {new_history, used_history_in_new_object?} =
+        with true <- use_history_in_new_object?,
+             updated_history when not is_nil(updated_history) <- maybe_history(opts[:new_data]) do
+          {updated_history, true}
+        else
+          _ ->
+            history = history_for(orig_object_data)
+
+            latest_history_item =
+              orig_object_data
+              |> Map.drop(["id", "formerRepresentations"])
+
+            updated_history =
+              history
+              |> Map.put("orderedItems", [latest_history_item | history["orderedItems"]])
+              |> Map.put("totalItems", history["totalItems"] + 1)
+
+            {updated_history, false}
+        end
+
+      updated_object =
+        updated_object
+        |> Map.put("formerRepresentations", new_history)
+
+      %{updated_object: updated_object, used_history_in_new_object?: used_history_in_new_object?}
+    end
+  end
+
+  defp maybe_update_poll(to_be_updated, updated_object) do
+    choice_key = fn data ->
+      if Map.has_key?(data, "anyOf"), do: "anyOf", else: "oneOf"
+    end
+
+    with true <- to_be_updated["type"] == "Question",
+         key <- choice_key.(updated_object),
+         true <- key == choice_key.(to_be_updated),
+         orig_choices <- to_be_updated[key] |> Enum.map(&Map.drop(&1, ["replies"])),
+         new_choices <- updated_object[key] |> Enum.map(&Map.drop(&1, ["replies"])),
+         true <- orig_choices == new_choices do
+      # Choices are the same, but counts are different
+      to_be_updated
+      |> Map.put(key, updated_object[key])
+    else
+      # Choices (or vote type) have changed, do not allow this
+      _ -> to_be_updated
+    end
+  end
+
+  # This calculates the data to be sent as the object of an Update.
+  # new_data's formerRepresentations is not considered.
+  # formerRepresentations is added to the returned data.
+  def make_update_object_data(original_data, new_data, date) do
+    %{data: updated_data, updated: updated} =
+      original_data
+      |> update_content_fields(new_data)
+
+    if not updated do
+      updated_data
+    else
+      %{updated_object: updated_data} =
+        updated_data
+        |> maybe_update_history(original_data, updated: updated, use_history_in_new_object?: false)
+
+      updated_data
+      |> Map.put("updated", date)
+    end
+  end
+
+  # This calculates the data of the new Object from an Update.
+  # new_data's formerRepresentations is considered.
+  def make_new_object_data_from_update_object(original_data, new_data) do
+    update_is_reasonable =
+      with {_, updated} when not is_nil(updated) <- {:cur_updated, new_data["updated"]},
+           {_, {:ok, updated_time, _}} <- {:cur_updated, DateTime.from_iso8601(updated)},
+           {_, last_updated} when not is_nil(last_updated) <-
+             {:last_updated, original_data["updated"] || original_data["published"]},
+           {_, {:ok, last_updated_time, _}} <-
+             {:last_updated, DateTime.from_iso8601(last_updated)},
+           :gt <- DateTime.compare(updated_time, last_updated_time) do
+        :update_everything
+      else
+        # only allow poll updates
+        {:cur_updated, _} -> :no_content_update
+        :eq -> :no_content_update
+        # allow all updates
+        {:last_updated, _} -> :update_everything
+        # allow no updates
+        _ -> false
+      end
+
+    %{
+      updated_object: updated_data,
+      used_history_in_new_object?: used_history_in_new_object?,
+      updated: updated
+    } =
+      if update_is_reasonable == :update_everything do
+        %{data: updated_data, updated: updated} =
+          original_data
+          |> update_content_fields(new_data)
+
+        updated_data
+        |> maybe_update_history(original_data,
+          updated: updated,
+          use_history_in_new_object?: true,
+          new_data: new_data
+        )
+        |> Map.put(:updated, updated)
+      else
+        %{
+          updated_object: original_data,
+          used_history_in_new_object?: false,
+          updated: false
+        }
+      end
+
+    updated_data =
+      if update_is_reasonable != false do
+        updated_data
+        |> maybe_update_poll(new_data)
+      else
+        updated_data
+      end
+
+    %{
+      updated_data: updated_data,
+      updated: updated,
+      used_history_in_new_object?: used_history_in_new_object?
+    }
+  end
+
+  def for_each_history_item(%{"orderedItems" => items} = history, _object, fun) do
+    new_items =
+      Enum.map(items, fun)
+      |> Enum.reduce_while(
+        {:ok, []},
+        fn
+          {:ok, item}, {:ok, acc} -> {:cont, {:ok, acc ++ [item]}}
+          e, _acc -> {:halt, e}
+        end
+      )
+
+    case new_items do
+      {:ok, items} -> {:ok, Map.put(history, "orderedItems", items)}
+      e -> e
+    end
+  end
+
+  def for_each_history_item(history, _, _) do
+    {:ok, history}
+  end
+
+  def do_with_history(object, fun) do
+    with history <- object["formerRepresentations"],
+         object <- Map.drop(object, ["formerRepresentations"]),
+         {_, {:ok, object}} <- {:main_body, fun.(object)},
+         {_, {:ok, history}} <- {:history_items, for_each_history_item(history, object, fun)} do
+      object =
+        if history do
+          Map.put(object, "formerRepresentations", history)
+        else
+          object
+        end
+
+      {:ok, object}
+    else
+      {:main_body, e} -> e
+      {:history_items, e} -> e
+    end
+  end
+end

lib/pleroma/reverse_proxy/client/hackney.ex

@@ -7,7 +7,6 @@ defmodule Pleroma.ReverseProxy.Client.Hackney do
 
   @impl true
   def request(method, url, headers, body, opts \\ []) do
-    opts = Keyword.put(opts, :ssl_options, versions: [:"tlsv1.2", :"tlsv1.1", :tlsv1])
     :hackney.request(method, url, headers, body, opts)
   end
 

lib/pleroma/signature.ex

@@ -10,17 +10,14 @@ defmodule Pleroma.Signature do
   alias Pleroma.User
   alias Pleroma.Web.ActivityPub.ActivityPub
 
+  @known_suffixes ["/publickey", "/main-key"]
+
   def key_id_to_actor_id(key_id) do
     uri =
-      URI.parse(key_id)
+      key_id
+      |> URI.parse()
       |> Map.put(:fragment, nil)
-
-    uri =
-      if not is_nil(uri.path) and String.ends_with?(uri.path, "/publickey") do
-        Map.put(uri, :path, String.replace(uri.path, "/publickey", ""))
-      else
-        uri
-      end
+      |> remove_suffix(@known_suffixes)
 
     maybe_ap_id = URI.to_string(uri)
 
@@ -36,6 +33,16 @@ defmodule Pleroma.Signature do
     end
   end
 
+  defp remove_suffix(uri, [test | rest]) do
+    if not is_nil(uri.path) and String.ends_with?(uri.path, test) do
+      Map.put(uri, :path, String.replace(uri.path, test, ""))
+    else
+      remove_suffix(uri, rest)
+    end
+  end
+
+  defp remove_suffix(uri, []), do: uri
+
   def fetch_public_key(conn) do
     with %{"keyId" => kid} <- HTTPSignatures.signature_for_conn(conn),
          {:ok, actor_id} <- key_id_to_actor_id(kid),
@@ -59,9 +66,8 @@ defmodule Pleroma.Signature do
     end
   end
 
-  def sign(%User{} = user, headers) do
-    with {:ok, %{keys: keys}} <- User.ensure_keys_present(user),
-         {:ok, private_key, _} <- Keys.keys_from_pem(keys) do
+  def sign(%User{keys: keys} = user, headers) do
+    with {:ok, private_key, _} <- Keys.keys_from_pem(keys) do
       HTTPSignatures.sign(private_key, user.ap_id <> "#main-key", headers)
     end
   end

lib/pleroma/upload.ex

@@ -36,6 +36,7 @@ defmodule Pleroma.Upload do
   alias Ecto.UUID
   alias Pleroma.Config
   alias Pleroma.Maps
+  alias Pleroma.Web.ActivityPub.Utils
   require Logger
 
   @type source ::
@@ -60,12 +61,23 @@ defmodule Pleroma.Upload do
           width: integer(),
           height: integer(),
           blurhash: String.t(),
+          description: String.t(),
           path: String.t()
         }
-  defstruct [:id, :name, :tempfile, :content_type, :width, :height, :blurhash, :path]
+  defstruct [
+    :id,
+    :name,
+    :tempfile,
+    :content_type,
+    :width,
+    :height,
+    :blurhash,
+    :description,
+    :path
+  ]
 
-  defp get_description(opts, upload) do
-    case {opts[:description], Pleroma.Config.get([Pleroma.Upload, :default_description])} do
+  defp get_description(upload) do
+    case {upload.description, Pleroma.Config.get([Pleroma.Upload, :default_description])} do
       {description, _} when is_binary(description) -> description
       {_, :filename} -> upload.name
       {_, str} when is_binary(str) -> str
@@ -81,13 +93,14 @@ defmodule Pleroma.Upload do
     with {:ok, upload} <- prepare_upload(upload, opts),
          upload = %__MODULE__{upload | path: upload.path || "#{upload.id}/#{upload.name}"},
          {:ok, upload} <- Pleroma.Upload.Filter.filter(opts.filters, upload),
-         description = get_description(opts, upload),
+         description = get_description(upload),
          {_, true} <-
            {:description_limit,
             String.length(description) <= Pleroma.Config.get([:instance, :description_limit])},
          {:ok, url_spec} <- Pleroma.Uploaders.Uploader.put_file(opts.uploader, upload) do
       {:ok,
        %{
+         "id" => Utils.generate_object_id(),
          "type" => opts.activity_type,
          "mediaType" => upload.content_type,
          "url" => [
@@ -152,7 +165,8 @@ defmodule Pleroma.Upload do
          id: UUID.generate(),
          name: file.filename,
          tempfile: file.path,
-         content_type: file.content_type
+         content_type: file.content_type,
+         description: opts.description
        }}
     end
   end
@@ -172,7 +186,8 @@ defmodule Pleroma.Upload do
          id: UUID.generate(),
          name: hash <> "." <> ext,
          tempfile: tmp_path,
-         content_type: content_type
+         content_type: content_type,
+         description: opts.description
        }}
     end
   end

lib/pleroma/upload/filter/exiftool/read_description.ex

@@ -0,0 +1,52 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Upload.Filter.Exiftool.ReadDescription do
+  @moduledoc """
+  Gets a valid description from the related EXIF tags and provides them in the response if no description is provided yet.
+  It will first check ImageDescription, when that doesn't probide a valid description, it will check iptc:Caption-Abstract.
+  A valid description means the fields are filled in and not too long (see `:instance, :description_limit`).
+  """
+  @behaviour Pleroma.Upload.Filter
+
+  @spec filter(Pleroma.Upload.t()) :: {:ok, any()} | {:error, String.t()}
+
+  def filter(%Pleroma.Upload{description: description})
+      when is_binary(description),
+      do: {:ok, :noop}
+
+  def filter(%Pleroma.Upload{tempfile: file} = upload),
+    do: {:ok, :filtered, upload |> Map.put(:description, read_description_from_exif_data(file))}
+
+  def filter(_, _), do: {:ok, :noop}
+
+  defp read_description_from_exif_data(file) do
+    nil
+    |> read_when_empty(file, "-ImageDescription")
+    |> read_when_empty(file, "-iptc:Caption-Abstract")
+  end
+
+  defp read_when_empty(current_description, _, _) when is_binary(current_description),
+    do: current_description
+
+  defp read_when_empty(_, file, tag) do
+    try do
+      {tag_content, 0} =
+        System.cmd("exiftool", ["-b", "-s3", tag, file],
+          stderr_to_stdout: false,
+          parallelism: true
+        )
+
+      tag_content = String.trim(tag_content)
+
+      if tag_content != "" and
+           String.length(tag_content) <=
+             Pleroma.Config.get([:instance, :description_limit]),
+         do: tag_content,
+         else: nil
+    rescue
+      _ in ErlangError -> nil
+    end
+  end
+end

lib/pleroma/upload/filter/exiftool/strip_location.ex

@@ -2,7 +2,7 @@
 # Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
-defmodule Pleroma.Upload.Filter.Exiftool do
+defmodule Pleroma.Upload.Filter.Exiftool.StripLocation do
   @moduledoc """
   Strips GPS related EXIF tags and overwrites the file in place.
   Also strips or replaces filesystem metadata e.g., timestamps.
@@ -14,6 +14,7 @@ defmodule Pleroma.Upload.Filter.Exiftool do
   # Formats not compatible with exiftool at this time
   def filter(%Pleroma.Upload{content_type: "image/heic"}), do: {:ok, :noop}
   def filter(%Pleroma.Upload{content_type: "image/webp"}), do: {:ok, :noop}
+  def filter(%Pleroma.Upload{content_type: "image/svg" <> _}), do: {:ok, :noop}
 
   def filter(%Pleroma.Upload{tempfile: file, content_type: "image" <> _}) do
     try do

lib/pleroma/user.ex

@@ -326,7 +326,7 @@ defmodule Pleroma.User do
   end
 
   def visible_for(%User{} = user, for_user) do
-    if superuser?(for_user) do
+    if privileged?(for_user, :users_manage_activation_state) do
       :visible
     else
       visible_account_status(user)
@@ -353,10 +353,45 @@ defmodule Pleroma.User do
     end
   end
 
-  @spec superuser?(User.t()) :: boolean()
-  def superuser?(%User{local: true, is_admin: true}), do: true
-  def superuser?(%User{local: true, is_moderator: true}), do: true
-  def superuser?(_), do: false
+  @spec privileged?(User.t(), atom()) :: boolean()
+  def privileged?(%User{is_admin: false, is_moderator: false}, _), do: false
+
+  def privileged?(
+        %User{local: true, is_admin: is_admin, is_moderator: is_moderator},
+        privilege_tag
+      ),
+      do:
+        privileged_for?(privilege_tag, is_admin, :admin_privileges) or
+          privileged_for?(privilege_tag, is_moderator, :moderator_privileges)
+
+  def privileged?(_, _), do: false
+
+  defp privileged_for?(privilege_tag, true, config_role_key),
+    do: privilege_tag in Config.get([:instance, config_role_key])
+
+  defp privileged_for?(_, _, _), do: false
+
+  @spec privileges(User.t()) :: [atom()]
+  def privileges(%User{local: false}) do
+    []
+  end
+
+  def privileges(%User{is_moderator: false, is_admin: false}) do
+    []
+  end
+
+  def privileges(%User{local: true, is_moderator: true, is_admin: true}) do
+    (Config.get([:instance, :moderator_privileges]) ++ Config.get([:instance, :admin_privileges]))
+    |> Enum.uniq()
+  end
+
+  def privileges(%User{local: true, is_moderator: true, is_admin: false}) do
+    Config.get([:instance, :moderator_privileges])
+  end
+
+  def privileges(%User{local: true, is_moderator: false, is_admin: true}) do
+    Config.get([:instance, :admin_privileges])
+  end
 
   @spec invisible?(User.t()) :: boolean()
   def invisible?(%User{invisible: true}), do: true
@@ -611,7 +646,13 @@ defmodule Pleroma.User do
          {:ok, new_value} <- value_function.(value) do
       put_change(changeset, map_field, new_value)
     else
-      _ -> changeset
+      {:error, :file_too_large} ->
+        Ecto.Changeset.validate_change(changeset, map_field, fn map_field, _value ->
+          [{map_field, "file is too large"}]
+        end)
+
+      _ ->
+        changeset
     end
   end
 
@@ -706,11 +747,12 @@ defmodule Pleroma.User do
     ])
     |> validate_required([:name, :nickname])
     |> unique_constraint(:nickname)
-    |> validate_exclusion(:nickname, Config.get([User, :restricted_nicknames]))
+    |> validate_not_restricted_nickname(:nickname)
     |> validate_format(:nickname, local_nickname_regex())
     |> put_ap_id()
     |> unique_constraint(:ap_id)
     |> put_following_and_follower_and_featured_address()
+    |> put_private_key()
   end
 
   def register_changeset(struct, params \\ %{}, opts \\ []) do
@@ -754,17 +796,9 @@ defmodule Pleroma.User do
     |> validate_confirmation(:password)
     |> unique_constraint(:email)
     |> validate_format(:email, @email_regex)
-    |> validate_change(:email, fn :email, email ->
-      valid? =
-        Config.get([User, :email_blacklist])
-        |> Enum.all?(fn blacklisted_domain ->
-          !String.ends_with?(email, ["@" <> blacklisted_domain, "." <> blacklisted_domain])
-        end)
-
-      if valid?, do: [], else: [email: "Invalid email"]
-    end)
+    |> validate_email_not_in_blacklisted_domain(:email)
     |> unique_constraint(:nickname)
-    |> validate_exclusion(:nickname, Config.get([User, :restricted_nicknames]))
+    |> validate_not_restricted_nickname(:nickname)
     |> validate_format(:nickname, local_nickname_regex())
     |> validate_length(:bio, max: bio_limit)
     |> validate_length(:name, min: 1, max: name_limit)
@@ -776,6 +810,36 @@ defmodule Pleroma.User do
     |> put_ap_id()
     |> unique_constraint(:ap_id)
     |> put_following_and_follower_and_featured_address()
+    |> put_private_key()
+  end
+
+  def validate_not_restricted_nickname(changeset, field) do
+    validate_change(changeset, field, fn _, value ->
+      valid? =
+        Config.get([User, :restricted_nicknames])
+        |> Enum.all?(fn restricted_nickname ->
+          String.downcase(value) != String.downcase(restricted_nickname)
+        end)
+
+      if valid?, do: [], else: [nickname: "Invalid nickname"]
+    end)
+  end
+
+  def validate_email_not_in_blacklisted_domain(changeset, field) do
+    validate_change(changeset, field, fn _, value ->
+      valid? =
+        Config.get([User, :email_blacklist])
+        |> Enum.all?(fn blacklisted_domain ->
+          blacklisted_domain_downcase = String.downcase(blacklisted_domain)
+
+          !String.ends_with?(String.downcase(value), [
+            "@" <> blacklisted_domain_downcase,
+            "." <> blacklisted_domain_downcase
+          ])
+        end)
+
+      if valid?, do: [], else: [email: "Invalid email"]
+    end)
   end
 
   def maybe_validate_required_email(changeset, true), do: changeset
@@ -825,6 +889,11 @@ defmodule Pleroma.User do
     |> put_change(:featured_address, featured)
   end
 
+  defp put_private_key(changeset) do
+    {:ok, pem} = Keys.generate_rsa_pem()
+    put_change(changeset, :keys, pem)
+  end
+
   defp autofollow_users(user) do
     candidates = Config.get([:instance, :autofollowed_nicknames])
 
@@ -877,7 +946,7 @@ defmodule Pleroma.User do
     end
   end
 
-  defp send_user_approval_email(user) do
+  defp send_user_approval_email(%User{email: email} = user) when is_binary(email) do
     user
     |> Pleroma.Emails.UserEmail.approval_pending_email()
     |> Pleroma.Emails.Mailer.deliver_async()
@@ -885,6 +954,10 @@ defmodule Pleroma.User do
     {:ok, :enqueued}
   end
 
+  defp send_user_approval_email(_user) do
+    {:ok, :skipped}
+  end
+
   defp send_admin_approval_emails(user) do
     all_superusers()
     |> Enum.filter(fn user -> not is_nil(user.email) end)
@@ -1129,24 +1202,10 @@ defmodule Pleroma.User do
     |> update_and_set_cache()
   end
 
-  def update_and_set_cache(%{data: %Pleroma.User{} = user} = changeset) do
-    was_superuser_before_update = User.superuser?(user)
-
+  def update_and_set_cache(changeset) do
     with {:ok, user} <- Repo.update(changeset, stale_error_field: :id) do
       set_cache(user)
     end
-    |> maybe_remove_report_notifications(was_superuser_before_update)
-  end
-
-  defp maybe_remove_report_notifications({:ok, %Pleroma.User{} = user} = result, true) do
-    if not User.superuser?(user),
-      do: user |> Notification.destroy_multiple_from_types(["pleroma:report"])
-
-    result
-  end
-
-  defp maybe_remove_report_notifications(result, _) do
-    result
   end
 
   def get_user_friends_ap_ids(user) do
@@ -1459,17 +1518,30 @@ defmodule Pleroma.User do
           {:ok, list(UserRelationship.t())} | {:error, String.t()}
   def mute(%User{} = muter, %User{} = mutee, params \\ %{}) do
     notifications? = Map.get(params, :notifications, true)
-    expires_in = Map.get(params, :expires_in, 0)
+    duration = Map.get(params, :duration, 0)
 
-    with {:ok, user_mute} <- UserRelationship.create_mute(muter, mutee),
+    expires_at =
+      if duration > 0 do
+        DateTime.utc_now()
+        |> DateTime.add(duration)
+      else
+        nil
+      end
+
+    with {:ok, user_mute} <- UserRelationship.create_mute(muter, mutee, expires_at),
          {:ok, user_notification_mute} <-
-           (notifications? && UserRelationship.create_notification_mute(muter, mutee)) ||
+           (notifications? &&
+              UserRelationship.create_notification_mute(
+                muter,
+                mutee,
+                expires_at
+              )) ||
              {:ok, nil} do
-      if expires_in > 0 do
+      if duration > 0 do
         Pleroma.Workers.MuteExpireWorker.enqueue(
           "unmute_user",
           %{"muter_id" => muter.id, "mutee_id" => mutee.id},
-          schedule_in: expires_in
+          scheduled_at: expires_at
         )
       end
 
@@ -1540,13 +1612,19 @@ defmodule Pleroma.User do
         blocker
       end
 
-    # clear any requested follows as well
+    # clear any requested follows from both sides as well
     blocked =
       case CommonAPI.reject_follow_request(blocked, blocker) do
         {:ok, %User{} = updated_blocked} -> updated_blocked
         nil -> blocked
       end
 
+    blocker =
+      case CommonAPI.reject_follow_request(blocker, blocked) do
+        {:ok, %User{} = updated_blocker} -> updated_blocker
+        nil -> blocker
+      end
+
     unsubscribe(blocked, blocker)
 
     unfollowing_blocked = Config.get([:activitypub, :unfollow_blocked], true)
@@ -2046,6 +2124,7 @@ defmodule Pleroma.User do
       follower_address: uri <> "/followers"
     }
     |> change
+    |> put_private_key()
     |> unique_constraint(:nickname)
     |> Repo.insert()
     |> set_cache()
@@ -2078,7 +2157,8 @@ defmodule Pleroma.User do
 
   @doc "Gets or fetch a user by uri or nickname."
   @spec get_or_fetch(String.t()) :: {:ok, User.t()} | {:error, String.t()}
-  def get_or_fetch("http" <> _host = uri), do: get_or_fetch_by_ap_id(uri)
+  def get_or_fetch("http://" <> _host = uri), do: get_or_fetch_by_ap_id(uri)
+  def get_or_fetch("https://" <> _host = uri), do: get_or_fetch_by_ap_id(uri)
   def get_or_fetch(nickname), do: get_or_fetch_by_nickname(nickname)
 
   # wait a period of time and return newest version of the User structs
@@ -2206,6 +2286,11 @@ defmodule Pleroma.User do
     |> Repo.all()
   end
 
+  @spec all_users_with_privilege(atom()) :: [User.t()]
+  def all_users_with_privilege(privilege) do
+    User.Query.build(%{is_privileged: privilege}) |> Repo.all()
+  end
+
   def muting_reblogs?(%User{} = user, %User{} = target) do
     UserRelationship.reblog_mute_exists?(user, target)
   end
@@ -2311,17 +2396,6 @@ defmodule Pleroma.User do
     }
   end
 
-  def ensure_keys_present(%{keys: keys} = user) when not is_nil(keys), do: {:ok, user}
-
-  def ensure_keys_present(%User{} = user) do
-    with {:ok, pem} <- Keys.generate_rsa_pem() do
-      user
-      |> cast(%{keys: pem}, [:keys])
-      |> validate_required([:keys])
-      |> update_and_set_cache()
-    end
-  end
-
   def get_ap_ids_by_nicknames(nicknames) do
     from(u in User,
       where: u.nickname in ^nicknames,
@@ -2364,6 +2438,38 @@ defmodule Pleroma.User do
     |> update_and_set_cache()
   end
 
+  def alias_users(user) do
+    user.also_known_as
+    |> Enum.map(&User.get_cached_by_ap_id/1)
+    |> Enum.filter(fn user -> user != nil end)
+  end
+
+  def add_alias(user, new_alias_user) do
+    current_aliases = user.also_known_as || []
+    new_alias_ap_id = new_alias_user.ap_id
+
+    if new_alias_ap_id in current_aliases do
+      {:ok, user}
+    else
+      user
+      |> cast(%{also_known_as: current_aliases ++ [new_alias_ap_id]}, [:also_known_as])
+      |> update_and_set_cache()
+    end
+  end
+
+  def delete_alias(user, alias_user) do
+    current_aliases = user.also_known_as || []
+    alias_ap_id = alias_user.ap_id
+
+    if alias_ap_id in current_aliases do
+      user
+      |> cast(%{also_known_as: current_aliases -- [alias_ap_id]}, [:also_known_as])
+      |> update_and_set_cache()
+    else
+      {:error, :no_such_alias}
+    end
+  end
+
   # Internal function; public one is `deactivate/2`
   defp set_activation_status(user, status) do
     user

lib/pleroma/user/backup.ex

@@ -32,9 +32,7 @@ defmodule Pleroma.User.Backup do
   end
 
   def create(user, admin_id \\ nil) do
-    with :ok <- validate_email_enabled(),
-         :ok <- validate_user_email(user),
-         :ok <- validate_limit(user, admin_id),
+    with :ok <- validate_limit(user, admin_id),
          {:ok, backup} <- user |> new() |> Repo.insert() do
       BackupWorker.process(backup, admin_id)
     end
@@ -86,20 +84,6 @@ defmodule Pleroma.User.Backup do
     end
   end
 
-  defp validate_email_enabled do
-    if Pleroma.Config.get([Pleroma.Emails.Mailer, :enabled]) do
-      :ok
-    else
-      {:error, dgettext("errors", "Backups require enabled email")}
-    end
-  end
-
-  defp validate_user_email(%User{email: nil}) do
-    {:error, dgettext("errors", "Email is required")}
-  end
-
-  defp validate_user_email(%User{email: email}) when is_binary(email), do: :ok
-
   def get_last(user_id) do
     __MODULE__
     |> where(user_id: ^user_id)

lib/pleroma/user/query.ex

@@ -29,6 +29,7 @@ defmodule Pleroma.User.Query do
   import Ecto.Query
   import Pleroma.Web.Utils.Guards, only: [not_empty_string: 1]
 
+  alias Pleroma.Config
   alias Pleroma.FollowingRelationship
   alias Pleroma.User
 
@@ -49,6 +50,7 @@ defmodule Pleroma.User.Query do
             is_suggested: boolean(),
             is_discoverable: boolean(),
             super_users: boolean(),
+            is_privileged: atom(),
             invisible: boolean(),
             internal: boolean(),
             followers: User.t(),
@@ -136,6 +138,43 @@ defmodule Pleroma.User.Query do
     )
   end
 
+  defp compose_query({:is_privileged, privilege}, query) do
+    moderator_privileged = privilege in Config.get([:instance, :moderator_privileges])
+    admin_privileged = privilege in Config.get([:instance, :admin_privileges])
+
+    query = compose_query({:active, true}, query)
+    query = compose_query({:local, true}, query)
+
+    case {admin_privileged, moderator_privileged} do
+      {false, false} ->
+        where(
+          query,
+          false
+        )
+
+      {true, true} ->
+        where(
+          query,
+          [u],
+          u.is_admin or u.is_moderator
+        )
+
+      {true, false} ->
+        where(
+          query,
+          [u],
+          u.is_admin
+        )
+
+      {false, true} ->
+        where(
+          query,
+          [u],
+          u.is_moderator
+        )
+    end
+  end
+
   defp compose_query({:local, _}, query), do: location_query(query, true)
 
   defp compose_query({:external, _}, query), do: location_query(query, false)

lib/pleroma/user/search.ex

@@ -94,6 +94,7 @@ defmodule Pleroma.User.Search do
     |> subquery()
     |> order_by(desc: :search_rank)
     |> maybe_restrict_local(for_user)
+    |> filter_deactivated_users()
   end
 
   defp select_top_users(query, top_user_ids) do
@@ -166,6 +167,10 @@ defmodule Pleroma.User.Search do
     from(q in query, where: q.actor_type != "Application")
   end
 
+  defp filter_deactivated_users(query) do
+    from(q in query, where: q.is_active == true)
+  end
+
   defp filter_blocked_user(query, %User{} = blocker) do
     query
     |> join(:left, [u], b in Pleroma.UserRelationship,

lib/pleroma/user_relationship.ex

@@ -18,16 +18,17 @@ defmodule Pleroma.UserRelationship do
     belongs_to(:source, User, type: FlakeId.Ecto.CompatType)
     belongs_to(:target, User, type: FlakeId.Ecto.CompatType)
     field(:relationship_type, Pleroma.UserRelationship.Type)
+    field(:expires_at, :utc_datetime)
 
     timestamps(updated_at: false)
   end
 
   for relationship_type <- Keyword.keys(Pleroma.UserRelationship.Type.__enum_map__()) do
-    # `def create_block/2`, `def create_mute/2`, `def create_reblog_mute/2`,
-    #   `def create_notification_mute/2`, `def create_inverse_subscription/2`,
-    #   `def endorsement/2`
-    def unquote(:"create_#{relationship_type}")(source, target),
-      do: create(unquote(relationship_type), source, target)
+    # `def create_block/3`, `def create_mute/3`, `def create_reblog_mute/3`,
+    #   `def create_notification_mute/3`, `def create_inverse_subscription/3`,
+    #   `def endorsement/3`
+    def unquote(:"create_#{relationship_type}")(source, target, expires_at \\ nil),
+      do: create(unquote(relationship_type), source, target, expires_at)
 
     # `def delete_block/2`, `def delete_mute/2`, `def delete_reblog_mute/2`,
     #   `def delete_notification_mute/2`, `def delete_inverse_subscription/2`,
@@ -37,9 +38,15 @@ defmodule Pleroma.UserRelationship do
 
     # `def block_exists?/2`, `def mute_exists?/2`, `def reblog_mute_exists?/2`,
     #   `def notification_mute_exists?/2`, `def inverse_subscription_exists?/2`,
-    #   `def inverse_endorsement?/2`
+    #   `def inverse_endorsement_exists?/2`
     def unquote(:"#{relationship_type}_exists?")(source, target),
       do: exists?(unquote(relationship_type), source, target)
+
+    # `def get_block_expire_date/2`, `def get_mute_expire_date/2`,
+    #   `def get_reblog_mute_expire_date/2`, `def get_notification_mute_exists?/2`,
+    #   `def get_inverse_subscription_expire_date/2`, `def get_inverse_endorsement_expire_date/2`
+    def unquote(:"get_#{relationship_type}_expire_date")(source, target),
+      do: get_expire_date(unquote(relationship_type), source, target)
   end
 
   def user_relationship_types, do: Keyword.keys(user_relationship_mappings())
@@ -48,7 +55,7 @@ defmodule Pleroma.UserRelationship do
 
   def changeset(%UserRelationship{} = user_relationship, params \\ %{}) do
     user_relationship
-    |> cast(params, [:relationship_type, :source_id, :target_id])
+    |> cast(params, [:relationship_type, :source_id, :target_id, :expires_at])
     |> validate_required([:relationship_type, :source_id, :target_id])
     |> unique_constraint(:relationship_type,
       name: :user_relationships_source_id_relationship_type_target_id_index
@@ -62,16 +69,31 @@ defmodule Pleroma.UserRelationship do
     |> Repo.exists?()
   end
 
-  def create(relationship_type, %User{} = source, %User{} = target) do
+  def get_expire_date(relationship_type, %User{} = source, %User{} = target) do
+    %UserRelationship{expires_at: expires_at} =
+      UserRelationship
+      |> where(
+        relationship_type: ^relationship_type,
+        source_id: ^source.id,
+        target_id: ^target.id
+      )
+      |> Repo.one!()
+
+    expires_at
+  end
+
+  def create(relationship_type, %User{} = source, %User{} = target, expires_at \\ nil) do
     %UserRelationship{}
     |> changeset(%{
       relationship_type: relationship_type,
       source_id: source.id,
-      target_id: target.id
+      target_id: target.id,
+      expires_at: expires_at
     })
     |> Repo.insert(
-      on_conflict: {:replace_all_except, [:id]},
-      conflict_target: [:source_id, :relationship_type, :target_id]
+      on_conflict: {:replace_all_except, [:id, :inserted_at]},
+      conflict_target: [:source_id, :relationship_type, :target_id],
+      returning: true
     )
   end
 

lib/pleroma/web/activity_pub/activity_pub.ex

@@ -190,7 +190,16 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
   def notify_and_stream(activity) do
     Notification.create_notifications(activity)
 
-    conversation = create_or_bump_conversation(activity, activity.actor)
+    original_activity =
+      case activity do
+        %{data: %{"type" => "Update"}, object: %{data: %{"id" => id}}} ->
+          Activity.get_create_by_object_ap_id_with_object(id)
+
+        _ ->
+          activity
+      end
+
+    conversation = create_or_bump_conversation(original_activity, original_activity.actor)
     participations = get_participations(conversation)
     stream_out(activity)
     stream_out_participations(participations)
@@ -256,7 +265,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
 
   @impl true
   def stream_out(%Activity{data: %{"type" => data_type}} = activity)
-      when data_type in ["Create", "Announce", "Delete"] do
+      when data_type in ["Create", "Announce", "Delete", "Update"] do
     activity
     |> Topics.get_activity_topics()
     |> Streamer.stream(activity)
@@ -392,11 +401,11 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
          _ <- notify_and_stream(activity),
          :ok <-
            maybe_federate(stripped_activity) do
-      User.all_superusers()
+      User.all_users_with_privilege(:reports_manage_reports)
       |> Enum.filter(fn user -> user.ap_id != actor end)
       |> Enum.filter(fn user -> not is_nil(user.email) end)
-      |> Enum.each(fn superuser ->
-        superuser
+      |> Enum.each(fn privileged_user ->
+        privileged_user
         |> Pleroma.Emails.AdminEmail.report(actor, account, statuses, content)
         |> Pleroma.Emails.Mailer.deliver_async()
       end)
@@ -413,7 +422,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
       "type" => "Move",
       "actor" => origin.ap_id,
       "object" => origin.ap_id,
-      "target" => target.ap_id
+      "target" => target.ap_id,
+      "to" => [origin.follower_address]
     }
 
     with true <- origin.ap_id in target.also_known_as,
@@ -501,9 +511,18 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
 
   @spec fetch_public_or_unlisted_activities(map(), Pagination.type()) :: [Activity.t()]
   def fetch_public_or_unlisted_activities(opts \\ %{}, pagination \\ :keyset) do
+    includes_local_public = Map.get(opts, :includes_local_public, false)
+
     opts = Map.delete(opts, :user)
 
-    [Constants.as_public()]
+    intended_recipients =
+      if includes_local_public do
+        [Constants.as_public(), as_local_public()]
+      else
+        [Constants.as_public()]
+      end
+
+    intended_recipients
     |> fetch_activities_query(opts)
     |> restrict_unlisted(opts)
     |> fetch_paginated_optimized(opts, pagination)
@@ -603,9 +622,11 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     do: query
 
   defp restrict_thread_visibility(query, %{user: %User{ap_id: ap_id}}, _) do
+    local_public = as_local_public()
+
     from(
       a in query,
-      where: fragment("thread_visibility(?, (?)->>'id') = true", ^ap_id, a.data)
+      where: fragment("thread_visibility(?, (?)->>'id', ?) = true", ^ap_id, a.data, ^local_public)
     )
   end
 
@@ -692,8 +713,12 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
   defp user_activities_recipients(%{godmode: true}), do: []
 
   defp user_activities_recipients(%{reading_user: reading_user}) do
-    if reading_user do
-      [Constants.as_public(), reading_user.ap_id | User.following(reading_user)]
+    if not is_nil(reading_user) and reading_user.local do
+      [
+        Constants.as_public(),
+        as_local_public(),
+        reading_user.ap_id | User.following(reading_user)
+      ]
     else
       [Constants.as_public()]
     end
@@ -1134,8 +1159,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
       [activity, object: o] in query,
       where:
         fragment(
-          "(?)->>'type' = 'Create' and coalesce((?)->'object'->>'id', (?)->>'object') = any (?)",
-          activity.data,
+          "(?)->>'type' = 'Create' and associated_object_id((?)) = any (?)",
           activity.data,
           activity.data,
           ^ids
@@ -1215,15 +1239,15 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     end
   end
 
+  defp exclude_invisible_actors(query, %{type: "Flag"}), do: query
   defp exclude_invisible_actors(query, %{invisible_actors: true}), do: query
 
   defp exclude_invisible_actors(query, _opts) do
-    invisible_ap_ids =
-      User.Query.build(%{invisible: true, select: [:ap_id]})
-      |> Repo.all()
-      |> Enum.map(fn %{ap_id: ap_id} -> ap_id end)
-
-    from([activity] in query, where: activity.actor not in ^invisible_ap_ids)
+    query
+    |> join(:inner, [activity], u in User,
+      as: :u,
+      on: activity.actor == u.ap_id and u.invisible == false
+    )
   end
 
   defp exclude_id(query, %{exclude_id: id}) when is_binary(id) do
@@ -1353,7 +1377,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
       |> restrict_instance(opts)
       |> restrict_announce_object_actor(opts)
       |> restrict_filtered(opts)
-      |> Activity.restrict_deactivated_users()
+      |> maybe_restrict_deactivated_users(opts)
       |> exclude_poll_votes(opts)
       |> exclude_chat_messages(opts)
       |> exclude_invisible_actors(opts)
@@ -1458,7 +1482,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
   defp normalize_image(urls) when is_list(urls), do: urls |> List.first() |> normalize_image()
   defp normalize_image(_), do: nil
 
-  defp object_to_user_data(data) do
+  defp object_to_user_data(data, additional) do
     fields =
       data
       |> Map.get("attachment", [])
@@ -1490,15 +1514,11 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     public_key =
       if is_map(data["publicKey"]) && is_binary(data["publicKey"]["publicKeyPem"]) do
         data["publicKey"]["publicKeyPem"]
-      else
-        nil
       end
 
     shared_inbox =
       if is_map(data["endpoints"]) && is_binary(data["endpoints"]["sharedInbox"]) do
         data["endpoints"]["sharedInbox"]
-      else
-        nil
       end
 
     birthday =
@@ -1507,13 +1527,15 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
           {:ok, date} -> date
           {:error, _} -> nil
         end
-      else
-        nil
       end
 
     show_birthday = !!birthday
 
-    user_data = %{
+    # if WebFinger request was already done, we probably have acct, otherwise
+    # we request WebFinger here
+    nickname = additional[:nickname_from_acct] || generate_nickname(data)
+
+    %{
       ap_id: data["id"],
       uri: get_actor_url(data["url"]),
       ap_enabled: true,
@@ -1535,23 +1557,29 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
       inbox: data["inbox"],
       shared_inbox: shared_inbox,
       accepts_chat_messages: accepts_chat_messages,
-      pinned_objects: pinned_objects,
       birthday: birthday,
-      show_birthday: show_birthday
+      show_birthday: show_birthday,
+      pinned_objects: pinned_objects,
+      nickname: nickname
     }
+  end
 
-    # nickname can be nil because of virtual actors
-    if data["preferredUsername"] do
-      Map.put(
-        user_data,
-        :nickname,
-        "#{data["preferredUsername"]}@#{URI.parse(data["id"]).host}"
-      )
+  defp generate_nickname(%{"preferredUsername" => username} = data) when is_binary(username) do
+    generated = "#{username}@#{URI.parse(data["id"]).host}"
+
+    if Config.get([WebFinger, :update_nickname_on_user_fetch]) do
+      case WebFinger.finger(generated) do
+        {:ok, %{"subject" => "acct:" <> acct}} -> acct
+        _ -> generated
+      end
     else
-      Map.put(user_data, :nickname, nil)
+      generated
     end
   end
 
+  # nickname can be nil because of virtual actors
+  defp generate_nickname(_), do: nil
+
   def fetch_follow_information_for_user(user) do
     with {:ok, following_data} <-
            Fetcher.fetch_and_contain_remote_object_from_id(user.following_address),
@@ -1623,17 +1651,17 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
 
   defp collection_private(_data), do: {:ok, true}
 
-  def user_data_from_user_object(data) do
+  def user_data_from_user_object(data, additional \\ []) do
     with {:ok, data} <- MRF.filter(data) do
-      {:ok, object_to_user_data(data)}
+      {:ok, object_to_user_data(data, additional)}
     else
       e -> {:error, e}
     end
   end
 
-  def fetch_and_prepare_user_from_ap_id(ap_id) do
+  def fetch_and_prepare_user_from_ap_id(ap_id, additional \\ []) do
     with {:ok, data} <- Fetcher.fetch_and_contain_remote_object_from_id(ap_id),
-         {:ok, data} <- user_data_from_user_object(data) do
+         {:ok, data} <- user_data_from_user_object(data, additional) do
       {:ok, maybe_update_follow_information(data)}
     else
       # If this has been deleted, only log a debug and not an error
@@ -1711,13 +1739,13 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     end
   end
 
-  def make_user_from_ap_id(ap_id) do
+  def make_user_from_ap_id(ap_id, additional \\ []) do
     user = User.get_cached_by_ap_id(ap_id)
 
     if user && !User.ap_enabled?(user) do
       Transmogrifier.upgrade_user_from_ap_id(ap_id)
     else
-      with {:ok, data} <- fetch_and_prepare_user_from_ap_id(ap_id) do
+      with {:ok, data} <- fetch_and_prepare_user_from_ap_id(ap_id, additional) do
         {:ok, _pid} = Task.start(fn -> pinned_fetch_task(data) end)
 
         if user do
@@ -1737,8 +1765,9 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
   end
 
   def make_user_from_nickname(nickname) do
-    with {:ok, %{"ap_id" => ap_id}} when not is_nil(ap_id) <- WebFinger.finger(nickname) do
-      make_user_from_ap_id(ap_id)
+    with {:ok, %{"ap_id" => ap_id, "subject" => "acct:" <> acct}} when not is_nil(ap_id) <-
+           WebFinger.finger(nickname) do
+      make_user_from_ap_id(ap_id, nickname_from_acct: acct)
     else
       _e -> {:error, "No AP id in WebFinger"}
     end
@@ -1760,4 +1789,9 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     |> restrict_visibility(%{visibility: "direct"})
     |> order_by([activity], asc: activity.id)
   end
+
+  defp maybe_restrict_deactivated_users(activity, %{type: "Flag"}), do: activity
+
+  defp maybe_restrict_deactivated_users(activity, _opts),
+    do: Activity.restrict_deactivated_users(activity)
 end

lib/pleroma/web/activity_pub/activity_pub_controller.ex

@@ -66,8 +66,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
   end
 
   def user(conn, %{"nickname" => nickname}) do
-    with %User{local: true} = user <- User.get_cached_by_nickname(nickname),
-         {:ok, user} <- User.ensure_keys_present(user) do
+    with %User{local: true} = user <- User.get_cached_by_nickname(nickname) do
       conn
       |> put_resp_content_type("application/activity+json")
       |> put_view(UserView)
@@ -84,6 +83,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
          user <- Map.get(assigns, :user, nil),
          {_, true} <- {:visible?, Visibility.visible_for_user?(object, user)} do
       conn
+      |> maybe_skip_cache(user)
       |> assign(:tracking_fun_data, object.id)
       |> set_cache_ttl_for(object)
       |> put_resp_content_type("application/activity+json")
@@ -112,6 +112,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
          user <- Map.get(assigns, :user, nil),
          {_, true} <- {:visible?, Visibility.visible_for_user?(activity, user)} do
       conn
+      |> maybe_skip_cache(user)
       |> maybe_set_tracking_data(activity)
       |> set_cache_ttl_for(activity)
       |> put_resp_content_type("application/activity+json")
@@ -151,6 +152,15 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
     assign(conn, :cache_ttl, ttl)
   end
 
+  def maybe_skip_cache(conn, user) do
+    if user do
+      conn
+      |> assign(:skip_cache, true)
+    else
+      conn
+    end
+  end
+
   # GET /relay/following
   def relay_following(conn, _params) do
     with %{halted: false} = conn <- FederatingPlug.call(conn, []) do
@@ -163,7 +173,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
 
   def following(%{assigns: %{user: for_user}} = conn, %{"nickname" => nickname, "page" => page}) do
     with %User{} = user <- User.get_cached_by_nickname(nickname),
-         {user, for_user} <- ensure_user_keys_present_and_maybe_refresh_for_user(user, for_user),
          {:show_follows, true} <-
            {:show_follows, (for_user && for_user == user) || !user.hide_follows} do
       {page, _} = Integer.parse(page)
@@ -181,8 +190,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
   end
 
   def following(%{assigns: %{user: for_user}} = conn, %{"nickname" => nickname}) do
-    with %User{} = user <- User.get_cached_by_nickname(nickname),
-         {user, for_user} <- ensure_user_keys_present_and_maybe_refresh_for_user(user, for_user) do
+    with %User{} = user <- User.get_cached_by_nickname(nickname) do
       conn
       |> put_resp_content_type("application/activity+json")
       |> put_view(UserView)
@@ -202,7 +210,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
 
   def followers(%{assigns: %{user: for_user}} = conn, %{"nickname" => nickname, "page" => page}) do
     with %User{} = user <- User.get_cached_by_nickname(nickname),
-         {user, for_user} <- ensure_user_keys_present_and_maybe_refresh_for_user(user, for_user),
          {:show_followers, true} <-
            {:show_followers, (for_user && for_user == user) || !user.hide_followers} do
       {page, _} = Integer.parse(page)
@@ -220,8 +227,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
   end
 
   def followers(%{assigns: %{user: for_user}} = conn, %{"nickname" => nickname}) do
-    with %User{} = user <- User.get_cached_by_nickname(nickname),
-         {user, for_user} <- ensure_user_keys_present_and_maybe_refresh_for_user(user, for_user) do
+    with %User{} = user <- User.get_cached_by_nickname(nickname) do
       conn
       |> put_resp_content_type("application/activity+json")
       |> put_view(UserView)
@@ -234,8 +240,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
         %{"nickname" => nickname, "page" => page?} = params
       )
       when page? in [true, "true"] do
-    with %User{} = user <- User.get_cached_by_nickname(nickname),
-         {:ok, user} <- User.ensure_keys_present(user) do
+    with %User{} = user <- User.get_cached_by_nickname(nickname) do
       # "include_poll_votes" is a hack because postgres generates inefficient
       # queries when filtering by 'Answer', poll votes will be hidden by the
       # visibility filter in this case anyway
@@ -259,8 +264,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
   end
 
   def outbox(conn, %{"nickname" => nickname}) do
-    with %User{} = user <- User.get_cached_by_nickname(nickname),
-         {:ok, user} <- User.ensure_keys_present(user) do
+    with %User{} = user <- User.get_cached_by_nickname(nickname) do
       conn
       |> put_resp_content_type("application/activity+json")
       |> put_view(UserView)
@@ -317,14 +321,10 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
   end
 
   defp represent_service_actor(%User{} = user, conn) do
-    with {:ok, user} <- User.ensure_keys_present(user) do
-      conn
-      |> put_resp_content_type("application/activity+json")
-      |> put_view(UserView)
-      |> render("user.json", %{user: user})
-    else
-      nil -> {:error, :not_found}
-    end
+    conn
+    |> put_resp_content_type("application/activity+json")
+    |> put_view(UserView)
+    |> render("user.json", %{user: user})
   end
 
   defp represent_service_actor(nil, _), do: {:error, :not_found}
@@ -377,12 +377,10 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
   def read_inbox(%{assigns: %{user: %User{nickname: nickname} = user}} = conn, %{
         "nickname" => nickname
       }) do
-    with {:ok, user} <- User.ensure_keys_present(user) do
-      conn
-      |> put_resp_content_type("application/activity+json")
-      |> put_view(UserView)
-      |> render("activity_collection.json", %{iri: "#{user.ap_id}/inbox"})
-    end
+    conn
+    |> put_resp_content_type("application/activity+json")
+    |> put_view(UserView)
+    |> render("activity_collection.json", %{iri: "#{user.ap_id}/inbox"})
   end
 
   def read_inbox(%{assigns: %{user: %User{nickname: as_nickname}}} = conn, %{
@@ -519,19 +517,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
     conn
   end
 
-  defp ensure_user_keys_present_and_maybe_refresh_for_user(user, for_user) do
-    {:ok, new_user} = User.ensure_keys_present(user)
-
-    for_user =
-      if new_user != user and match?(%User{}, for_user) do
-        User.get_cached_by_nickname(for_user.nickname)
-      else
-        for_user
-      end
-
-    {new_user, for_user}
-  end
-
   def upload_media(%{assigns: %{user: %User{} = user}} = conn, %{"file" => file} = data) do
     with {:ok, object} <-
            ActivityPub.upload(

lib/pleroma/web/activity_pub/builder.ex

@@ -218,10 +218,16 @@ defmodule Pleroma.Web.ActivityPub.Builder do
     end
   end
 
-  # Retricted to user updates for now, always public
   @spec update(User.t(), Object.t()) :: {:ok, map(), keyword()}
   def update(actor, object) do
-    to = [Pleroma.Constants.as_public(), actor.follower_address]
+    {to, cc} =
+      if object["type"] in Pleroma.Constants.actor_types() do
+        # User updates, always public
+        {[Pleroma.Constants.as_public(), actor.follower_address], []}
+      else
+        # Status updates, follow the recipients in the object
+        {object["to"] || [], object["cc"] || []}
+      end
 
     {:ok,
      %{
@@ -229,7 +235,8 @@ defmodule Pleroma.Web.ActivityPub.Builder do
        "type" => "Update",
        "actor" => actor.ap_id,
        "object" => object,
-       "to" => to
+       "to" => to,
+       "cc" => cc
      }, []}
   end
 

lib/pleroma/web/activity_pub/mrf.ex

@@ -53,10 +53,53 @@ defmodule Pleroma.Web.ActivityPub.MRF do
 
   @required_description_keys [:key, :related_policy]
 
+  def filter_one(policy, message) do
+    should_plug_history? =
+      if function_exported?(policy, :history_awareness, 0) do
+        policy.history_awareness()
+      else
+        :manual
+      end
+      |> Kernel.==(:auto)
+
+    if not should_plug_history? do
+      policy.filter(message)
+    else
+      main_result = policy.filter(message)
+
+      with {_, {:ok, main_message}} <- {:main, main_result},
+           {_,
+            %{
+              "formerRepresentations" => %{
+                "orderedItems" => [_ | _]
+              }
+            }} = {_, object} <- {:object, message["object"]},
+           {_, {:ok, new_history}} <-
+             {:history,
+              Pleroma.Object.Updater.for_each_history_item(
+                object["formerRepresentations"],
+                object,
+                fn item ->
+                  with {:ok, filtered} <- policy.filter(Map.put(message, "object", item)) do
+                    {:ok, filtered["object"]}
+                  else
+                    e -> e
+                  end
+                end
+              )} do
+        {:ok, put_in(main_message, ["object", "formerRepresentations"], new_history)}
+      else
+        {:main, _} -> main_result
+        {:object, _} -> main_result
+        {:history, e} -> e
+      end
+    end
+  end
+
   def filter(policies, %{} = message) do
     policies
     |> Enum.reduce({:ok, message}, fn
-      policy, {:ok, message} -> policy.filter(message)
+      policy, {:ok, message} -> filter_one(policy, message)
       _, error -> error
     end)
   end

lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex

@@ -24,7 +24,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.AntiFollowbotPolicy do
   defp score_displayname("fedibot"), do: 1.0
   defp score_displayname(_), do: 0.0
 
-  defp determine_if_followbot(%User{nickname: nickname, name: displayname}) do
+  defp determine_if_followbot(%User{nickname: nickname, name: displayname, actor_type: actor_type}) do
     # nickname will be a binary string except when following a relay
     nick_score =
       if is_binary(nickname) do
@@ -45,19 +45,32 @@ defmodule Pleroma.Web.ActivityPub.MRF.AntiFollowbotPolicy do
         0.0
       end
 
-    nick_score + name_score
+    # actor_type "Service" is a Bot account
+    actor_type_score =
+      if actor_type == "Service" do
+        1.0
+      else
+        0.0
+      end
+
+    nick_score + name_score + actor_type_score
   end
 
   defp determine_if_followbot(_), do: 0.0
 
+  defp bot_allowed?(%{"object" => target}, bot_actor) do
+    %User{} = user = normalize_by_ap_id(target)
+
+    User.following?(user, bot_actor)
+  end
+
   @impl true
   def filter(%{"type" => "Follow", "actor" => actor_id} = message) do
     %User{} = actor = normalize_by_ap_id(actor_id)
 
     score = determine_if_followbot(actor)
 
-    # TODO: scan biography data for keywords and score it somehow.
-    if score < 0.8 do
+    if score < 0.8 || bot_allowed?(message, actor) do
       {:ok, message}
     else
       {:reject, "[AntiFollowbotPolicy] Scored #{actor_id} as #{score}"}

lib/pleroma/web/activity_pub/mrf/anti_link_spam_policy.ex

@@ -9,6 +9,9 @@ defmodule Pleroma.Web.ActivityPub.MRF.AntiLinkSpamPolicy do
 
   require Logger
 
+  @impl true
+  def history_awareness, do: :auto
+
   # has the user successfully posted before?
   defp old_user?(%User{} = u) do
     u.note_count > 0 || u.follower_count > 0

lib/pleroma/web/activity_pub/mrf/ensure_re_prepended.ex

@@ -10,6 +10,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.EnsureRePrepended do
 
   @reply_prefix Regex.compile!("^re:[[:space:]]*", [:caseless])
 
+  def history_awareness, do: :auto
+
   def filter_by_summary(
         %{data: %{"summary" => parent_summary}} = _in_reply_to,
         %{"summary" => child_summary} = child
@@ -27,8 +29,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.EnsureRePrepended do
 
   def filter_by_summary(_in_reply_to, child), do: child
 
-  def filter(%{"type" => "Create", "object" => child_object} = object)
-      when is_map(child_object) do
+  def filter(%{"type" => type, "object" => child_object} = object)
+      when type in ["Create", "Update"] and is_map(child_object) do
     child =
       child_object["inReplyTo"]
       |> Object.normalize(fetch: false)

lib/pleroma/web/activity_pub/mrf/force_mentions_in_content.ex

@@ -11,6 +11,9 @@ defmodule Pleroma.Web.ActivityPub.MRF.ForceMentionsInContent do
 
   @behaviour Pleroma.Web.ActivityPub.MRF.Policy
 
+  @impl true
+  def history_awareness, do: :auto
+
   defp do_extract({:a, attrs, _}, acc) do
     if Enum.find(attrs, fn {name, value} ->
          name == "class" && value in ["mention", "u-url mention", "mention u-url"]
@@ -74,11 +77,11 @@ defmodule Pleroma.Web.ActivityPub.MRF.ForceMentionsInContent do
   @impl true
   def filter(
         %{
-          "type" => "Create",
+          "type" => type,
           "object" => %{"type" => "Note", "to" => to, "inReplyTo" => in_reply_to}
         } = object
       )
-      when is_list(to) and is_binary(in_reply_to) do
+      when type in ["Create", "Update"] and is_list(to) and is_binary(in_reply_to) do
     # image-only posts from pleroma apparently reach this MRF without the content field
     content = object["object"]["content"] || ""
 

lib/pleroma/web/activity_pub/mrf/hashtag_policy.ex

@@ -16,6 +16,9 @@ defmodule Pleroma.Web.ActivityPub.MRF.HashtagPolicy do
 
   @behaviour Pleroma.Web.ActivityPub.MRF.Policy
 
+  @impl true
+  def history_awareness, do: :manual
+
   defp check_reject(message, hashtags) do
     if Enum.any?(Config.get([:mrf_hashtag, :reject]), fn match -> match in hashtags end) do
       {:reject, "[HashtagPolicy] Matches with rejected keyword"}
@@ -47,22 +50,46 @@ defmodule Pleroma.Web.ActivityPub.MRF.HashtagPolicy do
 
   defp check_ftl_removal(message, _hashtags), do: {:ok, message}
 
-  defp check_sensitive(message, hashtags) do
-    if Enum.any?(Config.get([:mrf_hashtag, :sensitive]), fn match -> match in hashtags end) do
-      {:ok, Kernel.put_in(message, ["object", "sensitive"], true)}
-    else
-      {:ok, message}
-    end
+  defp check_sensitive(message) do
+    {:ok, new_object} =
+      Object.Updater.do_with_history(message["object"], fn object ->
+        hashtags = Object.hashtags(%Object{data: object})
+
+        if Enum.any?(Config.get([:mrf_hashtag, :sensitive]), fn match -> match in hashtags end) do
+          {:ok, Map.put(object, "sensitive", true)}
+        else
+          {:ok, object}
+        end
+      end)
+
+    {:ok, Map.put(message, "object", new_object)}
   end
 
   @impl true
-  def filter(%{"type" => "Create", "object" => object} = message) do
-    hashtags = Object.hashtags(%Object{data: object})
+  def filter(%{"type" => type, "object" => object} = message) when type in ["Create", "Update"] do
+    history_items =
+      with %{"formerRepresentations" => %{"orderedItems" => items}} <- object do
+        items
+      else
+        _ -> []
+      end
+
+    historical_hashtags =
+      Enum.reduce(history_items, [], fn item, acc ->
+        acc ++ Object.hashtags(%Object{data: item})
+      end)
+
+    hashtags = Object.hashtags(%Object{data: object}) ++ historical_hashtags
 
     if hashtags != [] do
       with {:ok, message} <- check_reject(message, hashtags),
-           {:ok, message} <- check_ftl_removal(message, hashtags),
-           {:ok, message} <- check_sensitive(message, hashtags) do
+           {:ok, message} <-
+             (if "type" == "Create" do
+                check_ftl_removal(message, hashtags)
+              else
+                {:ok, message}
+              end),
+           {:ok, message} <- check_sensitive(message) do
         {:ok, message}
       end
     else

lib/pleroma/web/activity_pub/mrf/keyword_policy.ex

@@ -27,24 +27,46 @@ defmodule Pleroma.Web.ActivityPub.MRF.KeywordPolicy do
   end
 
   defp check_reject(%{"object" => %{} = object} = message) do
-    payload = object_payload(object)
+    with {:ok, _new_object} <-
+           Pleroma.Object.Updater.do_with_history(object, fn object ->
+             payload = object_payload(object)
 
-    if Enum.any?(Pleroma.Config.get([:mrf_keyword, :reject]), fn pattern ->
-         string_matches?(payload, pattern)
-       end) do
-      {:reject, "[KeywordPolicy] Matches with rejected keyword"}
-    else
+             if Enum.any?(Pleroma.Config.get([:mrf_keyword, :reject]), fn pattern ->
+                  string_matches?(payload, pattern)
+                end) do
+               {:reject, "[KeywordPolicy] Matches with rejected keyword"}
+             else
+               {:ok, message}
+             end
+           end) do
       {:ok, message}
+    else
+      e -> e
     end
   end
 
-  defp check_ftl_removal(%{"to" => to, "object" => %{} = object} = message) do
-    payload = object_payload(object)
+  defp check_ftl_removal(%{"type" => "Create", "to" => to, "object" => %{} = object} = message) do
+    check_keyword = fn object ->
+      payload = object_payload(object)
 
-    if Pleroma.Constants.as_public() in to and
-         Enum.any?(Pleroma.Config.get([:mrf_keyword, :federated_timeline_removal]), fn pattern ->
+      if Enum.any?(Pleroma.Config.get([:mrf_keyword, :federated_timeline_removal]), fn pattern ->
            string_matches?(payload, pattern)
          end) do
+        {:should_delist, nil}
+      else
+        {:ok, %{}}
+      end
+    end
+
+    should_delist? = fn object ->
+      with {:ok, _} <- Pleroma.Object.Updater.do_with_history(object, check_keyword) do
+        false
+      else
+        _ -> true
+      end
+    end
+
+    if Pleroma.Constants.as_public() in to and should_delist?.(object) do
       to = List.delete(to, Pleroma.Constants.as_public())
       cc = [Pleroma.Constants.as_public() | message["cc"] || []]
 
@@ -59,8 +81,12 @@ defmodule Pleroma.Web.ActivityPub.MRF.KeywordPolicy do
     end
   end
 
+  defp check_ftl_removal(message) do
+    {:ok, message}
+  end
+
   defp check_replace(%{"object" => %{} = object} = message) do
-    object =
+    replace_kw = fn object ->
       ["content", "name", "summary"]
       |> Enum.filter(fn field -> Map.has_key?(object, field) && object[field] end)
       |> Enum.reduce(object, fn field, object ->
@@ -73,6 +99,10 @@ defmodule Pleroma.Web.ActivityPub.MRF.KeywordPolicy do
 
         Map.put(object, field, data)
       end)
+      |> (fn object -> {:ok, object} end).()
+    end
+
+    {:ok, object} = Pleroma.Object.Updater.do_with_history(object, replace_kw)
 
     message = Map.put(message, "object", object)
 
@@ -80,7 +110,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.KeywordPolicy do
   end
 
   @impl true
-  def filter(%{"type" => "Create", "object" => %{"content" => _content}} = message) do
+  def filter(%{"type" => type, "object" => %{"content" => _content}} = message)
+      when type in ["Create", "Update"] do
     with {:ok, message} <- check_reject(message),
          {:ok, message} <- check_ftl_removal(message),
          {:ok, message} <- check_replace(message) do

lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex

@@ -16,6 +16,9 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do
     recv_timeout: 10_000
   ]
 
+  @impl true
+  def history_awareness, do: :auto
+
   defp prefetch(url) do
     # Fetching only proxiable resources
     if MediaProxy.enabled?() and MediaProxy.url_proxiable?(url) do
@@ -54,10 +57,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do
   end
 
   @impl true
-  def filter(
-        %{"type" => "Create", "object" => %{"attachment" => attachments} = _object} = message
-      )
-      when is_list(attachments) and length(attachments) > 0 do
+  def filter(%{"type" => type, "object" => %{"attachment" => attachments} = _object} = message)
+      when type in ["Create", "Update"] and is_list(attachments) and length(attachments) > 0 do
     preload(message)
 
     {:ok, message}

lib/pleroma/web/activity_pub/mrf/no_empty_policy.ex

@@ -11,6 +11,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.NoEmptyPolicy do
   @impl true
   def filter(%{"actor" => actor} = object) do
     with true <- is_local?(actor),
+         true <- is_eligible_type?(object),
          true <- is_note?(object),
          false <- has_attachment?(object),
          true <- only_mentions?(object) do
@@ -32,7 +33,6 @@ defmodule Pleroma.Web.ActivityPub.MRF.NoEmptyPolicy do
   end
 
   defp has_attachment?(%{
-         "type" => "Create",
          "object" => %{"type" => "Note", "attachment" => attachments}
        })
        when length(attachments) > 0,
@@ -40,7 +40,13 @@ defmodule Pleroma.Web.ActivityPub.MRF.NoEmptyPolicy do
 
   defp has_attachment?(_), do: false
 
-  defp only_mentions?(%{"type" => "Create", "object" => %{"type" => "Note", "source" => source}}) do
+  defp only_mentions?(%{"object" => %{"type" => "Note", "source" => source}}) do
+    source =
+      case source do
+        %{"content" => text} -> text
+        _ -> source
+      end
+
     non_mentions =
       source |> String.split() |> Enum.filter(&(not String.starts_with?(&1, "@"))) |> length
 
@@ -53,9 +59,12 @@ defmodule Pleroma.Web.ActivityPub.MRF.NoEmptyPolicy do
 
   defp only_mentions?(_), do: false
 
-  defp is_note?(%{"type" => "Create", "object" => %{"type" => "Note"}}), do: true
+  defp is_note?(%{"object" => %{"type" => "Note"}}), do: true
   defp is_note?(_), do: false
 
+  defp is_eligible_type?(%{"type" => type}) when type in ["Create", "Update"], do: true
+  defp is_eligible_type?(_), do: false
+
   @impl true
   def describe, do: {:ok, %{}}
 end

lib/pleroma/web/activity_pub/mrf/no_placeholder_text_policy.ex

@@ -6,14 +6,17 @@ defmodule Pleroma.Web.ActivityPub.MRF.NoPlaceholderTextPolicy do
   @moduledoc "Ensure no content placeholder is present (such as the dot from mastodon)"
   @behaviour Pleroma.Web.ActivityPub.MRF.Policy
 
+  @impl true
+  def history_awareness, do: :auto
+
   @impl true
   def filter(
         %{
-          "type" => "Create",
+          "type" => type,
           "object" => %{"content" => content, "attachment" => _} = _child_object
         } = object
       )
-      when content in [".", "<p>.</p>"] do
+      when type in ["Create", "Update"] and content in [".", "<p>.</p>"] do
     {:ok, put_in(object, ["object", "content"], "")}
   end
 

lib/pleroma/web/activity_pub/mrf/normalize_markup.ex

@@ -9,7 +9,11 @@ defmodule Pleroma.Web.ActivityPub.MRF.NormalizeMarkup do
   @behaviour Pleroma.Web.ActivityPub.MRF.Policy
 
   @impl true
-  def filter(%{"type" => "Create", "object" => child_object} = object) do
+  def history_awareness, do: :auto
+
+  @impl true
+  def filter(%{"type" => type, "object" => child_object} = object)
+      when type in ["Create", "Update"] do
     scrub_policy = Pleroma.Config.get([:mrf_normalize_markup, :scrub_policy])
 
     content =

lib/pleroma/web/activity_pub/mrf/object_age_policy.ex

@@ -131,7 +131,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.ObjectAgePolicy do
           type: {:list, :atom},
           description:
             "A list of actions to apply to the post. `:delist` removes the post from public timelines; " <>
-              "`:strip_followers` removes followers from the ActivityPub recipient list ensuring they won't be delivered to home timelines; " <>
+              "`:strip_followers` removes followers from the ActivityPub recipient list ensuring they won't be delivered to home timelines, additionally for followers-only it degrades to a direct message; " <>
               "`:reject` rejects the message entirely",
           suggestions: [:delist, :strip_followers, :reject]
         }

lib/pleroma/web/activity_pub/mrf/policy.ex

@@ -12,5 +12,6 @@ defmodule Pleroma.Web.ActivityPub.MRF.Policy do
               label: String.t(),
               description: String.t()
             }
-  @optional_callbacks config_description: 0
+  @callback history_awareness() :: :auto | :manual
+  @optional_callbacks config_description: 0, history_awareness: 0
 end

lib/pleroma/web/activity_pub/mrf/simple_policy.ex

@@ -40,9 +40,9 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do
 
   defp check_media_removal(
          %{host: actor_host} = _actor_info,
-         %{"type" => "Create", "object" => %{"attachment" => child_attachment}} = object
+         %{"type" => type, "object" => %{"attachment" => child_attachment}} = object
        )
-       when length(child_attachment) > 0 do
+       when length(child_attachment) > 0 and type in ["Create", "Update"] do
     media_removal =
       instance_list(:media_removal)
       |> MRF.subdomains_regex()
@@ -63,10 +63,11 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do
   defp check_media_nsfw(
          %{host: actor_host} = _actor_info,
          %{
-           "type" => "Create",
+           "type" => type,
            "object" => %{} = _child_object
          } = object
-       ) do
+       )
+       when type in ["Create", "Update"] do
     media_nsfw =
       instance_list(:media_nsfw)
       |> MRF.subdomains_regex()

lib/pleroma/web/activity_pub/mrf/steal_emoji_policy.ex

@@ -12,6 +12,14 @@ defmodule Pleroma.Web.ActivityPub.MRF.StealEmojiPolicy do
 
   defp accept_host?(host), do: host in Config.get([:mrf_steal_emoji, :hosts], [])
 
+  defp shortcode_matches?(shortcode, pattern) when is_binary(pattern) do
+    shortcode == pattern
+  end
+
+  defp shortcode_matches?(shortcode, pattern) do
+    String.match?(shortcode, pattern)
+  end
+
   defp steal_emoji({shortcode, url}, emoji_dir_path) do
     url = Pleroma.Web.MediaProxy.url(url)
 
@@ -72,7 +80,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.StealEmojiPolicy do
           reject_emoji? =
             [:mrf_steal_emoji, :rejected_shortcodes]
             |> Config.get([])
-            |> Enum.find(false, fn regex -> String.match?(shortcode, regex) end)
+            |> Enum.find(false, fn pattern -> shortcode_matches?(shortcode, pattern) end)
 
           !reject_emoji?
         end)
@@ -122,8 +130,12 @@ defmodule Pleroma.Web.ActivityPub.MRF.StealEmojiPolicy do
         %{
           key: :rejected_shortcodes,
           type: {:list, :string},
-          description: "Regex-list of shortcodes to reject",
-          suggestions: [""]
+          description: """
+            A list of patterns or matches to reject shortcodes with.
+
+            Each pattern can be a string or [Regex](https://hexdocs.pm/elixir/Regex.html) in the format of `~r/PATTERN/`.
+          """,
+          suggestions: ["foo", ~r/foo/]
         },
         %{
           key: :size_limit,

lib/pleroma/web/activity_pub/mrf/tag_policy.ex

@@ -27,22 +27,22 @@ defmodule Pleroma.Web.ActivityPub.MRF.TagPolicy do
   defp process_tag(
          "mrf_tag:media-force-nsfw",
          %{
-           "type" => "Create",
+           "type" => type,
            "object" => %{"attachment" => child_attachment}
          } = message
        )
-       when length(child_attachment) > 0 do
+       when length(child_attachment) > 0 and type in ["Create", "Update"] do
     {:ok, Kernel.put_in(message, ["object", "sensitive"], true)}
   end
 
   defp process_tag(
          "mrf_tag:media-strip",
          %{
-           "type" => "Create",
+           "type" => type,
            "object" => %{"attachment" => child_attachment} = object
          } = message
        )
-       when length(child_attachment) > 0 do
+       when length(child_attachment) > 0 and type in ["Create", "Update"] do
     object = Map.delete(object, "attachment")
     message = Map.put(message, "object", object)
 
@@ -152,7 +152,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.TagPolicy do
     do: filter_message(target_actor, message)
 
   @impl true
-  def filter(%{"actor" => actor, "type" => "Create"} = message),
+  def filter(%{"actor" => actor, "type" => type} = message) when type in ["Create", "Update"],
     do: filter_message(actor, message)
 
   @impl true

lib/pleroma/web/activity_pub/object_validator.ex

@@ -103,8 +103,8 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do
         meta
       )
       when objtype in ~w[Question Answer Audio Video Event Article Note Page] do
-    with {:ok, object_data} <- cast_and_apply(object),
-         meta = Keyword.put(meta, :object_data, object_data |> stringify_keys),
+    with {:ok, object_data} <- cast_and_apply_and_stringify_with_history(object),
+         meta = Keyword.put(meta, :object_data, object_data),
          {:ok, create_activity} <-
            create_activity
            |> CreateGenericValidator.cast_and_validate(meta)
@@ -128,19 +128,53 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do
       end
 
     with {:ok, object} <-
-           object
-           |> validator.cast_and_validate()
-           |> Ecto.Changeset.apply_action(:insert) do
-      object = stringify_keys(object)
+           do_separate_with_history(object, fn object ->
+             with {:ok, object} <-
+                    object
+                    |> validator.cast_and_validate()
+                    |> Ecto.Changeset.apply_action(:insert) do
+               object = stringify_keys(object)
 
-      # Insert copy of hashtags as strings for the non-hashtag table indexing
-      tag = (object["tag"] || []) ++ Object.hashtags(%Object{data: object})
-      object = Map.put(object, "tag", tag)
+               # Insert copy of hashtags as strings for the non-hashtag table indexing
+               tag = (object["tag"] || []) ++ Object.hashtags(%Object{data: object})
+               object = Map.put(object, "tag", tag)
 
+               {:ok, object}
+             end
+           end) do
       {:ok, object, meta}
     end
   end
 
+  def validate(
+        %{"type" => "Update", "object" => %{"type" => objtype} = object} = update_activity,
+        meta
+      )
+      when objtype in ~w[Question Answer Audio Video Event Article Note Page] do
+    with {_, false} <- {:local, Access.get(meta, :local, false)},
+         {_, {:ok, object_data, _}} <- {:object_validation, validate(object, meta)},
+         meta = Keyword.put(meta, :object_data, object_data),
+         {:ok, update_activity} <-
+           update_activity
+           |> UpdateValidator.cast_and_validate()
+           |> Ecto.Changeset.apply_action(:insert) do
+      update_activity = stringify_keys(update_activity)
+      {:ok, update_activity, meta}
+    else
+      {:local, _} ->
+        with {:ok, object} <-
+               update_activity
+               |> UpdateValidator.cast_and_validate()
+               |> Ecto.Changeset.apply_action(:insert) do
+          object = stringify_keys(object)
+          {:ok, object, meta}
+        end
+
+      {:object_validation, e} ->
+        e
+    end
+  end
+
   def validate(%{"type" => type} = object, meta)
       when type in ~w[Accept Reject Follow Update Like EmojiReact Announce
       ChatMessage Answer] do
@@ -178,6 +212,15 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do
 
   def validate(o, m), do: {:error, {:validator_not_set, {o, m}}}
 
+  def cast_and_apply_and_stringify_with_history(object) do
+    do_separate_with_history(object, fn object ->
+      with {:ok, object_data} <- cast_and_apply(object),
+           object_data <- object_data |> stringify_keys() do
+        {:ok, object_data}
+      end
+    end)
+  end
+
   def cast_and_apply(%{"type" => "ChatMessage"} = object) do
     ChatMessageValidator.cast_and_apply(object)
   end
@@ -204,8 +247,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do
 
   def cast_and_apply(o), do: {:error, {:validator_not_set, o}}
 
-  # is_struct/1 appears in Elixir 1.11
-  def stringify_keys(%{__struct__: _} = object) do
+  def stringify_keys(object) when is_struct(object) do
     object
     |> Map.from_struct()
     |> stringify_keys
@@ -236,4 +278,54 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do
     Object.normalize(object["object"], fetch: true)
     :ok
   end
+
+  defp for_each_history_item(
+         %{"type" => "OrderedCollection", "orderedItems" => items} = history,
+         object,
+         fun
+       ) do
+    processed_items =
+      Enum.map(items, fn item ->
+        with item <- Map.put(item, "id", object["id"]),
+             {:ok, item} <- fun.(item) do
+          item
+        else
+          _ -> nil
+        end
+      end)
+
+    if Enum.all?(processed_items, &(not is_nil(&1))) do
+      {:ok, Map.put(history, "orderedItems", processed_items)}
+    else
+      {:error, :invalid_history}
+    end
+  end
+
+  defp for_each_history_item(nil, _object, _fun) do
+    {:ok, nil}
+  end
+
+  defp for_each_history_item(_, _object, _fun) do
+    {:error, :invalid_history}
+  end
+
+  # fun is (object -> {:ok, validated_object_with_string_keys})
+  defp do_separate_with_history(object, fun) do
+    with history <- object["formerRepresentations"],
+         object <- Map.drop(object, ["formerRepresentations"]),
+         {_, {:ok, object}} <- {:main_body, fun.(object)},
+         {_, {:ok, history}} <- {:history_items, for_each_history_item(history, object, fun)} do
+      object =
+        if history do
+          Map.put(object, "formerRepresentations", history)
+        else
+          object
+        end
+
+      {:ok, object}
+    else
+      {:main_body, e} -> e
+      {:history_items, e} -> e
+    end
+  end
 end

lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex

@@ -49,7 +49,10 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNotePageValidator do
   defp fix_url(%{"url" => url} = data) when is_map(url), do: Map.put(data, "url", url["href"])
   defp fix_url(data), do: data
 
-  defp fix_tag(%{"tag" => tag} = data) when is_list(tag), do: data
+  defp fix_tag(%{"tag" => tag} = data) when is_list(tag) do
+    Map.put(data, "tag", Enum.filter(tag, &is_map/1))
+  end
+
   defp fix_tag(%{"tag" => tag} = data) when is_map(tag), do: Map.put(data, "tag", [tag])
   defp fix_tag(data), do: Map.drop(data, ["tag"])
 
@@ -60,11 +63,19 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNotePageValidator do
   defp fix_replies(%{"replies" => %{"items" => replies}} = data) when is_list(replies),
     do: Map.put(data, "replies", replies)
 
-  defp fix_replies(%{"replies" => replies} = data) when is_bitstring(replies),
+  # TODO: Pleroma does not have any support for Collections at the moment.
+  # If the `replies` field is not something the ObjectID validator can handle,
+  # the activity/object would be rejected, which is bad behavior.
+  defp fix_replies(%{"replies" => replies} = data) when not is_list(replies),
     do: Map.drop(data, ["replies"])
 
   defp fix_replies(data), do: data
 
+  def fix_attachments(%{"attachment" => attachment} = data) when is_map(attachment),
+    do: Map.put(data, "attachment", [attachment])
+
+  def fix_attachments(data), do: data
+
   defp fix(data) do
     data
     |> CommonFixes.fix_actor()
@@ -72,6 +83,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNotePageValidator do
     |> fix_url()
     |> fix_tag()
     |> fix_replies()
+    |> fix_attachments()
     |> Transmogrifier.fix_emoji()
     |> Transmogrifier.fix_content_map()
   end
@@ -88,7 +100,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNotePageValidator do
   defp validate_data(data_cng) do
     data_cng
     |> validate_inclusion(:type, ["Article", "Note", "Page"])
-    |> validate_required([:id, :actor, :attributedTo, :type, :context, :context_id])
+    |> validate_required([:id, :actor, :attributedTo, :type, :context])
     |> CommonValidations.validate_any_presence([:cc, :to])
     |> CommonValidations.validate_fields_match([:actor, :attributedTo])
     |> CommonValidations.validate_actor_presence()

lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex

@@ -11,15 +11,16 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator do
 
   @primary_key false
   embedded_schema do
+    field(:id, :string)
     field(:type, :string)
-    field(:mediaType, :string, default: "application/octet-stream")
+    field(:mediaType, ObjectValidators.MIME, default: "application/octet-stream")
     field(:name, :string)
     field(:blurhash, :string)
 
     embeds_many :url, UrlObjectValidator, primary_key: false do
       field(:type, :string)
       field(:href, ObjectValidators.Uri)
-      field(:mediaType, :string, default: "application/octet-stream")
+      field(:mediaType, ObjectValidators.MIME, default: "application/octet-stream")
       field(:width, :integer)
       field(:height, :integer)
     end
@@ -43,10 +44,10 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator do
       |> fix_url()
 
     struct
-    |> cast(data, [:type, :mediaType, :name, :blurhash])
-    |> cast_embed(:url, with: &url_changeset/2)
+    |> cast(data, [:id, :type, :mediaType, :name, :blurhash])
+    |> cast_embed(:url, with: &url_changeset/2, required: true)
     |> validate_inclusion(:type, ~w[Link Document Audio Image Video])
-    |> validate_required([:type, :mediaType, :url])
+    |> validate_required([:type, :mediaType])
   end
 
   def url_changeset(struct, data) do
@@ -59,13 +60,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator do
   end
 
   def fix_media_type(data) do
-    data = Map.put_new(data, "mediaType", data["mimeType"])
-
-    if is_bitstring(data["mediaType"]) && MIME.extensions(data["mediaType"]) != [] do
-      data
-    else
-      Map.put(data, "mediaType", "application/octet-stream")
-    end
+    Map.put_new(data, "mediaType", data["mimeType"] || "application/octet-stream")
   end
 
   defp handle_href(href, mediaType, data) do
@@ -96,6 +91,6 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator do
   defp validate_data(cng) do
     cng
     |> validate_inclusion(:type, ~w[Document Audio Image Video])
-    |> validate_required([:mediaType, :url, :type])
+    |> validate_required([:mediaType, :type])
   end
 end

lib/pleroma/web/activity_pub/object_validators/audio_video_validator.ex

@@ -104,14 +104,14 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AudioVideoValidator do
 
     struct
     |> cast(data, __schema__(:fields) -- [:attachment, :tag])
-    |> cast_embed(:attachment)
+    |> cast_embed(:attachment, required: true)
     |> cast_embed(:tag)
   end
 
   defp validate_data(data_cng) do
     data_cng
     |> validate_inclusion(:type, ["Audio", "Video"])
-    |> validate_required([:id, :actor, :attributedTo, :type, :context, :attachment])
+    |> validate_required([:id, :actor, :attributedTo, :type, :context])
     |> CommonValidations.validate_any_presence([:cc, :to])
     |> CommonValidations.validate_fields_match([:actor, :attributedTo])
     |> CommonValidations.validate_actor_presence()

lib/pleroma/web/activity_pub/object_validators/common_fields.ex

@@ -33,6 +33,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonFields do
       field(:content, :string)
 
       field(:published, ObjectValidators.DateTime)
+      field(:updated, ObjectValidators.DateTime)
       field(:emoji, ObjectValidators.Emoji, default: %{})
       embeds_many(:attachment, AttachmentValidator)
     end
@@ -51,8 +52,6 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonFields do
       field(:summary, :string)
 
       field(:context, :string)
-      # short identifier for PleromaFE to group statuses by context
-      field(:context_id, :integer)
 
       field(:sensitive, :boolean, default: false)
       field(:replies_count, :integer, default: 0)

lib/pleroma/web/activity_pub/object_validators/common_fixes.ex

@@ -22,14 +22,15 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes do
   end
 
   def fix_object_defaults(data) do
-    %{data: %{"id" => context}, id: context_id} =
-      Utils.create_context(data["context"] || data["conversation"])
+    context =
+      Utils.maybe_create_context(
+        data["context"] || data["conversation"] || data["inReplyTo"] || data["id"]
+      )
 
     %User{follower_address: follower_collection} = User.get_cached_by_ap_id(data["attributedTo"])
 
     data
     |> Map.put("context", context)
-    |> Map.put("context_id", context_id)
     |> cast_and_filter_recipients("to", follower_collection)
     |> cast_and_filter_recipients("cc", follower_collection)
     |> cast_and_filter_recipients("bto", follower_collection)

lib/pleroma/web/activity_pub/object_validators/common_validations.ex

@@ -136,11 +136,11 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations do
 
   # This figures out if a user is able to create, delete or modify something
   # based on the domain and superuser status
-  @spec validate_modification_rights(Ecto.Changeset.t()) :: Ecto.Changeset.t()
-  def validate_modification_rights(cng) do
+  @spec validate_modification_rights(Ecto.Changeset.t(), atom()) :: Ecto.Changeset.t()
+  def validate_modification_rights(cng, privilege) do
     actor = User.get_cached_by_ap_id(get_field(cng, :actor))
 
-    if User.superuser?(actor) || same_domain?(cng) do
+    if User.privileged?(actor, privilege) || same_domain?(cng) do
       cng
     else
       cng

lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex

@@ -75,7 +75,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateGenericValidator do
 
     data
     |> CommonFixes.fix_actor()
-    |> Map.put_new("context", object["context"])
+    |> Map.put("context", object["context"])
     |> fix_addressing(object)
   end
 

lib/pleroma/web/activity_pub/object_validators/delete_validator.ex

@@ -61,7 +61,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.DeleteValidator do
     |> validate_required([:id, :type, :actor, :to, :cc, :object])
     |> validate_inclusion(:type, ["Delete"])
     |> validate_delete_actor(:actor)
-    |> validate_modification_rights()
+    |> validate_modification_rights(:messages_delete)
     |> validate_object_or_user_presence(allowed_types: @deletable_types)
     |> add_deleted_activity_id()
   end

lib/pleroma/web/activity_pub/object_validators/emoji_react_validator.ex

@@ -49,6 +49,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.EmojiReactValidator do
   defp fix(data) do
     data =
       data
+      |> fix_emoji_qualification()
       |> CommonFixes.fix_actor()
       |> CommonFixes.fix_activity_addressing()
 
@@ -61,6 +62,23 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.EmojiReactValidator do
     end
   end
 
+  defp fix_emoji_qualification(%{"content" => emoji} = data) do
+    new_emoji = Pleroma.Emoji.fully_qualify_emoji(emoji)
+
+    cond do
+      Pleroma.Emoji.is_unicode_emoji?(emoji) ->
+        data
+
+      Pleroma.Emoji.is_unicode_emoji?(new_emoji) ->
+        data |> Map.put("content", new_emoji)
+
+      true ->
+        data
+    end
+  end
+
+  defp fix_emoji_qualification(data), do: data
+
   defp validate_emoji(cng) do
     content = get_field(cng, :content)
 

lib/pleroma/web/activity_pub/object_validators/event_validator.ex

@@ -62,7 +62,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.EventValidator do
   defp validate_data(data_cng) do
     data_cng
     |> validate_inclusion(:type, ["Event"])
-    |> validate_required([:id, :actor, :attributedTo, :type, :context, :context_id])
+    |> validate_required([:id, :actor, :attributedTo, :type, :context])
     |> CommonValidations.validate_any_presence([:cc, :to])
     |> CommonValidations.validate_fields_match([:actor, :attributedTo])
     |> CommonValidations.validate_actor_presence()

lib/pleroma/web/activity_pub/object_validators/question_validator.ex

@@ -80,7 +80,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.QuestionValidator do
   defp validate_data(data_cng) do
     data_cng
     |> validate_inclusion(:type, ["Question"])
-    |> validate_required([:id, :actor, :attributedTo, :type, :context, :context_id])
+    |> validate_required([:id, :actor, :attributedTo, :type, :context])
     |> CommonValidations.validate_any_presence([:cc, :to])
     |> CommonValidations.validate_fields_match([:actor, :attributedTo])
     |> CommonValidations.validate_actor_presence()

lib/pleroma/web/activity_pub/object_validators/update_validator.ex

@@ -51,7 +51,9 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.UpdateValidator do
     with actor = get_field(cng, :actor),
          object = get_field(cng, :object),
          {:ok, object_id} <- ObjectValidators.ObjectID.cast(object),
-         true <- actor == object_id do
+         actor_uri <- URI.parse(actor),
+         object_uri <- URI.parse(object_id),
+         true <- actor_uri.host == object_uri.host do
       cng
     else
       _e ->

lib/pleroma/web/activity_pub/side_effects.ex

@@ -25,6 +25,7 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do
   alias Pleroma.Web.Streamer
   alias Pleroma.Workers.PollWorker
 
+  require Pleroma.Constants
   require Logger
 
   @cachex Pleroma.Config.get([:cachex, :provider], Cachex)
@@ -153,23 +154,26 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do
 
   # Tasks this handles:
   # - Update the user
+  # - Update a non-user object (Note, Question, etc.)
   #
   # For a local user, we also get a changeset with the full information, so we
   # can update non-federating, non-activitypub settings as well.
   @impl true
   def handle(%{data: %{"type" => "Update", "object" => updated_object}} = object, meta) do
-    if changeset = Keyword.get(meta, :user_update_changeset) do
-      changeset
-      |> User.update_and_set_cache()
+    updated_object_id = updated_object["id"]
+
+    with {_, true} <- {:has_id, is_binary(updated_object_id)},
+         %{"type" => type} <- updated_object,
+         {_, is_user} <- {:is_user, type in Pleroma.Constants.actor_types()} do
+      if is_user do
+        handle_update_user(object, meta)
+      else
+        handle_update_object(object, meta)
+      end
     else
-      {:ok, new_user_data} = ActivityPub.user_data_from_user_object(updated_object)
-
-      User.get_by_ap_id(updated_object["id"])
-      |> User.remote_user_changeset(new_user_data)
-      |> User.update_and_set_cache()
+      _ ->
+        {:ok, object, meta}
     end
-
-    {:ok, object, meta}
   end
 
   # Tasks this handles:
@@ -278,7 +282,6 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do
   # Tasks this handles:
   # - Delete and unpins the create activity
   # - Replace object with Tombstone
-  # - Set up notification
   # - Reduce the user note count
   # - Reduce the reply count
   # - Stream out the activity
@@ -320,7 +323,6 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do
       end
 
     if result == :ok do
-      Notification.create_notifications(object)
       {:ok, object, meta}
     else
       {:error, result}
@@ -390,6 +392,79 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do
     {:ok, object, meta}
   end
 
+  defp handle_update_user(
+         %{data: %{"type" => "Update", "object" => updated_object}} = object,
+         meta
+       ) do
+    if changeset = Keyword.get(meta, :user_update_changeset) do
+      changeset
+      |> User.update_and_set_cache()
+    else
+      {:ok, new_user_data} = ActivityPub.user_data_from_user_object(updated_object)
+
+      User.get_by_ap_id(updated_object["id"])
+      |> User.remote_user_changeset(new_user_data)
+      |> User.update_and_set_cache()
+    end
+
+    {:ok, object, meta}
+  end
+
+  defp handle_update_object(
+         %{data: %{"type" => "Update", "object" => updated_object}} = object,
+         meta
+       ) do
+    orig_object_ap_id = updated_object["id"]
+    orig_object = Object.get_by_ap_id(orig_object_ap_id)
+    orig_object_data = orig_object.data
+
+    updated_object =
+      if meta[:local] do
+        # If this is a local Update, we don't process it by transmogrifier,
+        # so we use the embedded object as-is.
+        updated_object
+      else
+        meta[:object_data]
+      end
+
+    if orig_object_data["type"] in Pleroma.Constants.updatable_object_types() do
+      %{
+        updated_data: updated_object_data,
+        updated: updated,
+        used_history_in_new_object?: used_history_in_new_object?
+      } = Object.Updater.make_new_object_data_from_update_object(orig_object_data, updated_object)
+
+      changeset =
+        orig_object
+        |> Repo.preload(:hashtags)
+        |> Object.change(%{data: updated_object_data})
+
+      with {:ok, new_object} <- Repo.update(changeset),
+           {:ok, _} <- Object.invalid_object_cache(new_object),
+           {:ok, _} <- Object.set_cache(new_object),
+           # The metadata/utils.ex uses the object id for the cache.
+           {:ok, _} <- Pleroma.Activity.HTML.invalidate_cache_for(new_object.id) do
+        if used_history_in_new_object? do
+          with create_activity when not is_nil(create_activity) <-
+                 Pleroma.Activity.get_create_by_object_ap_id(orig_object_ap_id),
+               {:ok, _} <- Pleroma.Activity.HTML.invalidate_cache_for(create_activity.id) do
+            nil
+          else
+            _ -> nil
+          end
+        end
+
+        if updated do
+          object
+          |> Activity.normalize()
+          |> ActivityPub.notify_and_stream()
+        end
+      end
+    end
+
+    {:ok, object, meta}
+  end
+
   def handle_object_creation(%{"type" => "ChatMessage"} = object, _activity, meta) do
     with {:ok, object, meta} <- Pipeline.common_pipeline(object, meta) do
       actor = User.get_cached_by_ap_id(object.data["actor"])

lib/pleroma/web/activity_pub/transmogrifier.ex

@@ -203,13 +203,13 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
 
         media_type =
           cond do
-            is_map(url) && MIME.extensions(url["mediaType"]) != [] ->
+            is_map(url) && url =~ Pleroma.Constants.mime_regex() ->
               url["mediaType"]
 
-            is_bitstring(data["mediaType"]) && MIME.extensions(data["mediaType"]) != [] ->
+            is_bitstring(data["mediaType"]) && data["mediaType"] =~ Pleroma.Constants.mime_regex() ->
               data["mediaType"]
 
-            is_bitstring(data["mimeType"]) && MIME.extensions(data["mimeType"]) != [] ->
+            is_bitstring(data["mimeType"]) && data["mimeType"] =~ Pleroma.Constants.mime_regex() ->
               data["mimeType"]
 
             true ->
@@ -687,6 +687,24 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
     |> strip_internal_fields
     |> strip_internal_tags
     |> set_type
+    |> maybe_process_history
+  end
+
+  defp maybe_process_history(%{"formerRepresentations" => %{"orderedItems" => history}} = object) do
+    processed_history =
+      Enum.map(
+        history,
+        fn
+          item when is_map(item) -> prepare_object(item)
+          item -> item
+        end
+      )
+
+    put_in(object, ["formerRepresentations", "orderedItems"], processed_history)
+  end
+
+  defp maybe_process_history(object) do
+    object
   end
 
   #  @doc
@@ -711,6 +729,21 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
     {:ok, data}
   end
 
+  def prepare_outgoing(%{"type" => "Update", "object" => %{"type" => objtype} = object} = data)
+      when objtype in Pleroma.Constants.updatable_object_types() do
+    object =
+      object
+      |> prepare_object
+
+    data =
+      data
+      |> Map.put("object", object)
+      |> Map.merge(Utils.make_json_ld_header())
+      |> Map.delete("bcc")
+
+    {:ok, data}
+  end
+
   def prepare_outgoing(%{"type" => "Announce", "actor" => ap_id, "object" => object_id} = data) do
     object =
       object_id

lib/pleroma/web/activity_pub/utils.ex

@@ -154,22 +154,7 @@ defmodule Pleroma.Web.ActivityPub.Utils do
     Notification.get_notified_from_activity(%Activity{data: object}, false)
   end
 
-  def create_context(context) do
-    context = context || generate_id("contexts")
-
-    # Ecto has problems accessing the constraint inside the jsonb,
-    # so we explicitly check for the existed object before insert
-    object = Object.get_cached_by_ap_id(context)
-
-    with true <- is_nil(object),
-         changeset <- Object.context_mapping(context),
-         {:ok, inserted_object} <- Repo.insert(changeset) do
-      inserted_object
-    else
-      _ ->
-        object
-    end
-  end
+  def maybe_create_context(context), do: context || generate_id("contexts")
 
   @doc """
   Enqueues an activity for federation if it's local
@@ -201,18 +186,16 @@ defmodule Pleroma.Web.ActivityPub.Utils do
     |> Map.put_new("id", "pleroma:fakeid")
     |> Map.put_new_lazy("published", &make_date/0)
     |> Map.put_new("context", "pleroma:fakecontext")
-    |> Map.put_new("context_id", -1)
     |> lazy_put_object_defaults(true)
   end
 
   def lazy_put_activity_defaults(map, _fake?) do
-    %{data: %{"id" => context}, id: context_id} = create_context(map["context"])
+    context = maybe_create_context(map["context"])
 
     map
     |> Map.put_new_lazy("id", &generate_activity_id/0)
     |> Map.put_new_lazy("published", &make_date/0)
     |> Map.put_new("context", context)
-    |> Map.put_new("context_id", context_id)
     |> lazy_put_object_defaults(false)
   end
 
@@ -226,7 +209,6 @@ defmodule Pleroma.Web.ActivityPub.Utils do
       |> Map.put_new("id", "pleroma:fake_object_id")
       |> Map.put_new_lazy("published", &make_date/0)
       |> Map.put_new("context", activity["context"])
-      |> Map.put_new("context_id", activity["context_id"])
       |> Map.put_new("fake", true)
 
     %{activity | "object" => object}
@@ -239,7 +221,6 @@ defmodule Pleroma.Web.ActivityPub.Utils do
       |> Map.put_new_lazy("id", &generate_object_id/0)
       |> Map.put_new_lazy("published", &make_date/0)
       |> Map.put_new("context", activity["context"])
-      |> Map.put_new("context_id", activity["context_id"])
 
     %{activity | "object" => object}
   end
@@ -714,20 +695,24 @@ defmodule Pleroma.Web.ActivityPub.Utils do
     Enum.map(statuses || [], &build_flag_object/1)
   end
 
-  defp build_flag_object(%Activity{data: %{"id" => id}, object: %{data: data}}) do
-    activity_actor = User.get_by_ap_id(data["actor"])
+  defp build_flag_object(%Activity{} = activity) do
+    object = Object.normalize(activity, fetch: false)
 
-    %{
-      "type" => "Note",
-      "id" => id,
-      "content" => data["content"],
-      "published" => data["published"],
-      "actor" =>
-        AccountView.render(
-          "show.json",
-          %{user: activity_actor, skip_visibility_check: true}
-        )
-    }
+    # Do not allow people to report Creates. Instead, report the Object that is Created.
+    if activity.data["type"] != "Create" do
+      build_flag_object_with_actor_and_id(
+        object,
+        User.get_by_ap_id(activity.data["actor"]),
+        activity.data["id"]
+      )
+    else
+      build_flag_object(object)
+    end
+  end
+
+  defp build_flag_object(%Object{} = object) do
+    actor = User.get_by_ap_id(object.data["actor"])
+    build_flag_object_with_actor_and_id(object, actor, object.data["id"])
   end
 
   defp build_flag_object(act) when is_map(act) or is_binary(act) do
@@ -739,12 +724,12 @@ defmodule Pleroma.Web.ActivityPub.Utils do
       end
 
     case Activity.get_by_ap_id_with_object(id) do
-      %Activity{} = activity ->
-        build_flag_object(activity)
+      %Activity{object: object} = _ ->
+        build_flag_object(object)
 
       nil ->
-        if activity = Activity.get_by_object_ap_id_with_object(id) do
-          build_flag_object(activity)
+        if %Object{} = object = Object.get_by_ap_id(id) do
+          build_flag_object(object)
         else
           %{"id" => id, "deleted" => true}
         end
@@ -753,6 +738,20 @@ defmodule Pleroma.Web.ActivityPub.Utils do
 
   defp build_flag_object(_), do: []
 
+  defp build_flag_object_with_actor_and_id(%Object{data: data}, actor, id) do
+    %{
+      "type" => "Note",
+      "id" => id,
+      "content" => data["content"],
+      "published" => data["published"],
+      "actor" =>
+        AccountView.render(
+          "show.json",
+          %{user: actor, skip_visibility_check: true}
+        )
+    }
+  end
+
   #### Report-related helpers
   def get_reports(params, page, page_size) do
     params =
@@ -767,22 +766,21 @@ defmodule Pleroma.Web.ActivityPub.Utils do
     ActivityPub.fetch_activities([], params, :offset)
   end
 
-  def update_report_state(%Activity{} = activity, state)
-      when state in @strip_status_report_states do
-    {:ok, stripped_activity} = strip_report_status_data(activity)
-
-    new_data =
-      activity.data
-      |> Map.put("state", state)
-      |> Map.put("object", stripped_activity.data["object"])
-
-    activity
-    |> Changeset.change(data: new_data)
-    |> Repo.update()
+  defp maybe_strip_report_status(data, state) do
+    with true <- Config.get([:instance, :report_strip_status]),
+         true <- state in @strip_status_report_states,
+         {:ok, stripped_activity} = strip_report_status_data(%Activity{data: data}) do
+      data |> Map.put("object", stripped_activity.data["object"])
+    else
+      _ -> data
+    end
   end
 
   def update_report_state(%Activity{} = activity, state) when state in @supported_report_states do
-    new_data = Map.put(activity.data, "state", state)
+    new_data =
+      activity.data
+      |> Map.put("state", state)
+      |> maybe_strip_report_status(state)
 
     activity
     |> Changeset.change(data: new_data)

lib/pleroma/web/activity_pub/views/object_view.ex

@@ -29,11 +29,11 @@ defmodule Pleroma.Web.ActivityPub.ObjectView do
 
   def render("object.json", %{object: %Activity{} = activity}) do
     base = Pleroma.Web.ActivityPub.Utils.make_json_ld_header()
-    object = Object.normalize(activity, fetch: false)
+    object_id = Object.normalize(activity, id_only: true)
 
     additional =
       Transmogrifier.prepare_object(activity.data)
-      |> Map.put("object", object.data["id"])
+      |> Map.put("object", object_id)
 
     Map.merge(base, additional)
   end

lib/pleroma/web/activity_pub/views/user_view.ex

@@ -34,7 +34,6 @@ defmodule Pleroma.Web.ActivityPub.UserView do
   def render("endpoints.json", _), do: %{}
 
   def render("service.json", %{user: user}) do
-    {:ok, user} = User.ensure_keys_present(user)
     {:ok, _, public_key} = Keys.keys_from_pem(user.keys)
     public_key = :public_key.pem_entry_encode(:SubjectPublicKeyInfo, public_key)
     public_key = :public_key.pem_encode([public_key])
@@ -71,7 +70,6 @@ defmodule Pleroma.Web.ActivityPub.UserView do
     do: render("service.json", %{user: user}) |> Map.put("preferredUsername", user.nickname)
 
   def render("user.json", %{user: user}) do
-    {:ok, user} = User.ensure_keys_present(user)
     {:ok, _, public_key} = Keys.keys_from_pem(user.keys)
     public_key = :public_key.pem_entry_encode(:SubjectPublicKeyInfo, public_key)
     public_key = :public_key.pem_encode([public_key])

lib/pleroma/web/activity_pub/visibility.ex

@@ -84,7 +84,10 @@ defmodule Pleroma.Web.ActivityPub.Visibility do
       when module in [Activity, Object] do
     x = [user.ap_id | User.following(user)]
     y = [message.data["actor"]] ++ message.data["to"] ++ (message.data["cc"] || [])
-    is_public?(message) || Enum.any?(x, &(&1 in y))
+
+    user_is_local = user.local
+    federatable = not is_local_public?(message)
+    (is_public?(message) || Enum.any?(x, &(&1 in y))) and (user_is_local || federatable)
   end
 
   def entire_thread_visible_for_user?(%Activity{} = activity, %User{} = user) do

lib/pleroma/web/admin_api/controllers/announcement_controller.ex

@@ -0,0 +1,83 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.AdminAPI.AnnouncementController do
+  use Pleroma.Web, :controller
+
+  alias Pleroma.Announcement
+  alias Pleroma.Web.ControllerHelper
+  alias Pleroma.Web.Plugs.OAuthScopesPlug
+
+  plug(Pleroma.Web.ApiSpec.CastAndValidate)
+  plug(OAuthScopesPlug, %{scopes: ["admin:write"]} when action in [:create, :delete, :change])
+  plug(OAuthScopesPlug, %{scopes: ["admin:read"]} when action in [:index, :show])
+  action_fallback(Pleroma.Web.AdminAPI.FallbackController)
+
+  defdelegate open_api_operation(action), to: Pleroma.Web.ApiSpec.Admin.AnnouncementOperation
+
+  defp default_limit, do: 20
+
+  def index(conn, params) do
+    limit = Map.get(params, :limit, default_limit())
+    offset = Map.get(params, :offset, 0)
+
+    announcements = Announcement.list_paginated(%{limit: limit, offset: offset})
+
+    render(conn, "index.json", announcements: announcements)
+  end
+
+  def show(conn, %{id: id} = _params) do
+    announcement = Announcement.get_by_id(id)
+
+    if is_nil(announcement) do
+      {:error, :not_found}
+    else
+      render(conn, "show.json", announcement: announcement)
+    end
+  end
+
+  def create(%{body_params: params} = conn, _params) do
+    with {:ok, announcement} <- Announcement.add(change_params(params)) do
+      render(conn, "show.json", announcement: announcement)
+    else
+      _ ->
+        {:error, 400}
+    end
+  end
+
+  def change_params(orig_params) do
+    data =
+      %{}
+      |> Pleroma.Maps.put_if_present("content", orig_params, &Map.fetch(&1, :content))
+      |> Pleroma.Maps.put_if_present("all_day", orig_params, &Map.fetch(&1, :all_day))
+
+    orig_params
+    |> Map.merge(%{data: data})
+  end
+
+  def change(%{body_params: params} = conn, %{id: id} = _params) do
+    with announcement <- Announcement.get_by_id(id),
+         {:exists, true} <- {:exists, not is_nil(announcement)},
+         {:ok, announcement} <- Announcement.update(announcement, change_params(params)) do
+      render(conn, "show.json", announcement: announcement)
+    else
+      {:exists, false} ->
+        {:error, :not_found}
+
+      _ ->
+        {:error, 400}
+    end
+  end
+
+  def delete(conn, %{id: id} = _params) do
+    case Announcement.delete_by_id(id) do
+      :ok ->
+        conn
+        |> ControllerHelper.json_response(:ok, %{})
+
+      _ ->
+        {:error, :not_found}
+    end
+  end
+end

lib/pleroma/web/admin_api/controllers/chat_controller.ex

@@ -8,7 +8,6 @@ defmodule Pleroma.Web.AdminAPI.ChatController do
   alias Pleroma.Activity
   alias Pleroma.Chat
   alias Pleroma.Chat.MessageReference
-  alias Pleroma.ModerationLog
   alias Pleroma.Pagination
   alias Pleroma.Web.AdminAPI
   alias Pleroma.Web.CommonAPI
@@ -42,12 +41,6 @@ defmodule Pleroma.Web.AdminAPI.ChatController do
          ^chat_id <- to_string(cm_ref.chat_id),
          %Activity{id: activity_id} <- Activity.get_create_by_object_ap_id(object_ap_id),
          {:ok, _} <- CommonAPI.delete(activity_id, user) do
-      ModerationLog.insert_log(%{
-        action: "chat_message_delete",
-        actor: user,
-        subject_id: message_id
-      })
-
       conn
       |> put_view(MessageReferenceView)
       |> render("show.json", chat_message_reference: cm_ref)

lib/pleroma/web/admin_api/controllers/config_controller.ex

@@ -22,10 +22,58 @@ defmodule Pleroma.Web.AdminAPI.ConfigController do
 
   defdelegate open_api_operation(action), to: Pleroma.Web.ApiSpec.Admin.ConfigOperation
 
+  defp translate_descriptions(descriptions, path \\ []) do
+    Enum.map(descriptions, fn desc -> translate_item(desc, path) end)
+  end
+
+  defp translate_string(str, path, type) do
+    Gettext.dpgettext(
+      Pleroma.Web.Gettext,
+      "config_descriptions",
+      Pleroma.Docs.Translator.Compiler.msgctxt_for(path, type),
+      str
+    )
+  end
+
+  defp maybe_put_translated(item, key, path) do
+    if item[key] do
+      Map.put(
+        item,
+        key,
+        translate_string(
+          item[key],
+          path ++ [Pleroma.Docs.Translator.Compiler.key_for(item)],
+          to_string(key)
+        )
+      )
+    else
+      item
+    end
+  end
+
+  defp translate_item(item, path) do
+    item
+    |> maybe_put_translated(:label, path)
+    |> maybe_put_translated(:description, path)
+    |> translate_children(path)
+  end
+
+  defp translate_children(%{children: children} = item, path) when is_list(children) do
+    item
+    |> Map.put(
+      :children,
+      translate_descriptions(children, path ++ [Pleroma.Docs.Translator.Compiler.key_for(item)])
+    )
+  end
+
+  defp translate_children(item, _path) do
+    item
+  end
+
   def descriptions(conn, _params) do
     descriptions = Enum.filter(Pleroma.Docs.JSON.compiled_descriptions(), &whitelisted_config?/1)
 
-    json(conn, descriptions)
+    json(conn, translate_descriptions(descriptions))
   end
 
   def show(conn, %{only_db: true}) do

lib/pleroma/web/admin_api/controllers/status_controller.ex

@@ -65,12 +65,6 @@ defmodule Pleroma.Web.AdminAPI.StatusController do
 
   def delete(%{assigns: %{user: user}} = conn, %{id: id}) do
     with {:ok, %Activity{}} <- CommonAPI.delete(id, user) do
-      ModerationLog.insert_log(%{
-        action: "status_delete",
-        actor: user,
-        subject_id: id
-      })
-
       json(conn, %{})
     end
   end

lib/pleroma/web/admin_api/report.ex

@@ -4,6 +4,7 @@
 
 defmodule Pleroma.Web.AdminAPI.Report do
   alias Pleroma.Activity
+  alias Pleroma.Object
   alias Pleroma.User
 
   def extract_report_info(
@@ -16,10 +17,44 @@ defmodule Pleroma.Web.AdminAPI.Report do
       status_ap_ids
       |> Enum.reject(&is_nil(&1))
       |> Enum.map(fn
-        act when is_map(act) -> Activity.get_by_ap_id_with_object(act["id"])
-        act when is_binary(act) -> Activity.get_by_ap_id_with_object(act)
+        act when is_map(act) ->
+          Activity.get_create_by_object_ap_id_with_object(act["id"]) ||
+            Activity.get_by_ap_id_with_object(act["id"]) || make_fake_activity(act, user)
+
+        act when is_binary(act) ->
+          Activity.get_create_by_object_ap_id_with_object(act) ||
+            Activity.get_by_ap_id_with_object(act)
       end)
 
     %{report: report, user: user, account: account, statuses: statuses}
   end
+
+  defp make_fake_activity(act, user) do
+    %Activity{
+      id: "pleroma:fake:#{act["id"]}",
+      data: %{
+        "actor" => user.ap_id,
+        "type" => "Create",
+        "to" => [],
+        "cc" => [],
+        "object" => act["id"],
+        "published" => act["published"],
+        "id" => act["id"],
+        "context" => "pleroma:fake"
+      },
+      recipients: [user.ap_id],
+      object: %Object{
+        data: %{
+          "actor" => user.ap_id,
+          "type" => "Note",
+          "content" => act["content"],
+          "published" => act["published"],
+          "to" => [],
+          "cc" => [],
+          "id" => act["id"],
+          "context" => "pleroma:fake"
+        }
+      }
+    }
+  end
 end

lib/pleroma/web/admin_api/views/announcement_view.ex

@@ -0,0 +1,15 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.AdminAPI.AnnouncementView do
+  use Pleroma.Web, :view
+
+  def render("index.json", %{announcements: announcements}) do
+    render_many(announcements, __MODULE__, "show.json")
+  end
+
+  def render("show.json", %{announcement: announcement}) do
+    Pleroma.Announcement.render_json(announcement, admin: true)
+  end
+end

lib/pleroma/web/api_spec.ex

@@ -95,7 +95,8 @@ defmodule Pleroma.Web.ApiSpec do
               "Relays",
               "Report managment",
               "Status administration",
-              "User administration"
+              "User administration",
+              "Announcement management"
             ]
           },
           %{"name" => "Applications", "tags" => ["Applications", "Push subscriptions"]},
@@ -110,10 +111,12 @@ defmodule Pleroma.Web.ApiSpec do
               "Follow requests",
               "Mascot",
               "Markers",
-              "Notifications"
+              "Notifications",
+              "Filters",
+              "Settings"
             ]
           },
-          %{"name" => "Instance", "tags" => ["Custom emojis"]},
+          %{"name" => "Instance", "tags" => ["Custom emojis", "Instance misc"]},
           %{"name" => "Messaging", "tags" => ["Chats", "Conversations"]},
           %{
             "name" => "Statuses",
@@ -125,10 +128,21 @@ defmodule Pleroma.Web.ApiSpec do
               "Retrieve status information",
               "Scheduled statuses",
               "Search",
-              "Status actions"
+              "Status actions",
+              "Media attachments"
             ]
           },
-          %{"name" => "Miscellaneous", "tags" => ["Emoji packs", "Reports", "Suggestions"]}
+          %{
+            "name" => "Miscellaneous",
+            "tags" => [
+              "Emoji packs",
+              "Reports",
+              "Suggestions",
+              "Announcements",
+              "Remote interaction",
+              "Others"
+            ]
+          }
         ]
       }
     }

lib/pleroma/web/api_spec/operations/account_operation.ex

@@ -64,7 +64,8 @@ defmodule Pleroma.Web.ApiSpec.AccountOperation do
       requestBody: request_body("Parameters", update_credentials_request(), required: true),
       responses: %{
         200 => Operation.response("Account", "application/json", Account),
-        403 => Operation.response("Error", "application/json", ApiError)
+        403 => Operation.response("Error", "application/json", ApiError),
+        413 => Operation.response("Error", "application/json", ApiError)
       }
     }
   end
@@ -223,12 +224,12 @@ defmodule Pleroma.Web.ApiSpec.AccountOperation do
             type: :object,
             properties: %{
               reblogs: %Schema{
-                type: :boolean,
+                allOf: [BooleanLike],
                 description: "Receive this account's reblogs in home timeline? Defaults to true.",
                 default: true
               },
               notify: %Schema{
-                type: :boolean,
+                allOf: [BooleanLike],
                 description:
                   "Receive notifications for all statuses posted by the account? Defaults to false.",
                 default: false
@@ -278,11 +279,17 @@ defmodule Pleroma.Web.ApiSpec.AccountOperation do
           %Schema{allOf: [BooleanLike], default: true},
           "Mute notifications in addition to statuses? Defaults to `true`."
         ),
+        Operation.parameter(
+          :duration,
+          :query,
+          %Schema{type: :integer},
+          "Expire the mute in `duration` seconds. Default 0 for infinity"
+        ),
         Operation.parameter(
           :expires_in,
           :query,
           %Schema{type: :integer, default: 0},
-          "Expire the mute in `expires_in` seconds. Default 0 for infinity"
+          "Deprecated, use `duration` instead"
         )
       ],
       responses: %{
@@ -370,6 +377,22 @@ defmodule Pleroma.Web.ApiSpec.AccountOperation do
     }
   end
 
+  def remove_from_followers_operation do
+    %Operation{
+      tags: ["Account actions"],
+      summary: "Remove from followers",
+      operationId: "AccountController.remove_from_followers",
+      security: [%{"oAuth" => ["follow", "write:follows"]}],
+      description: "Remove the given account from followers",
+      parameters: [%Reference{"$ref": "#/components/parameters/accountIdOrNickname"}],
+      responses: %{
+        200 => Operation.response("Relationship", "application/json", AccountRelationship),
+        400 => Operation.response("Error", "application/json", ApiError),
+        404 => Operation.response("Error", "application/json", ApiError)
+      }
+    }
+  end
+
   def note_operation do
     %Operation{
       tags: ["Account actions"],
@@ -438,7 +461,7 @@ defmodule Pleroma.Web.ApiSpec.AccountOperation do
 
   def lookup_operation do
     %Operation{
-      tags: ["Account lookup"],
+      tags: ["Retrieve account information"],
       summary: "Find a user by nickname",
       operationId: "AccountController.lookup",
       parameters: [
@@ -545,10 +568,18 @@ defmodule Pleroma.Web.ApiSpec.AccountOperation do
           description: "Invite token required when the registrations aren't public"
         },
         birthday: %Schema{
-          type: :string,
           nullable: true,
           description: "User's birthday",
-          format: :date
+          anyOf: [
+            %Schema{
+              type: :string,
+              format: :date
+            },
+            %Schema{
+              type: :string,
+              maxLength: 0
+            }
+          ]
         },
         language: %Schema{
           type: :string,
@@ -733,10 +764,18 @@ defmodule Pleroma.Web.ApiSpec.AccountOperation do
         },
         actor_type: ActorType,
         birthday: %Schema{
-          type: :string,
           nullable: true,
           description: "User's birthday",
-          format: :date
+          anyOf: [
+            %Schema{
+              type: :string,
+              format: :date
+            },
+            %Schema{
+              type: :string,
+              maxLength: 0
+            }
+          ]
         },
         show_birthday: %Schema{
           allOf: [BooleanLike],
@@ -861,10 +900,15 @@ defmodule Pleroma.Web.ApiSpec.AccountOperation do
           description: "Mute notifications in addition to statuses? Defaults to true.",
           default: true
         },
+        duration: %Schema{
+          type: :integer,
+          nullable: true,
+          description: "Expire the mute in `expires_in` seconds. Default 0 for infinity"
+        },
         expires_in: %Schema{
           type: :integer,
           nullable: true,
-          description: "Expire the mute in `expires_in` seconds. Default 0 for infinity",
+          description: "Deprecated, use `duration` instead",
           default: 0
         }
       },

lib/pleroma/web/api_spec/operations/admin/announcement_operation.ex

@@ -0,0 +1,165 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ApiSpec.Admin.AnnouncementOperation do
+  alias OpenApiSpex.Operation
+  alias OpenApiSpex.Schema
+  alias Pleroma.Web.ApiSpec.Schemas.Announcement
+  alias Pleroma.Web.ApiSpec.Schemas.ApiError
+
+  import Pleroma.Web.ApiSpec.Helpers
+
+  def open_api_operation(action) do
+    operation = String.to_existing_atom("#{action}_operation")
+    apply(__MODULE__, operation, [])
+  end
+
+  def index_operation do
+    %Operation{
+      tags: ["Announcement management"],
+      summary: "Retrieve a list of announcements",
+      operationId: "AdminAPI.AnnouncementController.index",
+      security: [%{"oAuth" => ["admin:read"]}],
+      parameters: [
+        Operation.parameter(
+          :limit,
+          :query,
+          %Schema{type: :integer, minimum: 1},
+          "the maximum number of announcements to return"
+        ),
+        Operation.parameter(
+          :offset,
+          :query,
+          %Schema{type: :integer, minimum: 0},
+          "the offset of the first announcement to return"
+        )
+        | admin_api_params()
+      ],
+      responses: %{
+        200 => Operation.response("Response", "application/json", list_of_announcements()),
+        400 => Operation.response("Forbidden", "application/json", ApiError),
+        403 => Operation.response("Forbidden", "application/json", ApiError)
+      }
+    }
+  end
+
+  def show_operation do
+    %Operation{
+      tags: ["Announcement management"],
+      summary: "Display one announcement",
+      operationId: "AdminAPI.AnnouncementController.show",
+      security: [%{"oAuth" => ["admin:read"]}],
+      parameters: [
+        Operation.parameter(
+          :id,
+          :path,
+          :string,
+          "announcement id"
+        )
+        | admin_api_params()
+      ],
+      responses: %{
+        200 => Operation.response("Response", "application/json", Announcement),
+        403 => Operation.response("Forbidden", "application/json", ApiError),
+        404 => Operation.response("Not Found", "application/json", ApiError)
+      }
+    }
+  end
+
+  def delete_operation do
+    %Operation{
+      tags: ["Announcement management"],
+      summary: "Delete one announcement",
+      operationId: "AdminAPI.AnnouncementController.delete",
+      security: [%{"oAuth" => ["admin:write"]}],
+      parameters: [
+        Operation.parameter(
+          :id,
+          :path,
+          :string,
+          "announcement id"
+        )
+        | admin_api_params()
+      ],
+      responses: %{
+        200 => Operation.response("Response", "application/json", %Schema{type: :object}),
+        403 => Operation.response("Forbidden", "application/json", ApiError),
+        404 => Operation.response("Not Found", "application/json", ApiError)
+      }
+    }
+  end
+
+  def create_operation do
+    %Operation{
+      tags: ["Announcement management"],
+      summary: "Create one announcement",
+      operationId: "AdminAPI.AnnouncementController.create",
+      security: [%{"oAuth" => ["admin:write"]}],
+      requestBody: request_body("Parameters", create_request(), required: true),
+      responses: %{
+        200 => Operation.response("Response", "application/json", Announcement),
+        400 => Operation.response("Bad Request", "application/json", ApiError),
+        403 => Operation.response("Forbidden", "application/json", ApiError)
+      }
+    }
+  end
+
+  def change_operation do
+    %Operation{
+      tags: ["Announcement management"],
+      summary: "Change one announcement",
+      operationId: "AdminAPI.AnnouncementController.change",
+      security: [%{"oAuth" => ["admin:write"]}],
+      parameters: [
+        Operation.parameter(
+          :id,
+          :path,
+          :string,
+          "announcement id"
+        )
+        | admin_api_params()
+      ],
+      requestBody: request_body("Parameters", change_request(), required: true),
+      responses: %{
+        200 => Operation.response("Response", "application/json", Announcement),
+        400 => Operation.response("Bad Request", "application/json", ApiError),
+        403 => Operation.response("Forbidden", "application/json", ApiError),
+        404 => Operation.response("Not Found", "application/json", ApiError)
+      }
+    }
+  end
+
+  defp create_or_change_props do
+    %{
+      content: %Schema{type: :string},
+      starts_at: %Schema{type: :string, format: "date-time", nullable: true},
+      ends_at: %Schema{type: :string, format: "date-time", nullable: true},
+      all_day: %Schema{type: :boolean}
+    }
+  end
+
+  def create_request do
+    %Schema{
+      title: "AnnouncementCreateRequest",
+      type: :object,
+      required: [:content],
+      properties: create_or_change_props()
+    }
+  end
+
+  def change_request do
+    %Schema{
+      title: "AnnouncementChangeRequest",
+      type: :object,
+      properties: create_or_change_props()
+    }
+  end
+
+  def list_of_announcements do
+    %Schema{
+      type: :array,
+      items: Announcement
+    }
+  end
+end

lib/pleroma/web/api_spec/operations/admin/status_operation.ex

@@ -70,7 +70,7 @@ defmodule Pleroma.Web.ApiSpec.Admin.StatusOperation do
 
   def show_operation do
     %Operation{
-      tags: ["Status adminitration)"],
+      tags: ["Status administration"],
       summary: "Get status",
       operationId: "AdminAPI.StatusController.show",
       parameters: [id_param() | admin_api_params()],
@@ -84,7 +84,7 @@ defmodule Pleroma.Web.ApiSpec.Admin.StatusOperation do
 
   def update_operation do
     %Operation{
-      tags: ["Status adminitration)"],
+      tags: ["Status administration"],
       summary: "Change the scope of a status",
       operationId: "AdminAPI.StatusController.update",
       parameters: [id_param() | admin_api_params()],
@@ -99,7 +99,7 @@ defmodule Pleroma.Web.ApiSpec.Admin.StatusOperation do
 
   def delete_operation do
     %Operation{
-      tags: ["Status adminitration)"],
+      tags: ["Status administration"],
       summary: "Delete status",
       operationId: "AdminAPI.StatusController.delete",
       parameters: [id_param() | admin_api_params()],
@@ -143,7 +143,7 @@ defmodule Pleroma.Web.ApiSpec.Admin.StatusOperation do
           }
         },
         tags: %Schema{type: :string},
-        is_confirmed: %Schema{type: :string}
+        is_confirmed: %Schema{type: :boolean}
       }
     }
   end

lib/pleroma/web/api_spec/operations/announcement_operation.ex

@@ -0,0 +1,57 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ApiSpec.AnnouncementOperation do
+  alias OpenApiSpex.Operation
+  alias OpenApiSpex.Schema
+  alias Pleroma.Web.ApiSpec.Schemas.Announcement
+  alias Pleroma.Web.ApiSpec.Schemas.ApiError
+
+  def open_api_operation(action) do
+    operation = String.to_existing_atom("#{action}_operation")
+    apply(__MODULE__, operation, [])
+  end
+
+  def index_operation do
+    %Operation{
+      tags: ["Announcements"],
+      summary: "Retrieve a list of announcements",
+      operationId: "MastodonAPI.AnnouncementController.index",
+      security: [%{"oAuth" => []}],
+      responses: %{
+        200 => Operation.response("Response", "application/json", list_of_announcements()),
+        403 => Operation.response("Forbidden", "application/json", ApiError)
+      }
+    }
+  end
+
+  def mark_read_operation do
+    %Operation{
+      tags: ["Announcements"],
+      summary: "Mark one announcement as read",
+      operationId: "MastodonAPI.AnnouncementController.mark_read",
+      security: [%{"oAuth" => ["write:accounts"]}],
+      parameters: [
+        Operation.parameter(
+          :id,
+          :path,
+          :string,
+          "announcement id"
+        )
+      ],
+      responses: %{
+        200 => Operation.response("Response", "application/json", %Schema{type: :object}),
+        403 => Operation.response("Forbidden", "application/json", ApiError),
+        404 => Operation.response("Not Found", "application/json", ApiError)
+      }
+    }
+  end
+
+  def list_of_announcements do
+    %Schema{
+      type: :array,
+      items: Announcement
+    }
+  end
+end

lib/pleroma/web/api_spec/operations/directory_operation.ex

@@ -17,7 +17,7 @@ defmodule Pleroma.Web.ApiSpec.DirectoryOperation do
 
   def index_operation do
     %Operation{
-      tags: ["Directory"],
+      tags: ["Others"],
       summary: "Profile directory",
       operationId: "DirectoryController.index",
       parameters:

lib/pleroma/web/api_spec/operations/instance_operation.ex

@@ -13,7 +13,7 @@ defmodule Pleroma.Web.ApiSpec.InstanceOperation do
 
   def show_operation do
     %Operation{
-      tags: ["Instance"],
+      tags: ["Instance misc"],
       summary: "Retrieve instance information",
       description: "Information about the server",
       operationId: "InstanceController.show",
@@ -25,7 +25,7 @@ defmodule Pleroma.Web.ApiSpec.InstanceOperation do
 
   def peers_operation do
     %Operation{
-      tags: ["Instance"],
+      tags: ["Instance misc"],
       summary: "Retrieve list of known instances",
       operationId: "InstanceController.peers",
       responses: %{

lib/pleroma/web/api_spec/operations/notification_operation.ex

@@ -51,6 +51,12 @@ defmodule Pleroma.Web.ApiSpec.NotificationOperation do
             :include_types,
             :query,
             %Schema{type: :array, items: notification_type()},
+            "Deprecated, use `types` instead"
+          ),
+          Operation.parameter(
+            :types,
+            :query,
+            %Schema{type: :array, items: notification_type()},
             "Include the notifications for activities with the given types"
           ),
           Operation.parameter(

lib/pleroma/web/api_spec/operations/pleroma_backup_operation.ex

@@ -16,7 +16,7 @@ defmodule Pleroma.Web.ApiSpec.PleromaBackupOperation do
     %Operation{
       tags: ["Backups"],
       summary: "List backups",
-      security: [%{"oAuth" => ["read:account"]}],
+      security: [%{"oAuth" => ["read:backups"]}],
       operationId: "PleromaAPI.BackupController.index",
       responses: %{
         200 =>
@@ -37,7 +37,7 @@ defmodule Pleroma.Web.ApiSpec.PleromaBackupOperation do
     %Operation{
       tags: ["Backups"],
       summary: "Create a backup",
-      security: [%{"oAuth" => ["read:account"]}],
+      security: [%{"oAuth" => ["read:backups"]}],
       operationId: "PleromaAPI.BackupController.create",
       responses: %{
         200 =>

lib/pleroma/web/api_spec/operations/pleroma_emoji_file_operation.ex

@@ -133,7 +133,11 @@ defmodule Pleroma.Web.ApiSpec.PleromaEmojiFileOperation do
   defp files_object do
     %Schema{
       type: :object,
-      additionalProperties: %Schema{type: :string},
+      additionalProperties: %Schema{
+        type: :string,
+        description: "Filename of the emoji",
+        extensions: %{"x-additionalPropertiesName": "Emoji name"}
+      },
       description: "Object with emoji names as keys and filenames as values"
     }
   end