Merge remote-tracking branch 'origin/develop' into shigusegubu

Mergeback 2.6.3

Closes #3245

See merge request pleroma/pleroma!4117

changelog.d/card-endpoint.remove

@@ -0,0 +1 @@
+Mastodon API: Remove deprecated GET /api/v1/statuses/:id/card endpoint https://github.com/mastodon/mastodon/pull/11213

changelog.d/card-image-description.add

@@ -0,0 +1 @@
+Include image description in status media cards

changelog.d/familiar-followers.add

@@ -0,0 +1 @@
+Implement `/api/v1/accounts/familiar_followers`

changelog.d/fep-2c59.add

@@ -0,0 +1 @@
+Implement FEP-2c59, add "webfinger" to user actor

changelog.d/ffmpeg-limiter.add

@@ -0,0 +1 @@
+Framegrabs with ffmpeg will execute with a 5 second timeout and cache the URLs of failures with a TTL of 15 minutes to prevent excessive retries.

changelog.d/fix-webfinger-spoofing.security

@@ -0,0 +1 @@
+Fix webfinger spoofing.

changelog.d/instance-rules.add

@@ -0,0 +1 @@
+Add instance rules

changelog.d/mark-read.fix

@@ -0,0 +1 @@
+The query for marking notifications as read has been simplified

changelog.d/mastodon_api_v2.add

@@ -0,0 +1 @@
+Add new parameters to /api/v2/instance: configuration[accounts][max_pinned_statuses] and configuration[statuses][characters_reserved_per_url]

changelog.d/missing-mrfs.add

@@ -0,0 +1 @@
+Startup detection for configured MRF modules that are missing or incorrectly defined

changelog.d/receiverworker-error-handling.fix

@@ -0,0 +1 @@
+ReceiverWorker: Make sure non-{:ok, _} is returned as {:error, …}

changelog.d/rich_media_refactor.change

@@ -0,0 +1 @@
+Refactored Rich Media to cache the content in the database. Fetching operations that could block status rendering have been eliminated.

changelog.d/status-notification-type.add

@@ -0,0 +1 @@
+Add "status" notification type

changelog.d/web_push_filtered.fix

@@ -0,0 +1 @@
+Web Push notifications are no longer generated for muted/blocked threads and users.

changelog.d/webfinger-validation.fix

@@ -0,0 +1 @@
+Fix validate_webfinger when running a different domain for Webfinger

config/config.exs

@@ -441,7 +441,11 @@ config :pleroma, :rich_media,
     Pleroma.Web.RichMedia.Parsers.OEmbed
   ],
   failure_backoff: 60_000,
-  ttl_setters: [Pleroma.Web.RichMedia.Parser.TTL.AwsSignedUrl]
+  ttl_setters: [
+    Pleroma.Web.RichMedia.Parser.TTL.AwsSignedUrl,
+    Pleroma.Web.RichMedia.Parser.TTL.Opengraph
+  ],
+  max_body: 5_000_000
 
 config :pleroma, :media_proxy,
   enabled: false,
@@ -588,7 +592,8 @@ config :pleroma, Oban,
     attachments_cleanup: 1,
     new_users_digest: 1,
     mute_expire: 5,
-    search_indexing: 10
+    search_indexing: 10,
+    rich_media_expiration: 2
   ],
   plugins: [Oban.Plugins.Pruner],
   crontab: [

config/description.exs

@@ -3522,7 +3522,7 @@ config :pleroma, :config_description, [
       },
       %{
         key: :initial_indexing_chunk_size,
-        type: :int,
+        type: :integer,
         description:
           "Amount of posts in a batch when running the initial indexing operation. Should probably not be more than 100000" <>
             " since there's a limit on maximum insert size",

config/test.exs

@@ -61,7 +61,8 @@ config :tesla, adapter: Tesla.Mock
 config :pleroma, :rich_media,
   enabled: false,
   ignore_hosts: [],
-  ignore_tld: ["local", "localdomain", "lan"]
+  ignore_tld: ["local", "localdomain", "lan"],
+  max_body: 2_000_000
 
 config :pleroma, :instance,
   multi_factor_authentication: [
@@ -174,6 +175,8 @@ config :pleroma, Pleroma.Uploaders.Uploader, timeout: 1_000
 
 config :pleroma, Pleroma.Emoji.Loader, test_emoji: true
 
+config :pleroma, Pleroma.Web.RichMedia.Backfill, provider: Pleroma.Web.RichMedia.Backfill
+
 if File.exists?("./config/test.secret.exs") do
   import_config "test.secret.exs"
 else

docs/development/API/admin_api.md

@@ -1751,3 +1751,53 @@ Note that this differs from the Mastodon API variant: Mastodon API only returns
 ```json
 {}
 ```
+
+
+## `GET /api/v1/pleroma/admin/rules`
+
+### List rules
+
+- Response: JSON, list of rules
+
+```json
+[
+  {
+    "id": "1",
+    "priority": 1,
+    "text": "There are no rules",
+    "hint": null
+  }
+]
+```
+
+## `POST /api/v1/pleroma/admin/rules`
+
+### Create a rule
+
+- Params:
+  - `text`: string, required, rule content
+  - `hint`: string, optional, rule description
+  - `priority`: integer, optional, rule ordering priority
+
+- Response: JSON, a single rule
+
+## `PATCH /api/v1/pleroma/admin/rules/:id`
+
+### Update a rule
+
+- Params:
+  - `text`: string, optional, rule content
+  - `hint`: string, optional, rule description
+  - `priority`: integer, optional, rule ordering priority
+
+- Response: JSON, a single rule
+
+## `DELETE /api/v1/pleroma/admin/rules/:id`
+
+### Delete a rule
+
+- Response: JSON, empty object
+
+```json
+{}
+```

lib/pleroma/application.ex

@@ -156,12 +156,14 @@ defmodule Pleroma.Application do
       build_cachex("web_resp", limit: 2500),
       build_cachex("emoji_packs", expiration: emoji_packs_expiration(), limit: 10),
       build_cachex("failed_proxy_url", limit: 2500),
+      build_cachex("failed_media_helper_url", default_ttl: :timer.minutes(15), limit: 2_500),
       build_cachex("banned_urls", default_ttl: :timer.hours(24 * 30), limit: 5_000),
       build_cachex("chat_message_id_idempotency_key",
         expiration: chat_message_id_idempotency_key_expiration(),
         limit: 500_000
       ),
-      build_cachex("rel_me", limit: 2500)
+      build_cachex("rel_me", limit: 2500),
+      build_cachex("host_meta", default_ttl: :timer.minutes(120), limit: 5000)
     ]
   end
 

lib/pleroma/application_requirements.ex

@@ -28,6 +28,7 @@ defmodule Pleroma.ApplicationRequirements do
     |> check_welcome_message_config!()
     |> check_rum!()
     |> check_repo_pool_size!()
+    |> check_mrfs()
     |> handle_result()
   end
 
@@ -234,4 +235,25 @@ defmodule Pleroma.ApplicationRequirements do
       true
     end
   end
+
+  defp check_mrfs(:ok) do
+    mrfs = Config.get!([:mrf, :policies])
+
+    missing_mrfs =
+      Enum.reduce(mrfs, [], fn x, acc ->
+        if Code.ensure_compiled(x) do
+          acc
+        else
+          acc ++ [x]
+        end
+      end)
+
+    if Enum.empty?(missing_mrfs) do
+      :ok
+    else
+      {:error, "The following MRF modules are configured but missing: #{inspect(missing_mrfs)}"}
+    end
+  end
+
+  defp check_mrfs(result), do: result
 end

lib/pleroma/constants.ex

@@ -19,7 +19,8 @@ defmodule Pleroma.Constants do
       "context_id",
       "deleted_activity_id",
       "pleroma_internal",
-      "generator"
+      "generator",
+      "rules"
     ]
   )
 

lib/pleroma/helpers/media_helper.ex

@@ -12,6 +12,8 @@ defmodule Pleroma.Helpers.MediaHelper do
 
   require Logger
 
+  @cachex Pleroma.Config.get([:cachex, :provider], Cachex)
+
   def missing_dependencies do
     Enum.reduce([ffmpeg: "ffmpeg"], [], fn {sym, executable}, acc ->
       if Pleroma.Utils.command_available?(executable) do
@@ -43,29 +45,40 @@ defmodule Pleroma.Helpers.MediaHelper do
   @spec video_framegrab(String.t()) :: {:ok, binary()} | {:error, any()}
   def video_framegrab(url) do
     with executable when is_binary(executable) <- System.find_executable("ffmpeg"),
+         false <- @cachex.exists?(:failed_media_helper_cache, url),
          {:ok, env} <- HTTP.get(url, [], pool: :media),
          {:ok, pid} <- StringIO.open(env.body) do
       body_stream = IO.binstream(pid, 1)
 
-      result =
-        Exile.stream!(
-          [
-            executable,
-            "-i",
-            "pipe:0",
-            "-vframes",
-            "1",
-            "-f",
-            "mjpeg",
-            "pipe:1"
-          ],
-          input: body_stream,
-          ignore_epipe: true,
-          stderr: :disable
-        )
-        |> Enum.into(<<>>)
+      task =
+        Task.async(fn ->
+          Exile.stream!(
+            [
+              executable,
+              "-i",
+              "pipe:0",
+              "-vframes",
+              "1",
+              "-f",
+              "mjpeg",
+              "pipe:1"
+            ],
+            input: body_stream,
+            ignore_epipe: true,
+            stderr: :disable
+          )
+          |> Enum.into(<<>>)
+        end)
 
-      {:ok, result}
+      case Task.yield(task, 5_000) do
+        nil ->
+          Task.shutdown(task)
+          @cachex.put(:failed_media_helper_cache, url, nil)
+          {:error, {:ffmpeg, :timeout}}
+
+        result ->
+          {:ok, result}
+      end
     else
       nil -> {:error, {:ffmpeg, :command_not_found}}
       {:error, _} = error -> error

lib/pleroma/html.ex

@@ -65,20 +65,16 @@ defmodule Pleroma.HTML do
     end
   end
 
-  @spec extract_first_external_url_from_object(Pleroma.Object.t()) ::
-          {:ok, String.t()} | {:error, :no_content}
+  @spec extract_first_external_url_from_object(Pleroma.Object.t()) :: String.t() | nil
   def extract_first_external_url_from_object(%{data: %{"content" => content}})
       when is_binary(content) do
-    url =
-      content
-      |> Floki.parse_fragment!()
-      |> Floki.find("a:not(.mention,.hashtag,.attachment,[rel~=\"tag\"])")
-      |> Enum.take(1)
-      |> Floki.attribute("href")
-      |> Enum.at(0)
-
-    {:ok, url}
+    content
+    |> Floki.parse_fragment!()
+    |> Floki.find("a:not(.mention,.hashtag,.attachment,[rel~=\"tag\"])")
+    |> Enum.take(1)
+    |> Floki.attribute("href")
+    |> Enum.at(0)
   end
 
-  def extract_first_external_url_from_object(_), do: {:error, :no_content}
+  def extract_first_external_url_from_object(_), do: nil
 end

lib/pleroma/notification.ex

@@ -73,6 +73,7 @@ defmodule Pleroma.Notification do
     pleroma:report
     reblog
     poll
+    status
   }
 
   def changeset(%Notification{} = notification, attrs) do
@@ -280,15 +281,10 @@ defmodule Pleroma.Notification do
         select: n.id
       )
 
-    {:ok, %{ids: {_, notification_ids}}} =
-      Multi.new()
-      |> Multi.update_all(:ids, query, set: [seen: true, updated_at: NaiveDateTime.utc_now()])
-      |> Marker.multi_set_last_read_id(user, "notifications")
-      |> Repo.transaction()
-
-    for_user_query(user)
-    |> where([n], n.id in ^notification_ids)
-    |> Repo.all()
+    Multi.new()
+    |> Multi.update_all(:ids, query, set: [seen: true, updated_at: NaiveDateTime.utc_now()])
+    |> Marker.multi_set_last_read_id(user, "notifications")
+    |> Repo.transaction()
   end
 
   @spec read_one(User.t(), String.t()) ::
@@ -299,10 +295,6 @@ defmodule Pleroma.Notification do
       |> Multi.update(:update, changeset(notification, %{seen: true}))
       |> Marker.multi_set_last_read_id(user, "notifications")
       |> Repo.transaction()
-      |> case do
-        {:ok, %{update: notification}} -> {:ok, notification}
-        {:error, :update, changeset, _} -> {:error, changeset}
-      end
     end
   end
 
@@ -361,37 +353,38 @@ defmodule Pleroma.Notification do
     end
   end
 
-  @spec create_notifications(Activity.t(), keyword()) :: {:ok, [Notification.t()] | []}
-  def create_notifications(activity, options \\ [])
+  @spec create_notifications(Activity.t()) :: {:ok, [Notification.t()] | []}
+  def create_notifications(activity)
 
-  def create_notifications(%Activity{data: %{"to" => _, "type" => "Create"}} = activity, options) do
+  def create_notifications(%Activity{data: %{"to" => _, "type" => "Create"}} = activity) do
     object = Object.normalize(activity, fetch: false)
 
     if object && object.data["type"] == "Answer" do
       {:ok, []}
     else
-      do_create_notifications(activity, options)
+      do_create_notifications(activity)
     end
   end
 
-  def create_notifications(%Activity{data: %{"type" => type}} = activity, options)
+  def create_notifications(%Activity{data: %{"type" => type}} = activity)
       when type in ["Follow", "Like", "Announce", "Move", "EmojiReact", "Flag", "Update"] do
-    do_create_notifications(activity, options)
+    do_create_notifications(activity)
   end
 
-  def create_notifications(_, _), do: {:ok, []}
+  def create_notifications(_), do: {:ok, []}
 
-  defp do_create_notifications(%Activity{} = activity, options) do
-    do_send = Keyword.get(options, :do_send, true)
+  defp do_create_notifications(%Activity{} = activity) do
+    enabled_receivers = get_notified_from_activity(activity)
 
-    {enabled_receivers, disabled_receivers} = get_notified_from_activity(activity)
-    potential_receivers = enabled_receivers ++ disabled_receivers
+    enabled_subscribers = get_notified_subscribers_from_activity(activity)
 
     notifications =
-      Enum.map(potential_receivers, fn user ->
-        do_send = do_send && user in enabled_receivers
-        create_notification(activity, user, do_send: do_send)
-      end)
+      (Enum.map(enabled_receivers, fn user ->
+         create_notification(activity, user)
+       end) ++
+         Enum.map(enabled_subscribers -- enabled_receivers, fn user ->
+           create_notification(activity, user, type: "status")
+         end))
       |> Enum.reject(&is_nil/1)
 
     {:ok, notifications}
@@ -450,7 +443,6 @@ defmodule Pleroma.Notification do
 
   # TODO move to sql, too.
   def create_notification(%Activity{} = activity, %User{} = user, opts \\ []) do
-    do_send = Keyword.get(opts, :do_send, true)
     type = Keyword.get(opts, :type, type_from_activity(activity))
 
     unless skip?(activity, user, opts) do
@@ -465,11 +457,6 @@ defmodule Pleroma.Notification do
         |> Marker.multi_set_last_read_id(user, "notifications")
         |> Repo.transaction()
 
-      if do_send do
-        Streamer.stream(["user", "user:notification"], notification)
-        Push.send(notification)
-      end
-
       notification
     end
   end
@@ -527,13 +514,28 @@ defmodule Pleroma.Notification do
       |> exclude_relationship_restricted_ap_ids(activity)
       |> exclude_thread_muter_ap_ids(activity)
 
-    notification_enabled_users =
-      Enum.filter(potential_receivers, fn u -> u.ap_id in notification_enabled_ap_ids end)
-
-    {notification_enabled_users, potential_receivers -- notification_enabled_users}
+    Enum.filter(potential_receivers, fn u -> u.ap_id in notification_enabled_ap_ids end)
   end
 
-  def get_notified_from_activity(_, _local_only), do: {[], []}
+  def get_notified_from_activity(_, _local_only), do: []
+
+  def get_notified_subscribers_from_activity(activity, local_only \\ true)
+
+  def get_notified_subscribers_from_activity(
+        %Activity{data: %{"type" => "Create"}} = activity,
+        local_only
+      ) do
+    notification_enabled_ap_ids =
+      []
+      |> Utils.maybe_notify_subscribers(activity)
+
+    potential_receivers =
+      User.get_users_from_set(notification_enabled_ap_ids, local_only: local_only)
+
+    Enum.filter(potential_receivers, fn u -> u.ap_id in notification_enabled_ap_ids end)
+  end
+
+  def get_notified_subscribers_from_activity(_, _), do: []
 
   # For some activities, only notify the author of the object
   def get_potential_receiver_ap_ids(%{data: %{"type" => type, "object" => object_id}})
@@ -576,7 +578,6 @@ defmodule Pleroma.Notification do
     []
     |> Utils.maybe_notify_to_recipients(activity)
     |> Utils.maybe_notify_mentioned_recipients(activity)
-    |> Utils.maybe_notify_subscribers(activity)
     |> Utils.maybe_notify_followers(activity)
     |> Enum.uniq()
   end
@@ -643,6 +644,7 @@ defmodule Pleroma.Notification do
   def skip?(%Activity{} = activity, %User{} = user, opts) do
     [
       :self,
+      :internal,
       :invisible,
       :block_from_strangers,
       :recently_followed,
@@ -662,6 +664,12 @@ defmodule Pleroma.Notification do
     end
   end
 
+  def skip?(:internal, %Activity{} = activity, _user, _opts) do
+    actor = activity.data["actor"]
+    user = User.get_cached_by_ap_id(actor)
+    User.internal?(user)
+  end
+
   def skip?(:invisible, %Activity{} = activity, _user, _opts) do
     actor = activity.data["actor"]
     user = User.get_cached_by_ap_id(actor)
@@ -748,4 +756,12 @@ defmodule Pleroma.Notification do
     )
     |> Repo.update_all(set: [seen: true])
   end
+
+  @spec send(list(Notification.t())) :: :ok
+  def send(notifications) do
+    Enum.each(notifications, fn notification ->
+      Streamer.stream(["user", "user:notification"], notification)
+      Push.send(notification)
+    end)
+  end
 end

lib/pleroma/rule.ex

@@ -0,0 +1,68 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Rule do
+  use Ecto.Schema
+
+  import Ecto.Changeset
+  import Ecto.Query
+
+  alias Pleroma.Repo
+  alias Pleroma.Rule
+
+  schema "rules" do
+    field(:priority, :integer, default: 0)
+    field(:text, :string)
+    field(:hint, :string)
+
+    timestamps()
+  end
+
+  def changeset(%Rule{} = rule, params \\ %{}) do
+    rule
+    |> cast(params, [:priority, :text, :hint])
+    |> validate_required([:text])
+  end
+
+  def query do
+    Rule
+    |> order_by(asc: :priority)
+    |> order_by(asc: :id)
+  end
+
+  def get(ids) when is_list(ids) do
+    from(r in __MODULE__, where: r.id in ^ids)
+    |> Repo.all()
+  end
+
+  def get(id), do: Repo.get(__MODULE__, id)
+
+  def exists?(id) do
+    from(r in __MODULE__, where: r.id == ^id)
+    |> Repo.exists?()
+  end
+
+  def create(params) do
+    {:ok, rule} =
+      %Rule{}
+      |> changeset(params)
+      |> Repo.insert()
+
+    rule
+  end
+
+  def update(params, id) do
+    {:ok, rule} =
+      get(id)
+      |> changeset(params)
+      |> Repo.update()
+
+    rule
+  end
+
+  def delete(id) do
+    get(id)
+    |> Repo.delete()
+  end
+end

lib/pleroma/user.ex

@@ -1404,6 +1404,40 @@ defmodule Pleroma.User do
     |> Repo.all()
   end
 
+  @spec get_familiar_followers_query(User.t(), User.t(), pos_integer() | nil) :: Ecto.Query.t()
+  def get_familiar_followers_query(%User{} = user, %User{} = current_user, nil) do
+    friends =
+      get_friends_query(current_user)
+      |> where([u], not u.hide_follows)
+      |> select([u], u.id)
+
+    User.Query.build(%{is_active: true})
+    |> where([u], u.id not in ^[user.id, current_user.id])
+    |> join(:inner, [u], r in FollowingRelationship,
+      as: :followers_relationships,
+      on: r.following_id == ^user.id and r.follower_id == u.id
+    )
+    |> where([followers_relationships: r], r.state == ^:follow_accept)
+    |> where([followers_relationships: r], r.follower_id in subquery(friends))
+  end
+
+  def get_familiar_followers_query(%User{} = user, %User{} = current_user, page) do
+    user
+    |> get_familiar_followers_query(current_user, nil)
+    |> User.Query.paginate(page, 20)
+  end
+
+  @spec get_familiar_followers_query(User.t(), User.t()) :: Ecto.Query.t()
+  def get_familiar_followers_query(%User{} = user, %User{} = current_user),
+    do: get_familiar_followers_query(user, current_user, nil)
+
+  @spec get_familiar_followers(User.t(), User.t(), pos_integer() | nil) :: {:ok, list(User.t())}
+  def get_familiar_followers(%User{} = user, %User{} = current_user, page \\ nil) do
+    user
+    |> get_familiar_followers_query(current_user, page)
+    |> Repo.all()
+  end
+
   def increase_note_count(%User{} = user) do
     User
     |> where(id: ^user.id)

lib/pleroma/web/activity_pub/activity_pub.ex

@@ -147,9 +147,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
       # Splice in the child object if we have one.
       activity = Maps.put_if_present(activity, :object, object)
 
-      ConcurrentLimiter.limit(Pleroma.Web.RichMedia.Helpers, fn ->
-        Task.start(fn -> Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity) end)
-      end)
+      Pleroma.Web.RichMedia.Card.get_by_activity(activity)
 
       # Add local posts to search index
       if local, do: Pleroma.Search.add_to_index(activity)
@@ -177,7 +175,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
           id: "pleroma:fakeid"
         }
 
-        Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity)
+        Pleroma.Web.RichMedia.Card.get_by_activity(activity)
         {:ok, activity}
 
       {:remote_limit_pass, _} ->
@@ -202,7 +200,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
   end
 
   def notify_and_stream(activity) do
-    Notification.create_notifications(activity)
+    {:ok, notifications} = Notification.create_notifications(activity)
+    Notification.send(notifications)
 
     original_activity =
       case activity do
@@ -1261,6 +1260,15 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
 
   defp restrict_quote_url(query, _), do: query
 
+  defp restrict_rule(query, %{rule_id: rule_id}) do
+    from(
+      activity in query,
+      where: fragment("(?)->'rules' \\? (?)", activity.data, ^rule_id)
+    )
+  end
+
+  defp restrict_rule(query, _), do: query
+
   defp exclude_poll_votes(query, %{include_poll_votes: true}), do: query
 
   defp exclude_poll_votes(query, _) do
@@ -1423,6 +1431,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
       |> restrict_instance(opts)
       |> restrict_announce_object_actor(opts)
       |> restrict_filtered(opts)
+      |> restrict_rule(opts)
       |> restrict_quote_url(opts)
       |> maybe_restrict_deactivated_users(opts)
       |> exclude_poll_votes(opts)

lib/pleroma/web/activity_pub/side_effects.ex

@@ -21,7 +21,6 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do
   alias Pleroma.Web.ActivityPub.Builder
   alias Pleroma.Web.ActivityPub.Pipeline
   alias Pleroma.Web.ActivityPub.Utils
-  alias Pleroma.Web.Push
   alias Pleroma.Web.Streamer
   alias Pleroma.Workers.PollWorker
 
@@ -125,7 +124,7 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do
         nil
     end
 
-    {:ok, notifications} = Notification.create_notifications(object, do_send: false)
+    {:ok, notifications} = Notification.create_notifications(object)
 
     meta =
       meta
@@ -184,7 +183,11 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do
     liked_object = Object.get_by_ap_id(object.data["object"])
     Utils.add_like_to_object(object, liked_object)
 
-    Notification.create_notifications(object)
+    {:ok, notifications} = Notification.create_notifications(object)
+
+    meta =
+      meta
+      |> add_notifications(notifications)
 
     {:ok, object, meta}
   end
@@ -202,7 +205,7 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do
   def handle(%{data: %{"type" => "Create"}} = activity, meta) do
     with {:ok, object, meta} <- handle_object_creation(meta[:object_data], activity, meta),
          %User{} = user <- User.get_cached_by_ap_id(activity.data["actor"]) do
-      {:ok, notifications} = Notification.create_notifications(activity, do_send: false)
+      {:ok, notifications} = Notification.create_notifications(activity)
       {:ok, _user} = ActivityPub.increase_note_count_if_public(user, object)
       {:ok, _user} = ActivityPub.update_last_status_at_if_public(user, object)
 
@@ -227,9 +230,7 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do
         end
       end
 
-      ConcurrentLimiter.limit(Pleroma.Web.RichMedia.Helpers, fn ->
-        Task.start(fn -> Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity) end)
-      end)
+      Pleroma.Web.RichMedia.Card.get_by_activity(activity)
 
       Pleroma.Search.add_to_index(Map.put(activity, :object, object))
 
@@ -258,11 +259,13 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do
 
     Utils.add_announce_to_object(object, announced_object)
 
-    if !User.internal?(user) do
-      Notification.create_notifications(object)
+    {:ok, notifications} = Notification.create_notifications(object)
 
-      ap_streamer().stream_out(object)
-    end
+    if !User.internal?(user), do: ap_streamer().stream_out(object)
+
+    meta =
+      meta
+      |> add_notifications(notifications)
 
     {:ok, object, meta}
   end
@@ -283,7 +286,11 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do
     reacted_object = Object.get_by_ap_id(object.data["object"])
     Utils.add_emoji_reaction_to_object(object, reacted_object)
 
-    Notification.create_notifications(object)
+    {:ok, notifications} = Notification.create_notifications(object)
+
+    meta =
+      meta
+      |> add_notifications(notifications)
 
     {:ok, object, meta}
   end
@@ -587,10 +594,7 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do
 
   defp send_notifications(meta) do
     Keyword.get(meta, :notifications, [])
-    |> Enum.each(fn notification ->
-      Streamer.stream(["user", "user:notification"], notification)
-      Push.send(notification)
-    end)
+    |> Notification.send()
 
     meta
   end

lib/pleroma/web/activity_pub/utils.ex

@@ -721,14 +721,18 @@ defmodule Pleroma.Web.ActivityPub.Utils do
 
   #### Flag-related helpers
   @spec make_flag_data(map(), map()) :: map()
-  def make_flag_data(%{actor: actor, context: context, content: content} = params, additional) do
+  def make_flag_data(
+        %{actor: actor, context: context, content: content} = params,
+        additional
+      ) do
     %{
       "type" => "Flag",
       "actor" => actor.ap_id,
       "content" => content,
       "object" => build_flag_object(params),
       "context" => context,
-      "state" => "open"
+      "state" => "open",
+      "rules" => Map.get(params, :rules, nil)
     }
     |> Map.merge(additional)
   end

lib/pleroma/web/activity_pub/views/user_view.ex

@@ -67,8 +67,13 @@ defmodule Pleroma.Web.ActivityPub.UserView do
   def render("user.json", %{user: %User{nickname: nil} = user}),
     do: render("service.json", %{user: user})
 
-  def render("user.json", %{user: %User{nickname: "internal." <> _} = user}),
-    do: render("service.json", %{user: user}) |> Map.put("preferredUsername", user.nickname)
+  def render("user.json", %{user: %User{nickname: "internal." <> _} = user}) do
+    render("service.json", %{user: user})
+    |> Map.merge(%{
+      "preferredUsername" => user.nickname,
+      "webfinger" => "acct:#{User.full_nickname(user)}"
+    })
+  end
 
   def render("user.json", %{user: user}) do
     {:ok, _, public_key} = Keys.keys_from_pem(user.keys)
@@ -121,7 +126,8 @@ defmodule Pleroma.Web.ActivityPub.UserView do
       "discoverable" => user.is_discoverable,
       "capabilities" => capabilities,
       "alsoKnownAs" => user.also_known_as,
-      "vcard:bday" => birthday
+      "vcard:bday" => birthday,
+      "webfinger" => "acct:#{User.full_nickname(user)}"
     }
     |> Map.merge(maybe_make_image(&User.avatar_url/2, "icon", user))
     |> Map.merge(maybe_make_image(&User.banner_url/2, "image", user))

lib/pleroma/web/admin_api/controllers/rule_controller.ex

@@ -0,0 +1,62 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.AdminAPI.RuleController do
+  use Pleroma.Web, :controller
+
+  alias Pleroma.Repo
+  alias Pleroma.Rule
+  alias Pleroma.Web.Plugs.OAuthScopesPlug
+
+  import Pleroma.Web.ControllerHelper,
+    only: [
+      json_response: 3
+    ]
+
+  plug(Pleroma.Web.ApiSpec.CastAndValidate)
+
+  plug(
+    OAuthScopesPlug,
+    %{scopes: ["admin:write"]}
+    when action in [:create, :update, :delete]
+  )
+
+  plug(OAuthScopesPlug, %{scopes: ["admin:read"]} when action == :index)
+
+  action_fallback(AdminAPI.FallbackController)
+
+  defdelegate open_api_operation(action), to: Pleroma.Web.ApiSpec.Admin.RuleOperation
+
+  def index(conn, _) do
+    rules =
+      Rule.query()
+      |> Repo.all()
+
+    render(conn, "index.json", rules: rules)
+  end
+
+  def create(%{body_params: params} = conn, _) do
+    rule =
+      params
+      |> Rule.create()
+
+    render(conn, "show.json", rule: rule)
+  end
+
+  def update(%{body_params: params} = conn, %{id: id}) do
+    rule =
+      params
+      |> Rule.update(id)
+
+    render(conn, "show.json", rule: rule)
+  end
+
+  def delete(conn, %{id: id}) do
+    with {:ok, _} <- Rule.delete(id) do
+      json(conn, %{})
+    else
+      _ -> json_response(conn, :bad_request, "")
+    end
+  end
+end

lib/pleroma/web/admin_api/views/report_view.ex

@@ -6,9 +6,11 @@ defmodule Pleroma.Web.AdminAPI.ReportView do
   use Pleroma.Web, :view
 
   alias Pleroma.HTML
+  alias Pleroma.Rule
   alias Pleroma.User
   alias Pleroma.Web.AdminAPI
   alias Pleroma.Web.AdminAPI.Report
+  alias Pleroma.Web.AdminAPI.RuleView
   alias Pleroma.Web.CommonAPI.Utils
   alias Pleroma.Web.MastodonAPI.StatusView
 
@@ -46,7 +48,8 @@ defmodule Pleroma.Web.AdminAPI.ReportView do
           as: :activity
         }),
       state: report.data["state"],
-      notes: render(__MODULE__, "index_notes.json", %{notes: report.report_notes})
+      notes: render(__MODULE__, "index_notes.json", %{notes: report.report_notes}),
+      rules: rules(Map.get(report.data, "rules", nil))
     }
   end
 
@@ -71,4 +74,16 @@ defmodule Pleroma.Web.AdminAPI.ReportView do
       created_at: Utils.to_masto_date(inserted_at)
     }
   end
+
+  defp rules(nil) do
+    []
+  end
+
+  defp rules(rule_ids) do
+    rules =
+      rule_ids
+      |> Rule.get()
+
+    render(RuleView, "index.json", rules: rules)
+  end
 end

lib/pleroma/web/admin_api/views/rule_view.ex

@@ -0,0 +1,22 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.AdminAPI.RuleView do
+  use Pleroma.Web, :view
+
+  require Pleroma.Constants
+
+  def render("index.json", %{rules: rules} = _opts) do
+    render_many(rules, __MODULE__, "show.json")
+  end
+
+  def render("show.json", %{rule: rule} = _opts) do
+    %{
+      id: to_string(rule.id),
+      priority: rule.priority,
+      text: rule.text,
+      hint: rule.hint
+    }
+  end
+end

lib/pleroma/web/api_spec.ex

@@ -97,6 +97,7 @@ defmodule Pleroma.Web.ApiSpec do
               "Frontend management",
               "Instance configuration",
               "Instance documents",
+              "Instance rule managment",
               "Invites",
               "MediaProxy cache",
               "OAuth application management",

lib/pleroma/web/api_spec/operations/account_operation.ex

@@ -11,6 +11,7 @@ defmodule Pleroma.Web.ApiSpec.AccountOperation do
   alias Pleroma.Web.ApiSpec.Schemas.ActorType
   alias Pleroma.Web.ApiSpec.Schemas.ApiError
   alias Pleroma.Web.ApiSpec.Schemas.BooleanLike
+  alias Pleroma.Web.ApiSpec.Schemas.FlakeID
   alias Pleroma.Web.ApiSpec.Schemas.List
   alias Pleroma.Web.ApiSpec.Schemas.Status
   alias Pleroma.Web.ApiSpec.Schemas.VisibilityScope
@@ -513,6 +514,48 @@ defmodule Pleroma.Web.ApiSpec.AccountOperation do
     }
   end
 
+  def familiar_followers_operation do
+    %Operation{
+      tags: ["Retrieve account information"],
+      summary: "Followers that you follow",
+      operationId: "AccountController.familiar_followers",
+      description:
+        "Obtain a list of all accounts that follow a given account, filtered for accounts you follow.",
+      security: [%{"oAuth" => ["read:follows"]}],
+      parameters: [
+        Operation.parameter(
+          :id,
+          :query,
+          %Schema{
+            oneOf: [%Schema{type: :array, items: %Schema{type: :string}}, %Schema{type: :string}]
+          },
+          "Account IDs",
+          example: "123"
+        )
+      ],
+      responses: %{
+        200 =>
+          Operation.response("Accounts", "application/json", %Schema{
+            title: "ArrayOfAccounts",
+            type: :array,
+            items: %Schema{
+              title: "Account",
+              type: :object,
+              properties: %{
+                id: FlakeID,
+                accounts: %Schema{
+                  title: "ArrayOfAccounts",
+                  type: :array,
+                  items: Account,
+                  example: [Account.schema().example]
+                }
+              }
+            }
+          })
+      }
+    }
+  end
+
   defp create_request do
     %Schema{
       title: "AccountCreateRequest",

lib/pleroma/web/api_spec/operations/admin/report_operation.ex

@@ -30,6 +30,12 @@ defmodule Pleroma.Web.ApiSpec.Admin.ReportOperation do
           report_state(),
           "Filter by report state"
         ),
+        Operation.parameter(
+          :rule_id,
+          :query,
+          %Schema{type: :string},
+          "Filter by selected rule id"
+        ),
         Operation.parameter(
           :limit,
           :query,
@@ -169,6 +175,17 @@ defmodule Pleroma.Web.ApiSpec.Admin.ReportOperation do
               inserted_at: %Schema{type: :string, format: :"date-time"}
             }
           }
+        },
+        rules: %Schema{
+          type: :array,
+          items: %Schema{
+            type: :object,
+            properties: %{
+              id: %Schema{type: :string},
+              text: %Schema{type: :string},
+              hint: %Schema{type: :string, nullable: true}
+            }
+          }
         }
       }
     }

lib/pleroma/web/api_spec/operations/admin/rule_operation.ex

@@ -0,0 +1,115 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ApiSpec.Admin.RuleOperation do
+  alias OpenApiSpex.Operation
+  alias OpenApiSpex.Schema
+  alias Pleroma.Web.ApiSpec.Schemas.ApiError
+
+  import Pleroma.Web.ApiSpec.Helpers
+
+  def open_api_operation(action) do
+    operation = String.to_existing_atom("#{action}_operation")
+    apply(__MODULE__, operation, [])
+  end
+
+  def index_operation do
+    %Operation{
+      tags: ["Instance rule managment"],
+      summary: "Retrieve list of instance rules",
+      operationId: "AdminAPI.RuleController.index",
+      security: [%{"oAuth" => ["admin:read"]}],
+      responses: %{
+        200 =>
+          Operation.response("Response", "application/json", %Schema{
+            type: :array,
+            items: rule()
+          }),
+        403 => Operation.response("Forbidden", "application/json", ApiError)
+      }
+    }
+  end
+
+  def create_operation do
+    %Operation{
+      tags: ["Instance rule managment"],
+      summary: "Create new rule",
+      operationId: "AdminAPI.RuleController.create",
+      security: [%{"oAuth" => ["admin:write"]}],
+      parameters: admin_api_params(),
+      requestBody: request_body("Parameters", create_request(), required: true),
+      responses: %{
+        200 => Operation.response("Response", "application/json", rule()),
+        400 => Operation.response("Bad Request", "application/json", ApiError),
+        403 => Operation.response("Forbidden", "application/json", ApiError)
+      }
+    }
+  end
+
+  def update_operation do
+    %Operation{
+      tags: ["Instance rule managment"],
+      summary: "Modify existing rule",
+      operationId: "AdminAPI.RuleController.update",
+      security: [%{"oAuth" => ["admin:write"]}],
+      parameters: [Operation.parameter(:id, :path, :string, "Rule ID")],
+      requestBody: request_body("Parameters", update_request(), required: true),
+      responses: %{
+        200 => Operation.response("Response", "application/json", rule()),
+        400 => Operation.response("Bad Request", "application/json", ApiError),
+        403 => Operation.response("Forbidden", "application/json", ApiError)
+      }
+    }
+  end
+
+  def delete_operation do
+    %Operation{
+      tags: ["Instance rule managment"],
+      summary: "Delete rule",
+      operationId: "AdminAPI.RuleController.delete",
+      parameters: [Operation.parameter(:id, :path, :string, "Rule ID")],
+      security: [%{"oAuth" => ["admin:write"]}],
+      responses: %{
+        200 => empty_object_response(),
+        404 => Operation.response("Not Found", "application/json", ApiError),
+        403 => Operation.response("Forbidden", "application/json", ApiError)
+      }
+    }
+  end
+
+  defp create_request do
+    %Schema{
+      type: :object,
+      required: [:text],
+      properties: %{
+        priority: %Schema{type: :integer},
+        text: %Schema{type: :string},
+        hint: %Schema{type: :string}
+      }
+    }
+  end
+
+  defp update_request do
+    %Schema{
+      type: :object,
+      properties: %{
+        priority: %Schema{type: :integer},
+        text: %Schema{type: :string},
+        hint: %Schema{type: :string}
+      }
+    }
+  end
+
+  defp rule do
+    %Schema{
+      type: :object,
+      properties: %{
+        id: %Schema{type: :string},
+        priority: %Schema{type: :integer},
+        text: %Schema{type: :string},
+        hint: %Schema{type: :string, nullable: true}
+      }
+    }
+  end
+end

lib/pleroma/web/api_spec/operations/instance_operation.ex

@@ -46,10 +46,30 @@ defmodule Pleroma.Web.ApiSpec.InstanceOperation do
     }
   end
 
+  def rules_operation do
+    %Operation{
+      tags: ["Instance misc"],
+      summary: "Retrieve list of instance rules",
+      operationId: "InstanceController.rules",
+      responses: %{
+        200 => Operation.response("Array of domains", "application/json", array_of_rules())
+      }
+    }
+  end
+
   defp instance do
     %Schema{
       type: :object,
       properties: %{
+        accounts: %Schema{
+          type: :object,
+          properties: %{
+            max_featured_tags: %Schema{
+              type: :integer,
+              description: "The maximum number of featured tags allowed for each account."
+            }
+          }
+        },
         uri: %Schema{type: :string, description: "The domain name of the instance"},
         title: %Schema{type: :string, description: "The title of the website"},
         description: %Schema{
@@ -172,7 +192,8 @@ defmodule Pleroma.Web.ApiSpec.InstanceOperation do
         "urls" => %{
           "streaming_api" => "wss://lain.com"
         },
-        "version" => "2.7.2 (compatible; Pleroma 2.0.50-536-g25eec6d7-develop)"
+        "version" => "2.7.2 (compatible; Pleroma 2.0.50-536-g25eec6d7-develop)",
+        "rules" => array_of_rules()
       }
     }
   end
@@ -272,6 +293,19 @@ defmodule Pleroma.Web.ApiSpec.InstanceOperation do
           type: :object,
           description: "Instance configuration",
           properties: %{
+            accounts: %Schema{
+              type: :object,
+              properties: %{
+                max_featured_tags: %Schema{
+                  type: :integer,
+                  description: "The maximum number of featured tags allowed for each account."
+                },
+                max_pinned_statuses: %Schema{
+                  type: :integer,
+                  description: "The maximum number of pinned statuses for each account."
+                }
+              }
+            },
             urls: %Schema{
               type: :object,
               properties: %{
@@ -285,6 +319,11 @@ defmodule Pleroma.Web.ApiSpec.InstanceOperation do
               type: :object,
               description: "A map with poll limits for local statuses",
               properties: %{
+                characters_reserved_per_url: %Schema{
+                  type: :integer,
+                  description:
+                    "Each URL in a status will be assumed to be exactly this many characters."
+                },
                 max_characters: %Schema{
                   type: :integer,
                   description: "Posts character limit (CW/Subject included in the counter)"
@@ -344,4 +383,18 @@ defmodule Pleroma.Web.ApiSpec.InstanceOperation do
       example: ["pleroma.site", "lain.com", "bikeshed.party"]
     }
   end
+
+  defp array_of_rules do
+    %Schema{
+      type: :array,
+      items: %Schema{
+        type: :object,
+        properties: %{
+          id: %Schema{type: :string},
+          text: %Schema{type: :string},
+          hint: %Schema{type: :string}
+        }
+      }
+    }
+  end
 end

lib/pleroma/web/api_spec/operations/notification_operation.ex

@@ -202,7 +202,8 @@ defmodule Pleroma.Web.ApiSpec.NotificationOperation do
         "pleroma:report",
         "move",
         "follow_request",
-        "poll"
+        "poll",
+        "status"
       ],
       description: """
       The type of event that resulted in the notification.
@@ -216,6 +217,7 @@ defmodule Pleroma.Web.ApiSpec.NotificationOperation do
       - `pleroma:emoji_reaction` - Someone reacted with emoji to your status
       - `pleroma:chat_mention` - Someone mentioned you in a chat message
       - `pleroma:report` - Someone was reported
+      - `status` - Someone you are subscribed to created a status
       """
     }
   end

lib/pleroma/web/api_spec/operations/pleroma_notification_operation.ex

@@ -5,7 +5,6 @@
 defmodule Pleroma.Web.ApiSpec.PleromaNotificationOperation do
   alias OpenApiSpex.Operation
   alias OpenApiSpex.Schema
-  alias Pleroma.Web.ApiSpec.NotificationOperation
   alias Pleroma.Web.ApiSpec.Schemas.ApiError
 
   import Pleroma.Web.ApiSpec.Helpers
@@ -35,12 +34,7 @@ defmodule Pleroma.Web.ApiSpec.PleromaNotificationOperation do
           Operation.response(
             "A Notification or array of Notifications",
             "application/json",
-            %Schema{
-              anyOf: [
-                %Schema{type: :array, items: NotificationOperation.notification()},
-                NotificationOperation.notification()
-              ]
-            }
+            %Schema{type: :string}
           ),
         400 => Operation.response("Bad Request", "application/json", ApiError)
       }

lib/pleroma/web/api_spec/operations/report_operation.ex

@@ -53,6 +53,12 @@ defmodule Pleroma.Web.ApiSpec.ReportOperation do
           default: false,
           description:
             "If the account is remote, should the report be forwarded to the remote admin?"
+        },
+        rule_ids: %Schema{
+          type: :array,
+          nullable: true,
+          items: %Schema{type: :string},
+          description: "Array of rules"
         }
       },
       required: [:account_id],
@@ -60,7 +66,8 @@ defmodule Pleroma.Web.ApiSpec.ReportOperation do
         "account_id" => "123",
         "status_ids" => ["1337"],
         "comment" => "bad status!",
-        "forward" => "false"
+        "forward" => "false",
+        "rule_ids" => ["3"]
       }
     }
   end

lib/pleroma/web/api_spec/schemas/status.ex

@@ -58,6 +58,10 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Status do
             format: :uri,
             description: "Preview thumbnail"
           },
+          image_description: %Schema{
+            type: :string,
+            description: "Alternate text that describes what is in the thumbnail"
+          },
           title: %Schema{type: :string, description: "Title of linked resource"},
           description: %Schema{type: :string, description: "Description of preview"}
         }

lib/pleroma/web/common_api.ex

@@ -8,6 +8,7 @@ defmodule Pleroma.Web.CommonAPI do
   alias Pleroma.Formatter
   alias Pleroma.ModerationLog
   alias Pleroma.Object
+  alias Pleroma.Rule
   alias Pleroma.ThreadMute
   alias Pleroma.User
   alias Pleroma.UserRelationship
@@ -568,14 +569,16 @@ defmodule Pleroma.Web.CommonAPI do
   def report(user, data) do
     with {:ok, account} <- get_reported_account(data.account_id),
          {:ok, {content_html, _, _}} <- make_report_content_html(data[:comment]),
-         {:ok, statuses} <- get_report_statuses(account, data) do
+         {:ok, statuses} <- get_report_statuses(account, data),
+         rules <- get_report_rules(Map.get(data, :rule_ids, nil)) do
       ActivityPub.flag(%{
         context: Utils.generate_context_id(),
         actor: user,
         account: account,
         statuses: statuses,
         content: content_html,
-        forward: Map.get(data, :forward, false)
+        forward: Map.get(data, :forward, false),
+        rules: rules
       })
     end
   end
@@ -587,6 +590,15 @@ defmodule Pleroma.Web.CommonAPI do
     end
   end
 
+  defp get_report_rules(nil) do
+    nil
+  end
+
+  defp get_report_rules(rule_ids) do
+    rule_ids
+    |> Enum.filter(&Rule.exists?/1)
+  end
+
   def update_report_state(activity_ids, state) when is_list(activity_ids) do
     case Utils.update_report_state(activity_ids, state) do
       :ok -> {:ok, activity_ids}

lib/pleroma/web/mastodon_api/controllers/account_controller.ex

@@ -72,7 +72,10 @@ defmodule Pleroma.Web.MastodonAPI.AccountController do
     %{scopes: ["follow", "write:blocks"]} when action in [:block, :unblock]
   )
 
-  plug(OAuthScopesPlug, %{scopes: ["read:follows"]} when action == :relationships)
+  plug(
+    OAuthScopesPlug,
+    %{scopes: ["read:follows"]} when action in [:relationships, :familiar_followers]
+  )
 
   plug(
     OAuthScopesPlug,
@@ -629,6 +632,35 @@ defmodule Pleroma.Web.MastodonAPI.AccountController do
     )
   end
 
+  @doc "GET /api/v1/accounts/familiar_followers"
+  def familiar_followers(
+        %{assigns: %{user: user}, private: %{open_api_spex: %{params: %{id: id}}}} = conn,
+        _id
+      ) do
+    users =
+      User.get_all_by_ids(List.wrap(id))
+      |> Enum.map(&%{id: &1.id, accounts: get_familiar_followers(&1, user)})
+
+    conn
+    |> render("familiar_followers.json",
+      for: user,
+      users: users,
+      as: :user
+    )
+  end
+
+  defp get_familiar_followers(%{id: id} = user, %{id: id}) do
+    User.get_familiar_followers(user, user)
+  end
+
+  defp get_familiar_followers(%{hide_followers: true}, _current_user) do
+    []
+  end
+
+  defp get_familiar_followers(user, current_user) do
+    User.get_familiar_followers(user, current_user)
+  end
+
   @doc "GET /api/v1/identity_proofs"
   def identity_proofs(conn, params), do: MastodonAPIController.empty_array(conn, params)
 end

lib/pleroma/web/mastodon_api/controllers/instance_controller.ex

@@ -25,4 +25,9 @@ defmodule Pleroma.Web.MastodonAPI.InstanceController do
   def peers(conn, _params) do
     json(conn, Pleroma.Stats.get_peers())
   end
+
+  @doc "GET /api/v1/instance/rules"
+  def rules(conn, _params) do
+    render(conn, "rules.json")
+  end
 end

lib/pleroma/web/mastodon_api/controllers/notification_controller.ex

@@ -34,6 +34,7 @@ defmodule Pleroma.Web.MastodonAPI.NotificationController do
       pleroma:emoji_reaction
       poll
       update
+      status
     }
 
   # GET /api/v1/notifications

lib/pleroma/web/mastodon_api/controllers/status_controller.ex

@@ -38,7 +38,6 @@ defmodule Pleroma.Web.MastodonAPI.StatusController do
     when action in [
            :index,
            :show,
-           :card,
            :context,
            :show_history,
            :show_source
@@ -473,21 +472,6 @@ defmodule Pleroma.Web.MastodonAPI.StatusController do
     end
   end
 
-  @doc "GET /api/v1/statuses/:id/card"
-  @deprecated "https://github.com/tootsuite/mastodon/pull/11213"
-  def card(
-        %{assigns: %{user: user}, private: %{open_api_spex: %{params: %{id: status_id}}}} = conn,
-        _
-      ) do
-    with %Activity{} = activity <- Activity.get_by_id(status_id),
-         true <- Visibility.visible_for_user?(activity, user) do
-      data = Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity)
-      render(conn, "card.json", data)
-    else
-      _ -> render_error(conn, :not_found, "Record not found")
-    end
-  end
-
   @doc "GET /api/v1/statuses/:id/favourited_by"
   def favourited_by(
         %{assigns: %{user: user}, private: %{open_api_spex: %{params: %{id: id}}}} = conn,

lib/pleroma/web/mastodon_api/views/account_view.ex

@@ -193,6 +193,25 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do
     render_many(targets, AccountView, "relationship.json", render_opts)
   end
 
+  def render("familiar_followers.json", %{users: users} = opts) do
+    opts =
+      opts
+      |> Map.merge(%{as: :user})
+      |> Map.delete(:users)
+
+    users
+    |> render_many(AccountView, "familiar_followers.json", opts)
+  end
+
+  def render("familiar_followers.json", %{user: %{id: id, accounts: accounts}} = opts) do
+    accounts =
+      accounts
+      |> render_many(AccountView, "show.json", opts)
+      |> Enum.filter(&Enum.any?/1)
+
+    %{id: id, accounts: accounts}
+  end
+
   defp do_render("show.json", %{user: user} = opts) do
     self = opts[:for] == user
 

lib/pleroma/web/mastodon_api/views/instance_view.ex

@@ -76,12 +76,26 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do
     })
   end
 
+  def render("rules.json", _) do
+    Pleroma.Rule.query()
+    |> Pleroma.Repo.all()
+    |> render_many(__MODULE__, "rule.json", as: :rule)
+  end
+
+  def render("rule.json", %{rule: rule}) do
+    %{
+      id: to_string(rule.id),
+      text: rule.text,
+      hint: rule.hint || ""
+    }
+  end
+
   defp common_information(instance) do
     %{
-      title: Keyword.get(instance, :name),
-      version: "#{@mastodon_api_level} (compatible; #{Pleroma.Application.named_version()})",
       languages: Keyword.get(instance, :languages, ["en"]),
-      rules: []
+      rules: render(__MODULE__, "rules.json"),
+      title: Keyword.get(instance, :name),
+      version: "#{@mastodon_api_level} (compatible; #{Pleroma.Application.named_version()})"
     }
   end
 
@@ -213,6 +227,8 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do
 
   defp configuration2 do
     configuration()
+    |> put_in([:accounts, :max_pinned_statuses], Config.get([:instance, :max_pinned_statuses], 0))
+    |> put_in([:statuses, :characters_reserved_per_url], 0)
     |> Map.merge(%{
       urls: %{
         streaming: Pleroma.Web.Endpoint.websocket_url(),

lib/pleroma/web/mastodon_api/views/notification_view.ex

@@ -108,6 +108,9 @@ defmodule Pleroma.Web.MastodonAPI.NotificationView do
       "mention" ->
         put_status(response, activity, reading_user, status_render_opts)
 
+      "status" ->
+        put_status(response, activity, reading_user, status_render_opts)
+
       "favourite" ->
         put_status(response, parent_activity_fn.(), reading_user, status_render_opts)
 

lib/pleroma/web/mastodon_api/views/status_view.ex

@@ -21,6 +21,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
   alias Pleroma.Web.MastodonAPI.StatusView
   alias Pleroma.Web.MediaProxy
   alias Pleroma.Web.PleromaAPI.EmojiReactionController
+  alias Pleroma.Web.RichMedia.Card
 
   import Pleroma.Web.ActivityPub.Visibility, only: [get_visibility: 1, visible_for_user?: 2]
 
@@ -29,9 +30,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
   # pagination is restricted to 40 activities at a time
   defp fetch_rich_media_for_activities(activities) do
     Enum.each(activities, fn activity ->
-      spawn(fn ->
-        Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity)
-      end)
+      spawn(fn -> Card.get_by_activity(activity) end)
     end)
   end
 
@@ -113,9 +112,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
     # To do: check AdminAPIControllerTest on the reasons behind nil activities in the list
     activities = Enum.filter(opts.activities, & &1)
 
-    # Start fetching rich media before doing anything else, so that later calls to get the cards
-    # only block for timeout in the worst case, as opposed to
-    # length(activities_with_links) * timeout
+    # Start prefetching rich media before doing anything else
     fetch_rich_media_for_activities(activities)
     replied_to_activities = get_replied_to_activities(activities)
     quoted_activities = get_quoted_activities(activities)
@@ -364,7 +361,11 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
 
     summary = object.data["summary"] || ""
 
-    card = render("card.json", Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity))
+    card =
+      case Card.get_by_activity(activity) do
+        %Card{} = result -> render("card.json", result)
+        _ -> nil
+      end
 
     url =
       if user.local do
@@ -567,15 +568,8 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
     }
   end
 
-  def render("card.json", %{rich_media: rich_media, page_url: page_url}) do
-    page_url_data = URI.parse(page_url)
-
-    page_url_data =
-      if is_binary(rich_media["url"]) do
-        URI.merge(page_url_data, URI.parse(rich_media["url"]))
-      else
-        page_url_data
-      end
+  def render("card.json", %Card{fields: rich_media}) do
+    page_url_data = URI.parse(rich_media["url"])
 
     page_url = page_url_data |> to_string
 
@@ -589,6 +583,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
       provider_url: page_url_data.scheme <> "://" <> page_url_data.host,
       url: page_url,
       image: image_url,
+      image_description: rich_media["image:alt"] || "",
       title: rich_media["title"] || "",
       description: rich_media["description"] || "",
       pleroma: %{

lib/pleroma/web/pleroma_api/controllers/notification_controller.ex

@@ -23,8 +23,9 @@ defmodule Pleroma.Web.PleromaAPI.NotificationController do
         } = conn,
         _
       ) do
-    with {:ok, notification} <- Notification.read_one(user, notification_id) do
-      render(conn, "show.json", notification: notification, for: user)
+    with {:ok, _} <- Notification.read_one(user, notification_id) do
+      conn
+      |> json("ok")
     else
       {:error, message} ->
         conn
@@ -38,11 +39,14 @@ defmodule Pleroma.Web.PleromaAPI.NotificationController do
           conn,
         _
       ) do
-    notifications =
-      user
-      |> Notification.set_read_up_to(max_id)
-      |> Enum.take(80)
-
-    render(conn, "index.json", notifications: notifications, for: user)
+    with {:ok, _} <- Notification.set_read_up_to(user, max_id) do
+      conn
+      |> json("ok")
+    else
+      {:error, message} ->
+        conn
+        |> put_status(:bad_request)
+        |> json(%{"error" => message})
+    end
   end
 end

lib/pleroma/web/pleroma_api/views/chat/message_reference_view.ex

@@ -9,6 +9,7 @@ defmodule Pleroma.Web.PleromaAPI.Chat.MessageReferenceView do
   alias Pleroma.User
   alias Pleroma.Web.CommonAPI.Utils
   alias Pleroma.Web.MastodonAPI.StatusView
+  alias Pleroma.Web.RichMedia.Card
 
   @cachex Pleroma.Config.get([:cachex, :provider], Cachex)
 
@@ -23,6 +24,12 @@ defmodule Pleroma.Web.PleromaAPI.Chat.MessageReferenceView do
           }
         }
       ) do
+    card =
+      case Card.get_by_object(object) do
+        %Card{} = card_data -> StatusView.render("card.json", card_data)
+        _ -> nil
+      end
+
     %{
       id: id |> to_string(),
       content: chat_message["content"],
@@ -34,11 +41,7 @@ defmodule Pleroma.Web.PleromaAPI.Chat.MessageReferenceView do
         chat_message["attachment"] &&
           StatusView.render("attachment.json", attachment: chat_message["attachment"]),
       unread: unread,
-      card:
-        StatusView.render(
-          "card.json",
-          Pleroma.Web.RichMedia.Helpers.fetch_data_for_object(object)
-        )
+      card: card
     }
     |> put_idempotency_key()
   end

lib/pleroma/web/push/impl.ex

@@ -192,6 +192,7 @@ defmodule Pleroma.Web.Push.Impl do
   def format_title(%{type: type}, mastodon_type) do
     case mastodon_type || type do
       "mention" -> "New Mention"
+      "status" -> "New Status"
       "follow" -> "New Follower"
       "follow_request" -> "New Follow Request"
       "reblog" -> "New Repeat"

lib/pleroma/web/rich_media/backfill.ex

@@ -0,0 +1,101 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.RichMedia.Backfill do
+  alias Pleroma.Web.RichMedia.Card
+  alias Pleroma.Web.RichMedia.Parser
+  alias Pleroma.Web.RichMedia.Parser.TTL
+  alias Pleroma.Workers.RichMediaExpirationWorker
+
+  require Logger
+
+  @backfiller Pleroma.Config.get([__MODULE__, :provider], Pleroma.Web.RichMedia.Backfill.Task)
+  @cachex Pleroma.Config.get([:cachex, :provider], Cachex)
+  @max_attempts 3
+  @retry 5_000
+
+  def start(%{url: url} = args) when is_binary(url) do
+    url_hash = Card.url_to_hash(url)
+
+    args =
+      args
+      |> Map.put(:attempt, 1)
+      |> Map.put(:url_hash, url_hash)
+
+    @backfiller.run(args)
+  end
+
+  def run(%{url: url, url_hash: url_hash, attempt: attempt} = args)
+      when attempt <= @max_attempts do
+    case Parser.parse(url) do
+      {:ok, fields} ->
+        {:ok, card} = Card.create(url, fields)
+
+        maybe_schedule_expiration(url, fields)
+
+        if Map.has_key?(args, :activity_id) do
+          stream_update(args)
+        end
+
+        warm_cache(url_hash, card)
+
+      {:error, {:invalid_metadata, fields}} ->
+        Logger.debug("Rich media incomplete or invalid metadata for #{url}: #{inspect(fields)}")
+        negative_cache(url_hash)
+
+      {:error, :body_too_large} ->
+        Logger.error("Rich media error for #{url}: :body_too_large")
+        negative_cache(url_hash)
+
+      {:error, {:content_type, type}} ->
+        Logger.debug("Rich media error for #{url}: :content_type is #{type}")
+        negative_cache(url_hash)
+
+      e ->
+        Logger.debug("Rich media error for #{url}: #{inspect(e)}")
+
+        :timer.sleep(@retry * attempt)
+
+        run(%{args | attempt: attempt + 1})
+    end
+  end
+
+  def run(%{url: url, url_hash: url_hash}) do
+    Logger.debug("Rich media failure for #{url}")
+
+    negative_cache(url_hash, :timer.minutes(15))
+  end
+
+  defp maybe_schedule_expiration(url, fields) do
+    case TTL.process(fields, url) do
+      {:ok, ttl} when is_number(ttl) ->
+        timestamp = DateTime.from_unix!(ttl)
+
+        RichMediaExpirationWorker.new(%{"url" => url}, scheduled_at: timestamp)
+        |> Oban.insert()
+
+      _ ->
+        :ok
+    end
+  end
+
+  defp stream_update(%{activity_id: activity_id}) do
+    Pleroma.Activity.get_by_id(activity_id)
+    |> Pleroma.Activity.normalize()
+    |> Pleroma.Web.ActivityPub.ActivityPub.stream_out()
+  end
+
+  defp warm_cache(key, val), do: @cachex.put(:rich_media_cache, key, val)
+  defp negative_cache(key, ttl \\ nil), do: @cachex.put(:rich_media_cache, key, nil, ttl: ttl)
+end
+
+defmodule Pleroma.Web.RichMedia.Backfill.Task do
+  alias Pleroma.Web.RichMedia.Backfill
+
+  def run(args) do
+    Task.Supervisor.start_child(Pleroma.TaskSupervisor, Backfill, :run, [args],
+      name: {:global, {:rich_media, args.url_hash}}
+    )
+  end
+end

lib/pleroma/web/rich_media/card.ex

@@ -0,0 +1,157 @@
+defmodule Pleroma.Web.RichMedia.Card do
+  use Ecto.Schema
+  import Ecto.Changeset
+  import Ecto.Query
+
+  alias Pleroma.Activity
+  alias Pleroma.HTML
+  alias Pleroma.Object
+  alias Pleroma.Repo
+  alias Pleroma.Web.RichMedia.Backfill
+  alias Pleroma.Web.RichMedia.Parser
+
+  @cachex Pleroma.Config.get([:cachex, :provider], Cachex)
+  @config_impl Application.compile_env(:pleroma, [__MODULE__, :config_impl], Pleroma.Config)
+
+  @type t :: %__MODULE__{}
+
+  schema "rich_media_card" do
+    field(:url_hash, :binary)
+    field(:fields, :map)
+
+    timestamps()
+  end
+
+  @doc false
+  def changeset(card, attrs) do
+    card
+    |> cast(attrs, [:url_hash, :fields])
+    |> validate_required([:url_hash, :fields])
+    |> unique_constraint(:url_hash)
+  end
+
+  @spec create(String.t(), map()) :: {:ok, t()}
+  def create(url, fields) do
+    url_hash = url_to_hash(url)
+
+    fields = Map.put_new(fields, "url", url)
+
+    %__MODULE__{}
+    |> changeset(%{url_hash: url_hash, fields: fields})
+    |> Repo.insert(on_conflict: {:replace, [:fields]}, conflict_target: :url_hash)
+  end
+
+  @spec delete(String.t()) :: {:ok, Ecto.Schema.t()} | {:error, Ecto.Changeset.t()} | :ok
+  def delete(url) do
+    url_hash = url_to_hash(url)
+    @cachex.del(:rich_media_cache, url_hash)
+
+    case get_by_url(url) do
+      %__MODULE__{} = card -> Repo.delete(card)
+      nil -> :ok
+    end
+  end
+
+  @spec get_by_url(String.t() | nil) :: t() | nil | :error
+  def get_by_url(url) when is_binary(url) do
+    if @config_impl.get([:rich_media, :enabled]) do
+      url_hash = url_to_hash(url)
+
+      @cachex.fetch!(:rich_media_cache, url_hash, fn _ ->
+        result =
+          __MODULE__
+          |> where(url_hash: ^url_hash)
+          |> Repo.one()
+
+        case result do
+          %__MODULE__{} = card -> {:commit, card}
+          _ -> {:ignore, nil}
+        end
+      end)
+    else
+      :error
+    end
+  end
+
+  def get_by_url(nil), do: nil
+
+  @spec get_or_backfill_by_url(String.t(), map()) :: t() | nil
+  def get_or_backfill_by_url(url, backfill_opts \\ %{}) do
+    case get_by_url(url) do
+      %__MODULE__{} = card ->
+        card
+
+      nil ->
+        backfill_opts = Map.put(backfill_opts, :url, url)
+
+        Backfill.start(backfill_opts)
+
+        nil
+
+      :error ->
+        nil
+    end
+  end
+
+  @spec get_by_object(Object.t()) :: t() | nil | :error
+  def get_by_object(object) do
+    case HTML.extract_first_external_url_from_object(object) do
+      nil -> nil
+      url -> get_or_backfill_by_url(url)
+    end
+  end
+
+  @spec get_by_activity(Activity.t()) :: t() | nil | :error
+  # Fake/Draft activity
+  def get_by_activity(%Activity{id: "pleroma:fakeid"} = activity) do
+    with %Object{} = object <- Object.normalize(activity, fetch: false),
+         url when not is_nil(url) <- HTML.extract_first_external_url_from_object(object) do
+      case get_by_url(url) do
+        # Cache hit
+        %__MODULE__{} = card ->
+          card
+
+        # Cache miss, but fetch for rendering the Draft
+        _ ->
+          with {:ok, fields} <- Parser.parse(url),
+               {:ok, card} <- create(url, fields) do
+            card
+          else
+            _ -> nil
+          end
+      end
+    else
+      _ ->
+        nil
+    end
+  end
+
+  def get_by_activity(activity) do
+    with %Object{} = object <- Object.normalize(activity, fetch: false),
+         {_, nil} <- {:cached, get_cached_url(object, activity.id)} do
+      nil
+    else
+      {:cached, url} ->
+        get_or_backfill_by_url(url, %{activity_id: activity.id})
+
+      _ ->
+        :error
+    end
+  end
+
+  @spec url_to_hash(String.t()) :: String.t()
+  def url_to_hash(url) do
+    :crypto.hash(:sha256, url) |> Base.encode16(case: :lower)
+  end
+
+  defp get_cached_url(object, activity_id) do
+    key = "URL|#{activity_id}"
+
+    @cachex.fetch!(:scrubber_cache, key, fn _ ->
+      url = HTML.extract_first_external_url_from_object(object)
+      Activity.HTML.add_cache_key_for(activity_id, key)
+
+      {:commit, url}
+    end)
+  end
+end

lib/pleroma/web/rich_media/helpers.ex

@@ -3,65 +3,13 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.RichMedia.Helpers do
-  alias Pleroma.Activity
-  alias Pleroma.HTML
-  alias Pleroma.Object
-  alias Pleroma.Web.RichMedia.Parser
-
-  @cachex Pleroma.Config.get([:cachex, :provider], Cachex)
-
-  @config_impl Application.compile_env(:pleroma, [__MODULE__, :config_impl], Pleroma.Config)
-
-  @options [
-    pool: :media,
-    max_body: 2_000_000,
-    recv_timeout: 2_000
-  ]
-
-  def fetch_data_for_object(object) do
-    with true <- @config_impl.get([:rich_media, :enabled]),
-         {:ok, page_url} <-
-           HTML.extract_first_external_url_from_object(object),
-         {:ok, rich_media} <- Parser.parse(page_url) do
-      %{page_url: page_url, rich_media: rich_media}
-    else
-      _ -> %{}
-    end
-  end
-
-  def fetch_data_for_activity(%Activity{data: %{"type" => "Create"}} = activity) do
-    with true <- @config_impl.get([:rich_media, :enabled]),
-         %Object{} = object <- Object.normalize(activity, fetch: false) do
-      if object.data["fake"] do
-        fetch_data_for_object(object)
-      else
-        key = "URL|#{activity.id}"
-
-        @cachex.fetch!(:scrubber_cache, key, fn _ ->
-          result = fetch_data_for_object(object)
-
-          cond do
-            match?(%{page_url: _, rich_media: _}, result) ->
-              Activity.HTML.add_cache_key_for(activity.id, key)
-              {:commit, result}
-
-            true ->
-              {:ignore, %{}}
-          end
-        end)
-      end
-    else
-      _ -> %{}
-    end
-  end
-
-  def fetch_data_for_activity(_), do: %{}
+  alias Pleroma.Config
 
   def rich_media_get(url) do
     headers = [{"user-agent", Pleroma.Application.user_agent() <> "; Bot"}]
 
     head_check =
-      case Pleroma.HTTP.head(url, headers, @options) do
+      case Pleroma.HTTP.head(url, headers, http_options()) do
         # If the HEAD request didn't reach the server for whatever reason,
         # we assume the GET that comes right after won't either
         {:error, _} = e ->
@@ -76,7 +24,7 @@ defmodule Pleroma.Web.RichMedia.Helpers do
           :ok
       end
 
-    with :ok <- head_check, do: Pleroma.HTTP.get(url, headers, @options)
+    with :ok <- head_check, do: Pleroma.HTTP.get(url, headers, http_options())
   end
 
   defp check_content_type(headers) do
@@ -92,12 +40,13 @@ defmodule Pleroma.Web.RichMedia.Helpers do
     end
   end
 
-  @max_body @options[:max_body]
   defp check_content_length(headers) do
+    max_body = Keyword.get(http_options(), :max_body)
+
     case List.keyfind(headers, "content-length", 0) do
       {_, maybe_content_length} ->
         case Integer.parse(maybe_content_length) do
-          {content_length, ""} when content_length <= @max_body -> :ok
+          {content_length, ""} when content_length <= max_body -> :ok
           {_, ""} -> {:error, :body_too_large}
           _ -> :ok
         end
@@ -106,4 +55,11 @@ defmodule Pleroma.Web.RichMedia.Helpers do
         :ok
     end
   end
+
+  defp http_options do
+    [
+      pool: :media,
+      max_body: Config.get([:rich_media, :max_body], 5_000_000)
+    ]
+  end
 end

lib/pleroma/web/rich_media/parser.ex

@@ -5,134 +5,28 @@
 defmodule Pleroma.Web.RichMedia.Parser do
   require Logger
 
-  @cachex Pleroma.Config.get([:cachex, :provider], Cachex)
   @config_impl Application.compile_env(:pleroma, [__MODULE__, :config_impl], Pleroma.Config)
 
   defp parsers do
     Pleroma.Config.get([:rich_media, :parsers])
   end
 
-  def parse(nil), do: {:error, "No URL provided"}
+  def parse(nil), do: nil
 
   @spec parse(String.t()) :: {:ok, map()} | {:error, any()}
   def parse(url) do
     with :ok <- validate_page_url(url),
-         {:ok, data} <- get_cached_or_parse(url),
-         {:ok, _} <- set_ttl_based_on_image(data, url) do
+         {:ok, data} <- parse_url(url) do
+      data = Map.put(data, "url", url)
       {:ok, data}
     end
   end
 
-  defp get_cached_or_parse(url) do
-    case @cachex.fetch(:rich_media_cache, url, fn ->
-           case parse_url(url) do
-             {:ok, _} = res ->
-               {:commit, res}
-
-             {:error, reason} = e ->
-               # Unfortunately we have to log errors here, instead of doing that
-               # along with ttl setting at the bottom. Otherwise we can get log spam
-               # if more than one process was waiting for the rich media card
-               # while it was generated. Ideally we would set ttl here as well,
-               # so we don't override it number_of_waiters_on_generation
-               # times, but one, obviously, can't set ttl for not-yet-created entry
-               # and Cachex doesn't support returning ttl from the fetch callback.
-               log_error(url, reason)
-               {:commit, e}
-           end
-         end) do
-      {action, res} when action in [:commit, :ok] ->
-        case res do
-          {:ok, _data} = res ->
-            res
-
-          {:error, reason} = e ->
-            if action == :commit, do: set_error_ttl(url, reason)
-            e
-        end
-
-      {:error, e} ->
-        {:error, {:cachex_error, e}}
-    end
-  end
-
-  defp set_error_ttl(_url, :body_too_large), do: :ok
-  defp set_error_ttl(_url, {:content_type, _}), do: :ok
-
-  # The TTL is not set for the errors above, since they are unlikely to change
-  # with time
-
-  defp set_error_ttl(url, _reason) do
-    ttl = Pleroma.Config.get([:rich_media, :failure_backoff], 60_000)
-    @cachex.expire(:rich_media_cache, url, ttl)
-    :ok
-  end
-
-  defp log_error(url, {:invalid_metadata, data}) do
-    Logger.debug(fn -> "Incomplete or invalid metadata for #{url}: #{inspect(data)}" end)
-  end
-
-  defp log_error(url, reason) do
-    Logger.warning(fn -> "Rich media error for #{url}: #{inspect(reason)}" end)
-  end
-
-  @doc """
-  Set the rich media cache based on the expiration time of image.
-
-  Adopt behaviour `Pleroma.Web.RichMedia.Parser.TTL`
-
-  ## Example
-
-      defmodule MyModule do
-        @behaviour Pleroma.Web.RichMedia.Parser.TTL
-        def ttl(data, url) do
-          image_url = Map.get(data, :image)
-          # do some parsing in the url and get the ttl of the image
-          # and return ttl is unix time
-          parse_ttl_from_url(image_url)
-        end
-      end
-
-  Define the module in the config
-
-      config :pleroma, :rich_media,
-        ttl_setters: [MyModule]
-  """
-  @spec set_ttl_based_on_image(map(), String.t()) ::
-          {:ok, integer() | :noop} | {:error, :no_key}
-  def set_ttl_based_on_image(data, url) do
-    case get_ttl_from_image(data, url) do
-      ttl when is_number(ttl) ->
-        ttl = ttl * 1000
-
-        case @cachex.expire_at(:rich_media_cache, url, ttl) do
-          {:ok, true} -> {:ok, ttl}
-          {:ok, false} -> {:error, :no_key}
-        end
-
-      _ ->
-        {:ok, :noop}
-    end
-  end
-
-  defp get_ttl_from_image(data, url) do
-    [:rich_media, :ttl_setters]
-    |> Pleroma.Config.get()
-    |> Enum.reduce({:ok, nil}, fn
-      module, {:ok, _ttl} ->
-        module.ttl(data, url)
-
-      _, error ->
-        error
-    end)
-  end
-
-  def parse_url(url) do
+  defp parse_url(url) do
     with {:ok, %Tesla.Env{body: html}} <- Pleroma.Web.RichMedia.Helpers.rich_media_get(url),
          {:ok, html} <- Floki.parse_document(html) do
       html
       |> maybe_parse()
-      |> Map.put("url", url)
       |> clean_parsed_data()
       |> check_parsed_data()
     end

lib/pleroma/web/rich_media/parser/ttl.ex

@@ -4,4 +4,17 @@
 
 defmodule Pleroma.Web.RichMedia.Parser.TTL do
   @callback ttl(map(), String.t()) :: integer() | nil
+
+  @spec process(map(), String.t()) :: {:ok, integer() | nil}
+  def process(data, url) do
+    [:rich_media, :ttl_setters]
+    |> Pleroma.Config.get()
+    |> Enum.reduce_while({:ok, nil}, fn
+      module, acc ->
+        case module.ttl(data, url) do
+          ttl when is_number(ttl) -> {:halt, {:ok, ttl}}
+          _ -> {:cont, acc}
+        end
+    end)
+  end
 end

lib/pleroma/web/rich_media/parser/ttl/aws_signed_url.ex

@@ -7,7 +7,7 @@ defmodule Pleroma.Web.RichMedia.Parser.TTL.AwsSignedUrl do
 
   @impl true
   def ttl(data, _url) do
-    image = Map.get(data, :image)
+    image = Map.get(data, "image")
 
     if aws_signed_url?(image) do
       image
@@ -15,14 +15,15 @@ defmodule Pleroma.Web.RichMedia.Parser.TTL.AwsSignedUrl do
       |> format_query_params()
       |> get_expiration_timestamp()
     else
-      {:error, "Not aws signed url #{inspect(image)}"}
+      nil
     end
   end
 
   defp aws_signed_url?(image) when is_binary(image) and image != "" do
     %URI{host: host, query: query} = URI.parse(image)
 
-    String.contains?(host, "amazonaws.com") and String.contains?(query, "X-Amz-Expires")
+    is_binary(host) and String.contains?(host, "amazonaws.com") and
+      String.contains?(query, "X-Amz-Expires")
   end
 
   defp aws_signed_url?(_), do: nil

lib/pleroma/web/rich_media/parser/ttl/opengraph.ex

@@ -0,0 +1,20 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.RichMedia.Parser.TTL.Opengraph do
+  @behaviour Pleroma.Web.RichMedia.Parser.TTL
+
+  @impl true
+  def ttl(%{"ttl" => ttl_string}, _url) when is_binary(ttl_string) do
+    try do
+      ttl = String.to_integer(ttl_string)
+      now = DateTime.utc_now() |> DateTime.to_unix()
+      now + ttl
+    rescue
+      _ -> nil
+    end
+  end
+
+  def ttl(_, _), do: nil
+end

lib/pleroma/web/router.ex

@@ -292,6 +292,11 @@ defmodule Pleroma.Web.Router do
     post("/frontends/install", FrontendController, :install)
 
     post("/backups", AdminAPIController, :create_backup)
+
+    get("/rules", RuleController, :index)
+    post("/rules", RuleController, :create)
+    patch("/rules/:id", RuleController, :update)
+    delete("/rules/:id", RuleController, :delete)
   end
 
   # AdminAPI: admins and mods (staff) can perform these actions (if privileged by role)
@@ -633,6 +638,7 @@ defmodule Pleroma.Web.Router do
     patch("/accounts/update_credentials", AccountController, :update_credentials)
 
     get("/accounts/relationships", AccountController, :relationships)
+    get("/accounts/familiar_followers", AccountController, :familiar_followers)
     get("/accounts/:id/lists", AccountController, :lists)
     get("/accounts/:id/identity_proofs", AccountController, :identity_proofs)
     get("/endorsements", AccountController, :endorsements)
@@ -764,11 +770,11 @@ defmodule Pleroma.Web.Router do
 
     get("/instance", InstanceController, :show)
     get("/instance/peers", InstanceController, :peers)
+    get("/instance/rules", InstanceController, :rules)
 
     get("/statuses", StatusController, :index)
     get("/statuses/:id", StatusController, :show)
     get("/statuses/:id/context", StatusController, :context)
-    get("/statuses/:id/card", StatusController, :card)
     get("/statuses/:id/favourited_by", StatusController, :favourited_by)
     get("/statuses/:id/reblogged_by", StatusController, :reblogged_by)
     get("/statuses/:id/history", StatusController, :show_history)

lib/pleroma/web/web_finger.ex

@@ -155,7 +155,16 @@ defmodule Pleroma.Web.WebFinger do
     end
   end
 
+  @cachex Pleroma.Config.get([:cachex, :provider], Cachex)
   def find_lrdd_template(domain) do
+    @cachex.fetch!(:host_meta_cache, domain, fn _ ->
+      {:commit, fetch_lrdd_template(domain)}
+    end)
+  rescue
+    e -> {:error, "Cachex error: #{inspect(e)}"}
+  end
+
+  defp fetch_lrdd_template(domain) do
     # WebFinger is restricted to HTTPS - https://tools.ietf.org/html/rfc7033#section-9.1
     meta_url = "https://#{domain}/.well-known/host-meta"
 
@@ -168,7 +177,7 @@ defmodule Pleroma.Web.WebFinger do
     end
   end
 
-  defp get_address_from_domain(domain, encoded_account) when is_binary(domain) do
+  defp get_address_from_domain(domain, "acct:" <> _ = encoded_account) when is_binary(domain) do
     case find_lrdd_template(domain) do
       {:ok, template} ->
         String.replace(template, "{uri}", encoded_account)
@@ -178,6 +187,11 @@ defmodule Pleroma.Web.WebFinger do
     end
   end
 
+  defp get_address_from_domain(domain, account) when is_binary(domain) do
+    encoded_account = URI.encode("acct:#{account}")
+    get_address_from_domain(domain, encoded_account)
+  end
+
   defp get_address_from_domain(_, _), do: {:error, :webfinger_no_domain}
 
   @spec finger(String.t()) :: {:ok, map()} | {:error, any()}
@@ -192,9 +206,7 @@ defmodule Pleroma.Web.WebFinger do
           URI.parse(account).host
       end
 
-    encoded_account = URI.encode("acct:#{account}")
-
-    with address when is_binary(address) <- get_address_from_domain(domain, encoded_account),
+    with address when is_binary(address) <- get_address_from_domain(domain, account),
          {:ok, %{status: status, body: body, headers: headers}} when status in 200..299 <-
            HTTP.get(
              address,
@@ -216,10 +228,28 @@ defmodule Pleroma.Web.WebFinger do
         _ ->
           {:error, {:content_type, nil}}
       end
+      |> case do
+        {:ok, data} -> validate_webfinger(address, data)
+        error -> error
+      end
     else
       error ->
         Logger.debug("Couldn't finger #{account}: #{inspect(error)}")
         error
     end
   end
+
+  defp validate_webfinger(request_url, %{"subject" => "acct:" <> acct = subject} = data) do
+    with [_name, acct_host] <- String.split(acct, "@"),
+         {_, url} <- {:address, get_address_from_domain(acct_host, subject)},
+         %URI{host: request_host} <- URI.parse(request_url),
+         %URI{host: acct_host} <- URI.parse(url),
+         {_, true} <- {:hosts_match, acct_host == request_host} do
+      {:ok, data}
+    else
+      _ -> {:error, {:webfinger_invalid, request_url, data}}
+    end
+  end
+
+  defp validate_webfinger(url, data), do: {:error, {:webfinger_invalid, url, data}}
 end

lib/pleroma/workers/receiver_worker.ex

@@ -52,7 +52,8 @@ defmodule Pleroma.Workers.ReceiverWorker do
       {:error, {:reject, reason}} -> {:cancel, reason}
       {:signature, false} -> {:cancel, :invalid_signature}
       {:error, {:error, reason = "Object has been deleted"}} -> {:cancel, reason}
-      e -> e
+      {:error, _} = e -> e
+      e -> {:error, e}
     end
   end
 end

lib/pleroma/workers/rich_media_expiration_worker.ex

@@ -0,0 +1,15 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Workers.RichMediaExpirationWorker do
+  alias Pleroma.Web.RichMedia.Card
+
+  use Oban.Worker,
+    queue: :rich_media_expiration
+
+  @impl Oban.Worker
+  def perform(%Job{args: %{"url" => url} = _args}) do
+    Card.delete(url)
+  end
+end

priv/repo/migrations/20220203224011_create_rules.exs

@@ -0,0 +1,12 @@
+defmodule Pleroma.Repo.Migrations.CreateRules do
+  use Ecto.Migration
+
+  def change do
+    create_if_not_exists table(:rules) do
+      add(:priority, :integer, default: 0, null: false)
+      add(:text, :text, null: false)
+
+      timestamps()
+    end
+  end
+end

priv/repo/migrations/20220319000000_add_status_to_notifications_enum.exs

@@ -0,0 +1,51 @@
+defmodule Pleroma.Repo.Migrations.AddStatusToNotificationsEnum do
+  use Ecto.Migration
+
+  @disable_ddl_transaction true
+
+  def up do
+    """
+    alter type notification_type add value 'status'
+    """
+    |> execute()
+  end
+
+  def down do
+    alter table(:notifications) do
+      modify(:type, :string)
+    end
+
+    """
+    delete from notifications where type = 'status'
+    """
+    |> execute()
+
+    """
+    drop type if exists notification_type
+    """
+    |> execute()
+
+    """
+    create type notification_type as enum (
+      'follow',
+      'follow_request',
+      'mention',
+      'move',
+      'pleroma:emoji_reaction',
+      'pleroma:chat_mention',
+      'reblog',
+      'favourite',
+      'pleroma:report',
+      'poll',
+      'update'
+    )
+    """
+    |> execute()
+
+    """
+    alter table notifications
+    alter column type type notification_type using (type::notification_type)
+    """
+    |> execute()
+  end
+end

priv/repo/migrations/20240207035927_create_rich_media_card.exs

@@ -0,0 +1,14 @@
+defmodule Pleroma.Repo.Migrations.CreateRichMediaCard do
+  use Ecto.Migration
+
+  def change do
+    create table(:rich_media_card) do
+      add(:url_hash, :bytea)
+      add(:fields, :map)
+
+      timestamps()
+    end
+
+    create(unique_index(:rich_media_card, [:url_hash]))
+  end
+end

priv/repo/migrations/20240406000000_add_hint_to_rules.exs

@@ -0,0 +1,13 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2024 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Repo.Migrations.AddHintToRules do
+  use Ecto.Migration
+
+  def change do
+    alter table(:rules) do
+      add_if_not_exists(:hint, :text)
+    end
+  end
+end

priv/static/schemas/litepub-0.1.jsonld

@@ -2,6 +2,7 @@
     "@context": [
         "https://www.w3.org/ns/activitystreams",
         "https://w3id.org/security/v1",
+        "https://purl.archive.org/socialweb/webfinger",
         {
             "Emoji": "toot:Emoji",
             "Hashtag": "as:Hashtag",

test/fixtures/tesla_mock/gleasonator.com_host_meta

@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<XRD xmlns="http://docs.oasis-open.org/ns/xri/xrd-1.0">
+  <Link rel="lrdd" template="https://gleasonator.com/.well-known/webfinger?resource={uri}" type="application/xrd+xml" />
+</XRD>

test/fixtures/tesla_mock/webfinger_spoof.json

@@ -0,0 +1,28 @@
+{
+  "aliases": [
+    "https://gleasonator.com/users/alex",
+    "https://mitra.social/users/alex"
+  ],
+  "links": [
+    {
+      "href": "https://gleasonator.com/users/alex",
+      "rel": "http://webfinger.net/rel/profile-page",
+      "type": "text/html"
+    },
+    {
+      "href": "https://gleasonator.com/users/alex",
+      "rel": "self",
+      "type": "application/activity+json"
+    },
+    {
+      "href": "https://gleasonator.com/users/alex",
+      "rel": "self",
+      "type": "application/ld+json; profile=\"https://www.w3.org/ns/activitystreams\""
+    },
+    {
+      "rel": "http://ostatus.org/schema/1.0/subscribe",
+      "template": "https://gleasonator.com/ostatus_subscribe?acct={uri}"
+    }
+  ],
+  "subject": "acct:trump@whitehouse.gov"
+}

test/fixtures/webfinger/graf-imposter-webfinger.json

@@ -0,0 +1,41 @@
+{
+    "subject": "acct:graf@poa.st",
+    "aliases": [
+        "https://fba.ryona.agenc/webfingertest"
+    ],
+    "links": [
+        {
+            "rel": "http://webfinger.net/rel/profile-page",
+            "type": "text/html",
+            "href": "https://fba.ryona.agenc/webfingertest"
+        },
+        {
+            "rel": "self",
+            "type": "application/activity+json",
+            "href": "https://fba.ryona.agenc/webfingertest"
+        },
+        {
+            "rel": "http://ostatus.org/schema/1.0/subscribe",
+            "template": "https://fba.ryona.agenc/contact/follow?url={uri}"
+        },
+        {
+            "rel": "http://schemas.google.com/g/2010#updates-from",
+            "type": "application/atom+xml",
+            "href": ""
+        },
+        {
+            "rel": "salmon",
+            "href": "https://fba.ryona.agenc/salmon/friendica"
+        },
+        {
+            "rel": "http://microformats.org/profile/hcard",
+            "type": "text/html",
+            "href": "https://fba.ryona.agenc/hcard/friendica"
+        },
+        {
+            "rel": "http://joindiaspora.com/seed_location",
+            "type": "text/html",
+            "href": "https://fba.ryona.agenc"
+        }
+    ]
+}

test/pleroma/html_test.exs

@@ -202,7 +202,7 @@ defmodule Pleroma.HTMLTest do
         })
 
       object = Object.normalize(activity, fetch: false)
-      {:ok, url} = HTML.extract_first_external_url_from_object(object)
+      url = HTML.extract_first_external_url_from_object(object)
       assert url == "https://github.com/komeiji-satori/Dress"
     end
 
@@ -217,7 +217,7 @@ defmodule Pleroma.HTMLTest do
         })
 
       object = Object.normalize(activity, fetch: false)
-      {:ok, url} = HTML.extract_first_external_url_from_object(object)
+      url = HTML.extract_first_external_url_from_object(object)
 
       assert url == "https://github.com/syuilo/misskey/blob/develop/docs/setup.en.md"
 
@@ -233,7 +233,7 @@ defmodule Pleroma.HTMLTest do
         })
 
       object = Object.normalize(activity, fetch: false)
-      {:ok, url} = HTML.extract_first_external_url_from_object(object)
+      url = HTML.extract_first_external_url_from_object(object)
 
       assert url == "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=72255140"
     end
@@ -249,7 +249,7 @@ defmodule Pleroma.HTMLTest do
         })
 
       object = Object.normalize(activity, fetch: false)
-      {:ok, url} = HTML.extract_first_external_url_from_object(object)
+      url = HTML.extract_first_external_url_from_object(object)
 
       assert url == "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=72255140"
     end
@@ -261,7 +261,7 @@ defmodule Pleroma.HTMLTest do
 
       object = Object.normalize(activity, fetch: false)
 
-      assert {:ok, nil} = HTML.extract_first_external_url_from_object(object)
+      assert nil == HTML.extract_first_external_url_from_object(object)
     end
 
     test "skips attachment links" do
@@ -275,7 +275,7 @@ defmodule Pleroma.HTMLTest do
 
       object = Object.normalize(activity, fetch: false)
 
-      assert {:ok, nil} = HTML.extract_first_external_url_from_object(object)
+      assert nil == HTML.extract_first_external_url_from_object(object)
     end
   end
 end

test/pleroma/notification_test.exs

@@ -6,7 +6,6 @@ defmodule Pleroma.NotificationTest do
   use Pleroma.DataCase, async: false
 
   import Pleroma.Factory
-  import Mock
 
   alias Pleroma.FollowingRelationship
   alias Pleroma.Notification
@@ -18,8 +17,6 @@ defmodule Pleroma.NotificationTest do
   alias Pleroma.Web.ActivityPub.Transmogrifier
   alias Pleroma.Web.CommonAPI
   alias Pleroma.Web.MastodonAPI.NotificationView
-  alias Pleroma.Web.Push
-  alias Pleroma.Web.Streamer
 
   setup do
     Mox.stub_with(Pleroma.UnstubbedConfigMock, Pleroma.Config)
@@ -115,6 +112,7 @@ defmodule Pleroma.NotificationTest do
       {:ok, [notification]} = Notification.create_notifications(status)
 
       assert notification.user_id == subscriber.id
+      assert notification.type == "status"
     end
 
     test "does not create a notification for subscribed users if status is a reply" do
@@ -139,6 +137,21 @@ defmodule Pleroma.NotificationTest do
       assert Enum.empty?(subscriber_notifications)
     end
 
+    test "does not create subscriber notification if mentioned" do
+      user = insert(:user)
+      subscriber = insert(:user)
+
+      User.subscribe(subscriber, user)
+
+      {:ok, status} = CommonAPI.post(user, %{status: "mentioning @#{subscriber.nickname}"})
+      {:ok, [notification] = notifications} = Notification.create_notifications(status)
+
+      assert length(notifications) == 1
+
+      assert notification.user_id == subscriber.id
+      assert notification.type == "mention"
+    end
+
     test "it sends edited notifications to those who repeated a status" do
       user = insert(:user)
       repeated_user = insert(:user)
@@ -175,158 +188,7 @@ defmodule Pleroma.NotificationTest do
     assert [user2.id, user3.id, user1.id] == Enum.map(notifications, & &1.user_id)
   end
 
-  describe "CommonApi.post/2 notification-related functionality" do
-    test_with_mock "creates but does NOT send notification to blocker user",
-                   Push,
-                   [:passthrough],
-                   [] do
-      user = insert(:user)
-      blocker = insert(:user)
-      {:ok, _user_relationship} = User.block(blocker, user)
-
-      {:ok, _activity} = CommonAPI.post(user, %{status: "hey @#{blocker.nickname}!"})
-
-      blocker_id = blocker.id
-      assert [%Notification{user_id: ^blocker_id}] = Repo.all(Notification)
-      refute called(Push.send(:_))
-    end
-
-    test_with_mock "creates but does NOT send notification to notification-muter user",
-                   Push,
-                   [:passthrough],
-                   [] do
-      user = insert(:user)
-      muter = insert(:user)
-      {:ok, _user_relationships} = User.mute(muter, user)
-
-      {:ok, _activity} = CommonAPI.post(user, %{status: "hey @#{muter.nickname}!"})
-
-      muter_id = muter.id
-      assert [%Notification{user_id: ^muter_id}] = Repo.all(Notification)
-      refute called(Push.send(:_))
-    end
-
-    test_with_mock "creates but does NOT send notification to thread-muter user",
-                   Push,
-                   [:passthrough],
-                   [] do
-      user = insert(:user)
-      thread_muter = insert(:user)
-
-      {:ok, activity} = CommonAPI.post(user, %{status: "hey @#{thread_muter.nickname}!"})
-
-      {:ok, _} = CommonAPI.add_mute(thread_muter, activity)
-
-      {:ok, _same_context_activity} =
-        CommonAPI.post(user, %{
-          status: "hey-hey-hey @#{thread_muter.nickname}!",
-          in_reply_to_status_id: activity.id
-        })
-
-      [pre_mute_notification, post_mute_notification] =
-        Repo.all(from(n in Notification, where: n.user_id == ^thread_muter.id, order_by: n.id))
-
-      pre_mute_notification_id = pre_mute_notification.id
-      post_mute_notification_id = post_mute_notification.id
-
-      assert called(
-               Push.send(
-                 :meck.is(fn
-                   %Notification{id: ^pre_mute_notification_id} -> true
-                   _ -> false
-                 end)
-               )
-             )
-
-      refute called(
-               Push.send(
-                 :meck.is(fn
-                   %Notification{id: ^post_mute_notification_id} -> true
-                   _ -> false
-                 end)
-               )
-             )
-    end
-  end
-
   describe "create_notification" do
-    @tag needs_streamer: true
-    test "it creates a notification for user and send to the 'user' and the 'user:notification' stream" do
-      %{user: user, token: oauth_token} = oauth_access(["read"])
-
-      task =
-        Task.async(fn ->
-          {:ok, _topic} = Streamer.get_topic_and_add_socket("user", user, oauth_token)
-          assert_receive {:render_with_user, _, _, _, _}, 4_000
-        end)
-
-      task_user_notification =
-        Task.async(fn ->
-          {:ok, _topic} =
-            Streamer.get_topic_and_add_socket("user:notification", user, oauth_token)
-
-          assert_receive {:render_with_user, _, _, _, _}, 4_000
-        end)
-
-      activity = insert(:note_activity)
-
-      notify = Notification.create_notification(activity, user)
-      assert notify.user_id == user.id
-      Task.await(task)
-      Task.await(task_user_notification)
-    end
-
-    test "it creates a notification for user if the user blocks the activity author" do
-      activity = insert(:note_activity)
-      author = User.get_cached_by_ap_id(activity.data["actor"])
-      user = insert(:user)
-      {:ok, _user_relationship} = User.block(user, author)
-
-      assert Notification.create_notification(activity, user)
-    end
-
-    test "it creates a notification for the user if the user mutes the activity author" do
-      muter = insert(:user)
-      muted = insert(:user)
-      {:ok, _} = User.mute(muter, muted)
-      muter = Repo.get(User, muter.id)
-      {:ok, activity} = CommonAPI.post(muted, %{status: "Hi @#{muter.nickname}"})
-
-      notification = Notification.create_notification(activity, muter)
-
-      assert notification.id
-      assert notification.seen
-    end
-
-    test "notification created if user is muted without notifications" do
-      muter = insert(:user)
-      muted = insert(:user)
-
-      {:ok, _user_relationships} = User.mute(muter, muted, %{notifications: false})
-
-      {:ok, activity} = CommonAPI.post(muted, %{status: "Hi @#{muter.nickname}"})
-
-      assert Notification.create_notification(activity, muter)
-    end
-
-    test "it creates a notification for an activity from a muted thread" do
-      muter = insert(:user)
-      other_user = insert(:user)
-      {:ok, activity} = CommonAPI.post(muter, %{status: "hey"})
-      CommonAPI.add_mute(muter, activity)
-
-      {:ok, activity} =
-        CommonAPI.post(other_user, %{
-          status: "Hi @#{muter.nickname}",
-          in_reply_to_status_id: activity.id
-        })
-
-      notification = Notification.create_notification(activity, muter)
-
-      assert notification.id
-      assert notification.seen
-    end
-
     test "it disables notifications from strangers" do
       follower = insert(:user)
 
@@ -603,9 +465,7 @@ defmodule Pleroma.NotificationTest do
           status: "hey yet again @#{other_user.nickname}!"
         })
 
-      [_, read_notification] = Notification.set_read_up_to(other_user, n2.id)
-
-      assert read_notification.activity.object
+      Notification.set_read_up_to(other_user, n2.id)
 
       [n3, n2, n1] = Notification.for_user(other_user)
 
@@ -680,7 +540,7 @@ defmodule Pleroma.NotificationTest do
           status: "hey @#{other_user.nickname}!"
         })
 
-      {enabled_receivers, _disabled_receivers} = Notification.get_notified_from_activity(activity)
+      enabled_receivers = Notification.get_notified_from_activity(activity)
 
       assert other_user in enabled_receivers
     end
@@ -712,7 +572,7 @@ defmodule Pleroma.NotificationTest do
 
       {:ok, activity} = Transmogrifier.handle_incoming(create_activity)
 
-      {enabled_receivers, _disabled_receivers} = Notification.get_notified_from_activity(activity)
+      enabled_receivers = Notification.get_notified_from_activity(activity)
 
       assert other_user in enabled_receivers
     end
@@ -739,7 +599,7 @@ defmodule Pleroma.NotificationTest do
 
       {:ok, activity} = Transmogrifier.handle_incoming(create_activity)
 
-      {enabled_receivers, _disabled_receivers} = Notification.get_notified_from_activity(activity)
+      enabled_receivers = Notification.get_notified_from_activity(activity)
 
       assert other_user not in enabled_receivers
     end
@@ -756,8 +616,7 @@ defmodule Pleroma.NotificationTest do
 
       {:ok, activity_two} = CommonAPI.favorite(third_user, activity_one.id)
 
-      {enabled_receivers, _disabled_receivers} =
-        Notification.get_notified_from_activity(activity_two)
+      enabled_receivers = Notification.get_notified_from_activity(activity_two)
 
       assert other_user not in enabled_receivers
     end
@@ -779,7 +638,7 @@ defmodule Pleroma.NotificationTest do
         |> Map.put("to", [other_user.ap_id | like_data["to"]])
         |> ActivityPub.persist(local: true)
 
-      {enabled_receivers, _disabled_receivers} = Notification.get_notified_from_activity(like)
+      enabled_receivers = Notification.get_notified_from_activity(like)
 
       assert other_user not in enabled_receivers
     end
@@ -796,39 +655,36 @@ defmodule Pleroma.NotificationTest do
 
       {:ok, activity_two} = CommonAPI.repeat(activity_one.id, third_user)
 
-      {enabled_receivers, _disabled_receivers} =
-        Notification.get_notified_from_activity(activity_two)
+      enabled_receivers = Notification.get_notified_from_activity(activity_two)
 
       assert other_user not in enabled_receivers
     end
 
-    test "it returns blocking recipient in disabled recipients list" do
+    test "it does not return blocking recipient in recipients list" do
       user = insert(:user)
       other_user = insert(:user)
       {:ok, _user_relationship} = User.block(other_user, user)
 
       {:ok, activity} = CommonAPI.post(user, %{status: "hey @#{other_user.nickname}!"})
 
-      {enabled_receivers, disabled_receivers} = Notification.get_notified_from_activity(activity)
+      enabled_receivers = Notification.get_notified_from_activity(activity)
 
       assert [] == enabled_receivers
-      assert [other_user] == disabled_receivers
     end
 
-    test "it returns notification-muting recipient in disabled recipients list" do
+    test "it does not return notification-muting recipient in recipients list" do
       user = insert(:user)
       other_user = insert(:user)
       {:ok, _user_relationships} = User.mute(other_user, user)
 
       {:ok, activity} = CommonAPI.post(user, %{status: "hey @#{other_user.nickname}!"})
 
-      {enabled_receivers, disabled_receivers} = Notification.get_notified_from_activity(activity)
+      enabled_receivers = Notification.get_notified_from_activity(activity)
 
       assert [] == enabled_receivers
-      assert [other_user] == disabled_receivers
     end
 
-    test "it returns thread-muting recipient in disabled recipients list" do
+    test "it does not return thread-muting recipient in recipients list" do
       user = insert(:user)
       other_user = insert(:user)
 
@@ -842,14 +698,12 @@ defmodule Pleroma.NotificationTest do
           in_reply_to_status_id: activity.id
         })
 
-      {enabled_receivers, disabled_receivers} =
-        Notification.get_notified_from_activity(same_context_activity)
+      enabled_receivers = Notification.get_notified_from_activity(same_context_activity)
 
-      assert [other_user] == disabled_receivers
       refute other_user in enabled_receivers
     end
 
-    test "it returns non-following domain-blocking recipient in disabled recipients list" do
+    test "it does not return non-following domain-blocking recipient in recipients list" do
       blocked_domain = "blocked.domain"
       user = insert(:user, %{ap_id: "https://#{blocked_domain}/@actor"})
       other_user = insert(:user)
@@ -858,10 +712,9 @@ defmodule Pleroma.NotificationTest do
 
       {:ok, activity} = CommonAPI.post(user, %{status: "hey @#{other_user.nickname}!"})
 
-      {enabled_receivers, disabled_receivers} = Notification.get_notified_from_activity(activity)
+      enabled_receivers = Notification.get_notified_from_activity(activity)
 
       assert [] == enabled_receivers
-      assert [other_user] == disabled_receivers
     end
 
     test "it returns following domain-blocking recipient in enabled recipients list" do
@@ -874,10 +727,9 @@ defmodule Pleroma.NotificationTest do
 
       {:ok, activity} = CommonAPI.post(user, %{status: "hey @#{other_user.nickname}!"})
 
-      {enabled_receivers, disabled_receivers} = Notification.get_notified_from_activity(activity)
+      enabled_receivers = Notification.get_notified_from_activity(activity)
 
       assert [other_user] == enabled_receivers
-      assert [] == disabled_receivers
     end
 
     test "it sends edited notifications to those who repeated a status" do
@@ -897,11 +749,10 @@ defmodule Pleroma.NotificationTest do
           status: "hey @#{other_user.nickname}! mew mew"
         })
 
-      {enabled_receivers, _disabled_receivers} =
-        Notification.get_notified_from_activity(edit_activity)
+      enabled_receivers = Notification.get_notified_from_activity(edit_activity)
 
       assert repeated_user in enabled_receivers
-      assert other_user not in enabled_receivers
+      refute other_user in enabled_receivers
     end
   end
 
@@ -1189,13 +1040,13 @@ defmodule Pleroma.NotificationTest do
       assert Notification.for_user(user) == []
     end
 
-    test "it returns notifications from a muted user when with_muted is set", %{user: user} do
+    test "it doesn't return notifications from a muted user when with_muted is set", %{user: user} do
       muted = insert(:user)
       {:ok, _user_relationships} = User.mute(user, muted)
 
       {:ok, _activity} = CommonAPI.post(muted, %{status: "hey @#{user.nickname}"})
 
-      assert length(Notification.for_user(user, %{with_muted: true})) == 1
+      assert Enum.empty?(Notification.for_user(user, %{with_muted: true}))
     end
 
     test "it doesn't return notifications from a blocked user when with_muted is set", %{

test/pleroma/rule_test.exs

@@ -0,0 +1,57 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.RuleTest do
+  use Pleroma.DataCase, async: true
+
+  alias Pleroma.Repo
+  alias Pleroma.Rule
+
+  test "getting a list of rules sorted by priority" do
+    %{id: id1} = Rule.create(%{text: "Example rule"})
+    %{id: id2} = Rule.create(%{text: "Second rule", priority: 2})
+    %{id: id3} = Rule.create(%{text: "Third rule", priority: 1})
+
+    rules =
+      Rule.query()
+      |> Repo.all()
+
+    assert [%{id: ^id1}, %{id: ^id3}, %{id: ^id2}] = rules
+  end
+
+  test "creating rules" do
+    %{id: id} = Rule.create(%{text: "Example rule"})
+
+    assert %{text: "Example rule"} = Rule.get(id)
+  end
+
+  test "editing rules" do
+    %{id: id} = Rule.create(%{text: "Example rule"})
+
+    Rule.update(%{text: "There are no rules", priority: 2}, id)
+
+    assert %{text: "There are no rules", priority: 2} = Rule.get(id)
+  end
+
+  test "deleting rules" do
+    %{id: id} = Rule.create(%{text: "Example rule"})
+
+    Rule.delete(id)
+
+    assert [] =
+             Rule.query()
+             |> Pleroma.Repo.all()
+  end
+
+  test "getting rules by ids" do
+    %{id: id1} = Rule.create(%{text: "Example rule"})
+    %{id: id2} = Rule.create(%{text: "Second rule"})
+    %{id: _id3} = Rule.create(%{text: "Third rule"})
+
+    rules = Rule.get([id1, id2])
+
+    assert Enum.all?(rules, &(&1.id in [id1, id2]))
+    assert length(rules) == 2
+  end
+end

test/pleroma/user_test.exs

@@ -877,109 +877,19 @@ defmodule Pleroma.UserTest do
     setup do: clear_config([Pleroma.Web.WebFinger, :update_nickname_on_user_fetch], true)
 
     test "for mastodon" do
-      Tesla.Mock.mock(fn
-        %{url: "https://example.com/.well-known/host-meta"} ->
-          %Tesla.Env{
-            status: 302,
-            headers: [{"location", "https://sub.example.com/.well-known/host-meta"}]
-          }
-
-        %{url: "https://sub.example.com/.well-known/host-meta"} ->
-          %Tesla.Env{
-            status: 200,
-            body:
-              "test/fixtures/webfinger/masto-host-meta.xml"
-              |> File.read!()
-              |> String.replace("{{domain}}", "sub.example.com")
-          }
-
-        %{url: "https://sub.example.com/.well-known/webfinger?resource=acct:a@example.com"} ->
-          %Tesla.Env{
-            status: 200,
-            body:
-              "test/fixtures/webfinger/masto-webfinger.json"
-              |> File.read!()
-              |> String.replace("{{nickname}}", "a")
-              |> String.replace("{{domain}}", "example.com")
-              |> String.replace("{{subdomain}}", "sub.example.com"),
-            headers: [{"content-type", "application/jrd+json"}]
-          }
-
-        %{url: "https://sub.example.com/users/a"} ->
-          %Tesla.Env{
-            status: 200,
-            body:
-              "test/fixtures/webfinger/masto-user.json"
-              |> File.read!()
-              |> String.replace("{{nickname}}", "a")
-              |> String.replace("{{domain}}", "sub.example.com"),
-            headers: [{"content-type", "application/activity+json"}]
-          }
-
-        %{url: "https://sub.example.com/users/a/collections/featured"} ->
-          %Tesla.Env{
-            status: 200,
-            body:
-              File.read!("test/fixtures/users_mock/masto_featured.json")
-              |> String.replace("{{domain}}", "sub.example.com")
-              |> String.replace("{{nickname}}", "a"),
-            headers: [{"content-type", "application/activity+json"}]
-          }
-      end)
-
-      ap_id = "a@example.com"
+      ap_id = "a@mastodon.example"
       {:ok, fetched_user} = User.get_or_fetch(ap_id)
 
-      assert fetched_user.ap_id == "https://sub.example.com/users/a"
-      assert fetched_user.nickname == "a@example.com"
+      assert fetched_user.ap_id == "https://sub.mastodon.example/users/a"
+      assert fetched_user.nickname == "a@mastodon.example"
     end
 
     test "for pleroma" do
-      Tesla.Mock.mock(fn
-        %{url: "https://example.com/.well-known/host-meta"} ->
-          %Tesla.Env{
-            status: 302,
-            headers: [{"location", "https://sub.example.com/.well-known/host-meta"}]
-          }
-
-        %{url: "https://sub.example.com/.well-known/host-meta"} ->
-          %Tesla.Env{
-            status: 200,
-            body:
-              "test/fixtures/webfinger/pleroma-host-meta.xml"
-              |> File.read!()
-              |> String.replace("{{domain}}", "sub.example.com")
-          }
-
-        %{url: "https://sub.example.com/.well-known/webfinger?resource=acct:a@example.com"} ->
-          %Tesla.Env{
-            status: 200,
-            body:
-              "test/fixtures/webfinger/pleroma-webfinger.json"
-              |> File.read!()
-              |> String.replace("{{nickname}}", "a")
-              |> String.replace("{{domain}}", "example.com")
-              |> String.replace("{{subdomain}}", "sub.example.com"),
-            headers: [{"content-type", "application/jrd+json"}]
-          }
-
-        %{url: "https://sub.example.com/users/a"} ->
-          %Tesla.Env{
-            status: 200,
-            body:
-              "test/fixtures/webfinger/pleroma-user.json"
-              |> File.read!()
-              |> String.replace("{{nickname}}", "a")
-              |> String.replace("{{domain}}", "sub.example.com"),
-            headers: [{"content-type", "application/activity+json"}]
-          }
-      end)
-
-      ap_id = "a@example.com"
+      ap_id = "a@pleroma.example"
       {:ok, fetched_user} = User.get_or_fetch(ap_id)
 
-      assert fetched_user.ap_id == "https://sub.example.com/users/a"
-      assert fetched_user.nickname == "a@example.com"
+      assert fetched_user.ap_id == "https://sub.pleroma.example/users/a"
+      assert fetched_user.nickname == "a@pleroma.example"
     end
   end
 
@@ -2894,6 +2804,20 @@ defmodule Pleroma.UserTest do
     end
   end
 
+  describe "get_familiar_followers/3" do
+    test "returns familiar followers for a pair of users" do
+      user1 = insert(:user)
+      %{id: id2} = user2 = insert(:user)
+      user3 = insert(:user)
+      _user4 = insert(:user)
+
+      User.follow(user1, user2)
+      User.follow(user2, user3)
+
+      assert [%{id: ^id2}] = User.get_familiar_followers(user3, user1)
+    end
+  end
+
   describe "account endorsements" do
     test "it pins people" do
       user = insert(:user)

test/pleroma/web/activity_pub/side_effects_test.exs

@@ -827,31 +827,6 @@ defmodule Pleroma.Web.ActivityPub.SideEffectsTest do
       {:ok, announce, _} = SideEffects.handle(announce)
       assert Repo.get_by(Notification, user_id: poster.id, activity_id: announce.id)
     end
-
-    test "it streams out the announce", %{announce: announce} do
-      with_mocks([
-        {
-          Pleroma.Web.Streamer,
-          [],
-          [
-            stream: fn _, _ -> nil end
-          ]
-        },
-        {
-          Pleroma.Web.Push,
-          [],
-          [
-            send: fn _ -> nil end
-          ]
-        }
-      ]) do
-        {:ok, announce, _} = SideEffects.handle(announce)
-
-        assert called(Pleroma.Web.Streamer.stream(["user", "list"], announce))
-
-        assert called(Pleroma.Web.Push.send(:_))
-      end
-    end
   end
 
   describe "removing a follower" do

test/pleroma/web/activity_pub/views/user_view_test.exs

@@ -91,6 +91,13 @@ defmodule Pleroma.Web.ActivityPub.UserViewTest do
     assert %{"alsoKnownAs" => ^akas} = UserView.render("user.json", %{user: user})
   end
 
+  test "renders full nickname" do
+    clear_config([Pleroma.Web.WebFinger, :domain], "plemora.dev")
+
+    user = insert(:user, nickname: "user")
+    assert %{"webfinger" => "acct:user@plemora.dev"} = UserView.render("user.json", %{user: user})
+  end
+
   describe "endpoints" do
     test "local users have a usable endpoints structure" do
       user = insert(:user)

test/pleroma/web/admin_api/controllers/report_controller_test.exs

@@ -11,6 +11,7 @@ defmodule Pleroma.Web.AdminAPI.ReportControllerTest do
   alias Pleroma.ModerationLog
   alias Pleroma.Repo
   alias Pleroma.ReportNote
+  alias Pleroma.Rule
   alias Pleroma.Web.CommonAPI
 
   setup do
@@ -436,6 +437,34 @@ defmodule Pleroma.Web.AdminAPI.ReportControllerTest do
                "error" => "Invalid credentials."
              }
     end
+
+    test "returns reports with specified role_id", %{conn: conn} do
+      [reporter, target_user] = insert_pair(:user)
+
+      %{id: rule_id} = Rule.create(%{text: "Example rule"})
+
+      rule_id = to_string(rule_id)
+
+      {:ok, %{id: report_id}} =
+        CommonAPI.report(reporter, %{
+          account_id: target_user.id,
+          comment: "",
+          rule_ids: [rule_id]
+        })
+
+      {:ok, _report} =
+        CommonAPI.report(reporter, %{
+          account_id: target_user.id,
+          comment: ""
+        })
+
+      response =
+        conn
+        |> get("/api/pleroma/admin/reports?rule_id=#{rule_id}")
+        |> json_response_and_validate_schema(:ok)
+
+      assert %{"reports" => [%{"id" => ^report_id}]} = response
+    end
   end
 
   describe "POST /api/pleroma/admin/reports/:id/notes" do

test/pleroma/web/admin_api/controllers/rule_controller_test.exs

@@ -0,0 +1,82 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.AdminAPI.RuleControllerTest do
+  use Pleroma.Web.ConnCase, async: true
+
+  import Pleroma.Factory
+
+  alias Pleroma.Rule
+
+  setup do
+    admin = insert(:user, is_admin: true)
+    token = insert(:oauth_admin_token, user: admin)
+
+    conn =
+      build_conn()
+      |> assign(:user, admin)
+      |> assign(:token, token)
+
+    {:ok, %{admin: admin, token: token, conn: conn}}
+  end
+
+  describe "GET /api/pleroma/admin/rules" do
+    test "sorts rules by priority", %{conn: conn} do
+      %{id: id1} = Rule.create(%{text: "Example rule"})
+      %{id: id2} = Rule.create(%{text: "Second rule", priority: 2})
+      %{id: id3} = Rule.create(%{text: "Third rule", priority: 1})
+
+      id1 = to_string(id1)
+      id2 = to_string(id2)
+      id3 = to_string(id3)
+
+      response =
+        conn
+        |> get("/api/pleroma/admin/rules")
+        |> json_response_and_validate_schema(:ok)
+
+      assert [%{"id" => ^id1}, %{"id" => ^id3}, %{"id" => ^id2}] = response
+    end
+  end
+
+  describe "POST /api/pleroma/admin/rules" do
+    test "creates a rule", %{conn: conn} do
+      %{"id" => id} =
+        conn
+        |> put_req_header("content-type", "application/json")
+        |> post("/api/pleroma/admin/rules", %{text: "Example rule"})
+        |> json_response_and_validate_schema(:ok)
+
+      assert %{text: "Example rule"} = Rule.get(id)
+    end
+  end
+
+  describe "PATCH /api/pleroma/admin/rules" do
+    test "edits a rule", %{conn: conn} do
+      %{id: id} = Rule.create(%{text: "Example rule"})
+
+      conn
+      |> put_req_header("content-type", "application/json")
+      |> patch("/api/pleroma/admin/rules/#{id}", %{text: "There are no rules", priority: 2})
+      |> json_response_and_validate_schema(:ok)
+
+      assert %{text: "There are no rules", priority: 2} = Rule.get(id)
+    end
+  end
+
+  describe "DELETE /api/pleroma/admin/rules" do
+    test "deletes a rule", %{conn: conn} do
+      %{id: id} = Rule.create(%{text: "Example rule"})
+
+      conn
+      |> put_req_header("content-type", "application/json")
+      |> delete("/api/pleroma/admin/rules/#{id}")
+      |> json_response_and_validate_schema(:ok)
+
+      assert [] =
+               Rule.query()
+               |> Pleroma.Repo.all()
+    end
+  end
+end

test/pleroma/web/admin_api/views/report_view_test.exs

@@ -7,6 +7,7 @@ defmodule Pleroma.Web.AdminAPI.ReportViewTest do
 
   import Pleroma.Factory
 
+  alias Pleroma.Rule
   alias Pleroma.Web.AdminAPI
   alias Pleroma.Web.AdminAPI.Report
   alias Pleroma.Web.AdminAPI.ReportView
@@ -38,7 +39,8 @@ defmodule Pleroma.Web.AdminAPI.ReportViewTest do
       statuses: [],
       notes: [],
       state: "open",
-      id: activity.id
+      id: activity.id,
+      rules: []
     }
 
     result =
@@ -76,7 +78,8 @@ defmodule Pleroma.Web.AdminAPI.ReportViewTest do
       statuses: [StatusView.render("show.json", %{activity: activity})],
       state: "open",
       notes: [],
-      id: report_activity.id
+      id: report_activity.id,
+      rules: []
     }
 
     result =
@@ -168,4 +171,22 @@ defmodule Pleroma.Web.AdminAPI.ReportViewTest do
     assert report2.id == rendered |> Enum.at(0) |> Map.get(:id)
     assert report1.id == rendered |> Enum.at(1) |> Map.get(:id)
   end
+
+  test "renders included rules" do
+    user = insert(:user)
+    other_user = insert(:user)
+
+    %{id: rule_id, text: text} = Rule.create(%{text: "Example rule"})
+
+    rule_id = to_string(rule_id)
+
+    {:ok, activity} =
+      CommonAPI.report(user, %{
+        account_id: other_user.id,
+        rule_ids: [rule_id]
+      })
+
+    assert %{rules: [%{id: ^rule_id, text: ^text}]} =
+             ReportView.render("show.json", Report.extract_report_info(activity))
+  end
 end

test/pleroma/web/common_api_test.exs

@@ -12,6 +12,7 @@ defmodule Pleroma.Web.CommonAPITest do
   alias Pleroma.Notification
   alias Pleroma.Object
   alias Pleroma.Repo
+  alias Pleroma.Rule
   alias Pleroma.UnstubbedConfigMock, as: ConfigMock
   alias Pleroma.User
   alias Pleroma.Web.ActivityPub.ActivityPub
@@ -1363,6 +1364,33 @@ defmodule Pleroma.Web.CommonAPITest do
       assert first_report.data["state"] == "resolved"
       assert second_report.data["state"] == "resolved"
     end
+
+    test "creates a report with provided rules" do
+      reporter = insert(:user)
+      target_user = insert(:user)
+
+      %{id: rule_id} = Rule.create(%{text: "There are no rules"})
+
+      reporter_ap_id = reporter.ap_id
+      target_ap_id = target_user.ap_id
+
+      report_data = %{
+        account_id: target_user.id,
+        rule_ids: [rule_id]
+      }
+
+      assert {:ok, flag_activity} = CommonAPI.report(reporter, report_data)
+
+      assert %Activity{
+               actor: ^reporter_ap_id,
+               data: %{
+                 "type" => "Flag",
+                 "object" => [^target_ap_id],
+                 "state" => "open",
+                 "rules" => [^rule_id]
+               }
+             } = flag_activity
+    end
   end
 
   describe "reblog muting" do

test/pleroma/web/mastodon_api/controllers/account_controller_test.exs

@@ -2172,6 +2172,55 @@ defmodule Pleroma.Web.MastodonAPI.AccountControllerTest do
     end
   end
 
+  describe "familiar followers" do
+    setup do: oauth_access(["read:follows"])
+
+    test "fetch user familiar followers", %{user: user, conn: conn} do
+      %{id: id1} = other_user1 = insert(:user)
+      %{id: id2} = other_user2 = insert(:user)
+      _ = insert(:user)
+
+      User.follow(user, other_user1)
+      User.follow(other_user1, other_user2)
+
+      assert [%{"accounts" => [%{"id" => ^id1}], "id" => ^id2}] =
+               conn
+               |> put_req_header("content-type", "application/json")
+               |> get("/api/v1/accounts/familiar_followers?id[]=#{id2}")
+               |> json_response_and_validate_schema(200)
+    end
+
+    test "returns empty array if followers are hidden", %{user: user, conn: conn} do
+      other_user1 = insert(:user, hide_follows: true)
+      %{id: id2} = other_user2 = insert(:user)
+      _ = insert(:user)
+
+      User.follow(user, other_user1)
+      User.follow(other_user1, other_user2)
+
+      assert [%{"accounts" => [], "id" => ^id2}] =
+               conn
+               |> put_req_header("content-type", "application/json")
+               |> get("/api/v1/accounts/familiar_followers?id[]=#{id2}")
+               |> json_response_and_validate_schema(200)
+    end
+
+    test "it respects hide_followers", %{user: user, conn: conn} do
+      other_user1 = insert(:user)
+      %{id: id2} = other_user2 = insert(:user, hide_followers: true)
+      _ = insert(:user)
+
+      User.follow(user, other_user1)
+      User.follow(other_user1, other_user2)
+
+      assert [%{"accounts" => [], "id" => ^id2}] =
+               conn
+               |> put_req_header("content-type", "application/json")
+               |> get("/api/v1/accounts/familiar_followers?id[]=#{id2}")
+               |> json_response_and_validate_schema(200)
+    end
+  end
+
   describe "remove from followers" do
     setup do: oauth_access(["follow"])
 

test/pleroma/web/mastodon_api/controllers/instance_controller_test.exs

@@ -6,6 +6,7 @@ defmodule Pleroma.Web.MastodonAPI.InstanceControllerTest do
   # TODO: Should not need Cachex
   use Pleroma.Web.ConnCase
 
+  alias Pleroma.Rule
   alias Pleroma.User
   import Pleroma.Factory
 
@@ -40,7 +41,8 @@ defmodule Pleroma.Web.MastodonAPI.InstanceControllerTest do
              "banner_upload_limit" => _,
              "background_image" => from_config_background,
              "shout_limit" => _,
-             "description_limit" => _
+             "description_limit" => _,
+             "rules" => _
            } = result
 
     assert result["pleroma"]["metadata"]["account_activation_required"] != nil
@@ -125,4 +127,29 @@ defmodule Pleroma.Web.MastodonAPI.InstanceControllerTest do
     assert get(conn, "/api/v2/instance")
            |> json_response_and_validate_schema(200)
   end
+
+  test "get instance rules", %{conn: conn} do
+    Rule.create(%{text: "Example rule", hint: "Rule description", priority: 1})
+    Rule.create(%{text: "Third rule", priority: 2})
+    Rule.create(%{text: "Second rule", priority: 1})
+
+    conn = get(conn, "/api/v1/instance")
+
+    assert result = json_response_and_validate_schema(conn, 200)
+
+    assert [
+             %{
+               "text" => "Example rule",
+               "hint" => "Rule description"
+             },
+             %{
+               "text" => "Second rule",
+               "hint" => ""
+             },
+             %{
+               "text" => "Third rule",
+               "hint" => ""
+             }
+           ] = result["rules"]
+  end
 end

test/pleroma/web/mastodon_api/controllers/report_controller_test.exs

@@ -7,6 +7,7 @@ defmodule Pleroma.Web.MastodonAPI.ReportControllerTest do
 
   alias Pleroma.Activity
   alias Pleroma.Repo
+  alias Pleroma.Rule
   alias Pleroma.Web.CommonAPI
 
   import Pleroma.Factory
@@ -81,6 +82,44 @@ defmodule Pleroma.Web.MastodonAPI.ReportControllerTest do
              |> json_response_and_validate_schema(200)
   end
 
+  test "submit a report with rule_ids", %{
+    conn: conn,
+    target_user: target_user
+  } do
+    %{id: rule_id} = Rule.create(%{text: "There are no rules"})
+
+    rule_id = to_string(rule_id)
+
+    assert %{"action_taken" => false, "id" => id} =
+             conn
+             |> put_req_header("content-type", "application/json")
+             |> post("/api/v1/reports", %{
+               "account_id" => target_user.id,
+               "forward" => "false",
+               "rule_ids" => [rule_id]
+             })
+             |> json_response_and_validate_schema(200)
+
+    assert %Activity{data: %{"rules" => [^rule_id]}} = Activity.get_report(id)
+  end
+
+  test "rules field is empty if provided wrong rule id", %{
+    conn: conn,
+    target_user: target_user
+  } do
+    assert %{"id" => id} =
+             conn
+             |> put_req_header("content-type", "application/json")
+             |> post("/api/v1/reports", %{
+               "account_id" => target_user.id,
+               "forward" => "false",
+               "rule_ids" => ["-1"]
+             })
+             |> json_response_and_validate_schema(200)
+
+    assert %Activity{data: %{"rules" => []}} = Activity.get_report(id)
+  end
+
   test "account_id is required", %{
     conn: conn,
     activity: activity

test/pleroma/web/mastodon_api/controllers/status_controller_test.exs

@@ -329,62 +329,6 @@ defmodule Pleroma.Web.MastodonAPI.StatusControllerTest do
       assert real_status == fake_status
     end
 
-    test "fake statuses' preview card is not cached", %{conn: conn} do
-      Pleroma.StaticStubbedConfigMock
-      |> stub(:get, fn
-        [:rich_media, :enabled] -> true
-        path -> Pleroma.Test.StaticConfig.get(path)
-      end)
-
-      Tesla.Mock.mock_global(fn
-        env ->
-          apply(HttpRequestMock, :request, [env])
-      end)
-
-      conn1 =
-        conn
-        |> put_req_header("content-type", "application/json")
-        |> post("/api/v1/statuses", %{
-          "status" => "https://example.com/ogp",
-          "preview" => true
-        })
-
-      conn2 =
-        conn
-        |> put_req_header("content-type", "application/json")
-        |> post("/api/v1/statuses", %{
-          "status" => "https://example.com/twitter-card",
-          "preview" => true
-        })
-
-      assert %{"card" => %{"title" => "The Rock"}} = json_response_and_validate_schema(conn1, 200)
-
-      assert %{"card" => %{"title" => "Small Island Developing States Photo Submission"}} =
-               json_response_and_validate_schema(conn2, 200)
-    end
-
-    test "posting a status with OGP link preview", %{conn: conn} do
-      Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end)
-
-      Pleroma.StaticStubbedConfigMock
-      |> stub(:get, fn
-        [:rich_media, :enabled] -> true
-        path -> Pleroma.Test.StaticConfig.get(path)
-      end)
-
-      conn =
-        conn
-        |> put_req_header("content-type", "application/json")
-        |> post("/api/v1/statuses", %{
-          "status" => "https://example.com/ogp"
-        })
-
-      assert %{"id" => id, "card" => %{"title" => "The Rock"}} =
-               json_response_and_validate_schema(conn, 200)
-
-      assert Activity.get_by_id(id)
-    end
-
     test "posting a direct status", %{conn: conn} do
       user2 = insert(:user)
       content = "direct cofe @#{user2.nickname}"
@@ -1699,91 +1643,6 @@ defmodule Pleroma.Web.MastodonAPI.StatusControllerTest do
     end
   end
 
-  describe "cards" do
-    setup do
-      Pleroma.StaticStubbedConfigMock
-      |> stub(:get, fn
-        [:rich_media, :enabled] -> true
-        path -> Pleroma.Test.StaticConfig.get(path)
-      end)
-
-      oauth_access(["read:statuses"])
-    end
-
-    test "returns rich-media card", %{conn: conn, user: user} do
-      Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end)
-
-      {:ok, activity} = CommonAPI.post(user, %{status: "https://example.com/ogp"})
-
-      card_data = %{
-        "image" => "http://ia.media-imdb.com/images/rock.jpg",
-        "provider_name" => "example.com",
-        "provider_url" => "https://example.com",
-        "title" => "The Rock",
-        "type" => "link",
-        "url" => "https://example.com/ogp",
-        "description" =>
-          "Directed by Michael Bay. With Sean Connery, Nicolas Cage, Ed Harris, John Spencer.",
-        "pleroma" => %{
-          "opengraph" => %{
-            "image" => "http://ia.media-imdb.com/images/rock.jpg",
-            "title" => "The Rock",
-            "type" => "video.movie",
-            "url" => "https://example.com/ogp",
-            "description" =>
-              "Directed by Michael Bay. With Sean Connery, Nicolas Cage, Ed Harris, John Spencer."
-          }
-        }
-      }
-
-      response =
-        conn
-        |> get("/api/v1/statuses/#{activity.id}/card")
-        |> json_response_and_validate_schema(200)
-
-      assert response == card_data
-
-      # works with private posts
-      {:ok, activity} =
-        CommonAPI.post(user, %{status: "https://example.com/ogp", visibility: "direct"})
-
-      response_two =
-        conn
-        |> get("/api/v1/statuses/#{activity.id}/card")
-        |> json_response_and_validate_schema(200)
-
-      assert response_two == card_data
-    end
-
-    test "replaces missing description with an empty string", %{conn: conn, user: user} do
-      Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end)
-
-      {:ok, activity} = CommonAPI.post(user, %{status: "https://example.com/ogp-missing-data"})
-
-      response =
-        conn
-        |> get("/api/v1/statuses/#{activity.id}/card")
-        |> json_response_and_validate_schema(:ok)
-
-      assert response == %{
-               "type" => "link",
-               "title" => "Pleroma",
-               "description" => "",
-               "image" => nil,
-               "provider_name" => "example.com",
-               "provider_url" => "https://example.com",
-               "url" => "https://example.com/ogp-missing-data",
-               "pleroma" => %{
-                 "opengraph" => %{
-                   "title" => "Pleroma",
-                   "type" => "website",
-                   "url" => "https://example.com/ogp-missing-data"
-                 }
-               }
-             }
-    end
-  end
-
   test "bookmarks" do
     bookmarks_uri = "/api/v1/bookmarks"
 

test/pleroma/web/mastodon_api/views/notification_view_test.exs

@@ -331,4 +331,31 @@ defmodule Pleroma.Web.MastodonAPI.NotificationViewTest do
 
     test_notifications_rendering([notification], user, [expected])
   end
+
+  test "Subscribed status notification" do
+    user = insert(:user)
+    subscriber = insert(:user)
+
+    User.subscribe(subscriber, user)
+
+    {:ok, activity} = CommonAPI.post(user, %{status: "hi"})
+    {:ok, [notification]} = Notification.create_notifications(activity)
+
+    user = User.get_cached_by_id(user.id)
+
+    expected = %{
+      id: to_string(notification.id),
+      pleroma: %{is_seen: false, is_muted: false},
+      type: "status",
+      account:
+        AccountView.render("show.json", %{
+          user: user,
+          for: subscriber
+        }),
+      status: StatusView.render("show.json", %{activity: activity, for: subscriber}),
+      created_at: Utils.to_masto_date(notification.inserted_at)
+    }
+
+    test_notifications_rendering([notification], subscriber, [expected])
+  end
 end

test/pleroma/web/mastodon_api/views/status_view_test.exs

@@ -17,6 +17,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusViewTest do
   alias Pleroma.Web.CommonAPI
   alias Pleroma.Web.MastodonAPI.AccountView
   alias Pleroma.Web.MastodonAPI.StatusView
+  alias Pleroma.Web.RichMedia.Card
 
   require Bitwise
 
@@ -732,56 +733,72 @@ defmodule Pleroma.Web.MastodonAPI.StatusViewTest do
 
   describe "rich media cards" do
     test "a rich media card without a site name renders correctly" do
-      page_url = "http://example.com"
+      page_url = "https://example.com"
 
-      card = %{
-        url: page_url,
-        image: page_url <> "/example.jpg",
-        title: "Example website"
-      }
+      {:ok, card} =
+        Card.create(page_url, %{image: page_url <> "/example.jpg", title: "Example website"})
 
-      %{provider_name: "example.com"} =
-        StatusView.render("card.json", %{page_url: page_url, rich_media: card})
+      assert match?(%{provider_name: "example.com"}, StatusView.render("card.json", card))
     end
 
     test "a rich media card without a site name or image renders correctly" do
-      page_url = "http://example.com"
+      page_url = "https://example.com"
 
-      card = %{
-        url: page_url,
-        title: "Example website"
+      fields = %{
+        "url" => page_url,
+        "title" => "Example website"
       }
 
-      %{provider_name: "example.com"} =
-        StatusView.render("card.json", %{page_url: page_url, rich_media: card})
+      {:ok, card} = Card.create(page_url, fields)
+
+      assert match?(%{provider_name: "example.com"}, StatusView.render("card.json", card))
     end
 
     test "a rich media card without an image renders correctly" do
-      page_url = "http://example.com"
+      page_url = "https://example.com"
 
-      card = %{
-        url: page_url,
-        site_name: "Example site name",
-        title: "Example website"
+      fields = %{
+        "url" => page_url,
+        "site_name" => "Example site name",
+        "title" => "Example website"
       }
 
-      %{provider_name: "example.com"} =
-        StatusView.render("card.json", %{page_url: page_url, rich_media: card})
+      {:ok, card} = Card.create(page_url, fields)
+
+      assert match?(%{provider_name: "example.com"}, StatusView.render("card.json", card))
+    end
+
+    test "a rich media card without descriptions returns the fields with empty strings" do
+      page_url = "https://example.com"
+
+      fields = %{
+        "url" => page_url,
+        "site_name" => "Example site name",
+        "title" => "Example website"
+      }
+
+      {:ok, card} = Card.create(page_url, fields)
+
+      assert match?(
+               %{description: "", image_description: ""},
+               StatusView.render("card.json", card)
+             )
     end
 
     test "a rich media card with all relevant data renders correctly" do
-      page_url = "http://example.com"
+      page_url = "https://example.com"
 
-      card = %{
-        url: page_url,
-        site_name: "Example site name",
-        title: "Example website",
-        image: page_url <> "/example.jpg",
-        description: "Example description"
+      fields = %{
+        "url" => page_url,
+        "site_name" => "Example site name",
+        "title" => "Example website",
+        "image" => page_url <> "/example.jpg",
+        "description" => "Example description"
       }
 
-      %{provider_name: "example.com"} =
-        StatusView.render("card.json", %{page_url: page_url, rich_media: card})
+      {:ok, card} = Card.create(page_url, fields)
+
+      assert match?(%{provider_name: "example.com"}, StatusView.render("card.json", card))
     end
 
     test "a rich media card has all media proxied" do
@@ -791,25 +808,25 @@ defmodule Pleroma.Web.MastodonAPI.StatusViewTest do
       ConfigMock
       |> stub_with(Pleroma.Test.StaticConfig)
 
-      page_url = "http://example.com"
+      page_url = "https://example.com"
 
-      card = %{
-        url: page_url,
-        site_name: "Example site name",
-        title: "Example website",
-        image: page_url <> "/example.jpg",
-        audio: page_url <> "/example.ogg",
-        video: page_url <> "/example.mp4",
-        description: "Example description"
+      fields = %{
+        "url" => page_url,
+        "site_name" => "Example site name",
+        "title" => "Example website",
+        "image" => page_url <> "/example.jpg",
+        "audio" => page_url <> "/example.ogg",
+        "video" => page_url <> "/example.mp4",
+        "description" => "Example description"
       }
 
-      strcard = for {k, v} <- card, into: %{}, do: {to_string(k), v}
+      {:ok, card} = Card.create(page_url, fields)
 
       %{
         provider_name: "example.com",
         image: image,
         pleroma: %{opengraph: og}
-      } = StatusView.render("card.json", %{page_url: page_url, rich_media: strcard})
+      } = StatusView.render("card.json", card)
 
       assert String.match?(image, ~r/\/proxy\//)
       assert String.match?(og["image"], ~r/\/proxy\//)

test/pleroma/web/pleroma_api/controllers/notification_controller_test.exs

@@ -21,13 +21,11 @@ defmodule Pleroma.Web.PleromaAPI.NotificationControllerTest do
       {:ok, [notification1]} = Notification.create_notifications(activity1)
       {:ok, [notification2]} = Notification.create_notifications(activity2)
 
-      response =
-        conn
-        |> put_req_header("content-type", "application/json")
-        |> post("/api/v1/pleroma/notifications/read", %{id: notification1.id})
-        |> json_response_and_validate_schema(:ok)
+      conn
+      |> put_req_header("content-type", "application/json")
+      |> post("/api/v1/pleroma/notifications/read", %{id: notification1.id})
+      |> json_response_and_validate_schema(:ok)
 
-      assert %{"pleroma" => %{"is_seen" => true}} = response
       assert Repo.get(Notification, notification1.id).seen
       refute Repo.get(Notification, notification2.id).seen
     end
@@ -40,14 +38,17 @@ defmodule Pleroma.Web.PleromaAPI.NotificationControllerTest do
 
       [notification3, notification2, notification1] = Notification.for_user(user1, %{limit: 3})
 
-      [response1, response2] =
-        conn
-        |> put_req_header("content-type", "application/json")
-        |> post("/api/v1/pleroma/notifications/read", %{max_id: notification2.id})
-        |> json_response_and_validate_schema(:ok)
+      refute Repo.get(Notification, notification1.id).seen
+      refute Repo.get(Notification, notification2.id).seen
+      refute Repo.get(Notification, notification3.id).seen
+
+      conn
+      |> put_req_header("content-type", "application/json")
+      |> post("/api/v1/pleroma/notifications/read", %{max_id: notification2.id})
+      |> json_response_and_validate_schema(:ok)
+
+      [notification3, notification2, notification1] = Notification.for_user(user1, %{limit: 3})
 
-      assert %{"pleroma" => %{"is_seen" => true}} = response1
-      assert %{"pleroma" => %{"is_seen" => true}} = response2
       assert Repo.get(Notification, notification1.id).seen
       assert Repo.get(Notification, notification2.id).seen
       refute Repo.get(Notification, notification3.id).seen

test/pleroma/web/pleroma_api/views/chat_message_reference_view_test.exs

@@ -9,7 +9,6 @@ defmodule Pleroma.Web.PleromaAPI.ChatMessageReferenceViewTest do
   alias Pleroma.Chat
   alias Pleroma.Chat.MessageReference
   alias Pleroma.Object
-  alias Pleroma.StaticStubbedConfigMock
   alias Pleroma.UnstubbedConfigMock, as: ConfigMock
   alias Pleroma.Web.ActivityPub.ActivityPub
   alias Pleroma.Web.CommonAPI
@@ -18,6 +17,8 @@ defmodule Pleroma.Web.PleromaAPI.ChatMessageReferenceViewTest do
   import Mox
   import Pleroma.Factory
 
+  setup do: clear_config([:rich_media, :enabled], true)
+
   test "it displays a chat message" do
     user = insert(:user)
     recipient = insert(:user)
@@ -62,16 +63,7 @@ defmodule Pleroma.Web.PleromaAPI.ChatMessageReferenceViewTest do
     assert match?([%{shortcode: "firefox"}], chat_message[:emojis])
     assert chat_message[:idempotency_key] == "123"
 
-    StaticStubbedConfigMock
-    |> stub(:get, fn
-      [:rich_media, :enabled] -> true
-      path -> Pleroma.Test.StaticConfig.get(path)
-    end)
-
-    Tesla.Mock.mock_global(fn
-      %{url: "https://example.com/ogp"} ->
-        %Tesla.Env{status: 200, body: File.read!("test/fixtures/rich_media/ogp.html")}
-    end)
+    Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end)
 
     {:ok, activity} =
       CommonAPI.post_chat_message(recipient, user, "gkgkgk https://example.com/ogp",

test/pleroma/web/rich_media/card_test.exs

@@ -0,0 +1,71 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2024 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.RichMedia.CardTest do
+  use Pleroma.DataCase, async: true
+
+  alias Pleroma.UnstubbedConfigMock, as: ConfigMock
+  alias Pleroma.Web.CommonAPI
+  alias Pleroma.Web.RichMedia.Card
+
+  import Mox
+  import Pleroma.Factory
+  import Tesla.Mock
+
+  setup do
+    mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end)
+
+    ConfigMock
+    |> stub_with(Pleroma.Test.StaticConfig)
+
+    :ok
+  end
+
+  setup do: clear_config([:rich_media, :enabled], true)
+
+  test "crawls URL in activity" do
+    user = insert(:user)
+
+    url = "https://example.com/ogp"
+    url_hash = Card.url_to_hash(url)
+
+    {:ok, activity} =
+      CommonAPI.post(user, %{
+        status: "[test](#{url})",
+        content_type: "text/markdown"
+      })
+
+    assert %Card{url_hash: ^url_hash, fields: _} = Card.get_by_activity(activity)
+  end
+
+  test "recrawls URLs on status edits/updates" do
+    original_url = "https://google.com/"
+    original_url_hash = Card.url_to_hash(original_url)
+    updated_url = "https://yahoo.com/"
+    updated_url_hash = Card.url_to_hash(updated_url)
+
+    user = insert(:user)
+    {:ok, activity} = CommonAPI.post(user, %{status: "I like this site #{original_url}"})
+
+    # Force a backfill
+    Card.get_by_activity(activity)
+
+    assert match?(
+             %Card{url_hash: ^original_url_hash, fields: _},
+             Card.get_by_activity(activity)
+           )
+
+    {:ok, _} = CommonAPI.update(user, activity, %{status: "I like this site #{updated_url}"})
+
+    activity = Pleroma.Activity.get_by_id(activity.id)
+
+    # Force a backfill
+    Card.get_by_activity(activity)
+
+    assert match?(
+             %Card{url_hash: ^updated_url_hash, fields: _},
+             Card.get_by_activity(activity)
+           )
+  end
+end

test/pleroma/web/rich_media/helpers_test.exs

@@ -1,137 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Pleroma.Web.RichMedia.HelpersTest do
-  use Pleroma.DataCase, async: false
-
-  alias Pleroma.StaticStubbedConfigMock, as: ConfigMock
-  alias Pleroma.Web.CommonAPI
-  alias Pleroma.Web.RichMedia.Helpers
-
-  import Mox
-  import Pleroma.Factory
-  import Tesla.Mock
-
-  setup do
-    mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end)
-
-    ConfigMock
-    |> stub(:get, fn
-      [:rich_media, :enabled] -> false
-      path -> Pleroma.Test.StaticConfig.get(path)
-    end)
-    |> stub(:get, fn
-      path, default -> Pleroma.Test.StaticConfig.get(path, default)
-    end)
-
-    :ok
-  end
-
-  test "refuses to crawl incomplete URLs" do
-    user = insert(:user)
-
-    {:ok, activity} =
-      CommonAPI.post(user, %{
-        status: "[test](example.com/ogp)",
-        content_type: "text/markdown"
-      })
-
-    ConfigMock
-    |> stub(:get, fn
-      [:rich_media, :enabled] -> true
-      path -> Pleroma.Test.StaticConfig.get(path)
-    end)
-
-    assert %{} == Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity)
-  end
-
-  test "refuses to crawl malformed URLs" do
-    user = insert(:user)
-
-    {:ok, activity} =
-      CommonAPI.post(user, %{
-        status: "[test](example.com[]/ogp)",
-        content_type: "text/markdown"
-      })
-
-    ConfigMock
-    |> stub(:get, fn
-      [:rich_media, :enabled] -> true
-      path -> Pleroma.Test.StaticConfig.get(path)
-    end)
-
-    assert %{} == Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity)
-  end
-
-  test "crawls valid, complete URLs" do
-    user = insert(:user)
-
-    {:ok, activity} =
-      CommonAPI.post(user, %{
-        status: "[test](https://example.com/ogp)",
-        content_type: "text/markdown"
-      })
-
-    ConfigMock
-    |> stub(:get, fn
-      [:rich_media, :enabled] -> true
-      path -> Pleroma.Test.StaticConfig.get(path)
-    end)
-
-    assert %{page_url: "https://example.com/ogp", rich_media: _} =
-             Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity)
-  end
-
-  test "recrawls URLs on updates" do
-    original_url = "https://google.com/"
-    updated_url = "https://yahoo.com/"
-
-    Pleroma.StaticStubbedConfigMock
-    |> stub(:get, fn
-      [:rich_media, :enabled] -> true
-      path -> Pleroma.Test.StaticConfig.get(path)
-    end)
-
-    user = insert(:user)
-    {:ok, activity} = CommonAPI.post(user, %{status: "I like this site #{original_url}"})
-
-    assert match?(
-             %{page_url: ^original_url, rich_media: _},
-             Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity)
-           )
-
-    {:ok, _} = CommonAPI.update(user, activity, %{status: "I like this site #{updated_url}"})
-
-    activity = Pleroma.Activity.get_by_id(activity.id)
-
-    assert match?(
-             %{page_url: ^updated_url, rich_media: _},
-             Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity)
-           )
-  end
-
-  test "refuses to crawl URLs of private network from posts" do
-    user = insert(:user)
-
-    {:ok, activity} =
-      CommonAPI.post(user, %{status: "http://127.0.0.1:4000/notice/9kCP7VNyPJXFOXDrgO"})
-
-    {:ok, activity2} = CommonAPI.post(user, %{status: "https://10.111.10.1/notice/9kCP7V"})
-    {:ok, activity3} = CommonAPI.post(user, %{status: "https://172.16.32.40/notice/9kCP7V"})
-    {:ok, activity4} = CommonAPI.post(user, %{status: "https://192.168.10.40/notice/9kCP7V"})
-    {:ok, activity5} = CommonAPI.post(user, %{status: "https://pleroma.local/notice/9kCP7V"})
-
-    ConfigMock
-    |> stub(:get, fn
-      [:rich_media, :enabled] -> true
-      path -> Pleroma.Test.StaticConfig.get(path)
-    end)
-
-    assert %{} == Helpers.fetch_data_for_activity(activity)
-    assert %{} == Helpers.fetch_data_for_activity(activity2)
-    assert %{} == Helpers.fetch_data_for_activity(activity3)
-    assert %{} == Helpers.fetch_data_for_activity(activity4)
-    assert %{} == Helpers.fetch_data_for_activity(activity5)
-  end
-end

test/pleroma/web/rich_media/parser/ttl/aws_signed_url_test.exs

@@ -3,8 +3,22 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.RichMedia.Parser.TTL.AwsSignedUrlTest do
-  # Relies on Cachex, needs to be synchronous
-  use Pleroma.DataCase
+  use Pleroma.DataCase, async: false
+  use Oban.Testing, repo: Pleroma.Repo
+
+  import Mox
+
+  alias Pleroma.UnstubbedConfigMock, as: ConfigMock
+  alias Pleroma.Web.RichMedia.Card
+
+  setup do
+    ConfigMock
+    |> stub_with(Pleroma.Test.StaticConfig)
+
+    clear_config([:rich_media, :enabled], true)
+
+    :ok
+  end
 
   test "s3 signed url is parsed correct for expiration time" do
     url = "https://pleroma.social/amz"
@@ -43,26 +57,29 @@ defmodule Pleroma.Web.RichMedia.Parser.TTL.AwsSignedUrlTest do
     <meta name="twitter:site" content="Pleroma" />
     <meta name="twitter:title" content="Pleroma" />
     <meta name="twitter:description" content="Pleroma" />
-    <meta name="twitter:image" content="#{Map.get(metadata, :image)}" />
+    <meta name="twitter:image" content="#{Map.get(metadata, "image")}" />
     """
 
     Tesla.Mock.mock(fn
       %{
         method: :get,
-        url: "https://pleroma.social/amz"
+        url: ^url
       } ->
         %Tesla.Env{status: 200, body: body}
+
+      %{method: :head} ->
+        %Tesla.Env{status: 200}
     end)
 
-    Cachex.put(:rich_media_cache, url, metadata)
+    Card.get_or_backfill_by_url(url)
 
-    Pleroma.Web.RichMedia.Parser.set_ttl_based_on_image(metadata, url)
+    assert_enqueued(worker: Pleroma.Workers.RichMediaExpirationWorker, args: %{"url" => url})
 
-    {:ok, cache_ttl} = Cachex.ttl(:rich_media_cache, url)
+    [%Oban.Job{scheduled_at: scheduled_at}] = all_enqueued()
 
-    # as there is delay in setting and pulling the data from cache we ignore 1 second
-    # make it 2 seconds for flakyness
-    assert_in_delta(valid_till * 1000, cache_ttl, 2000)
+    timestamp_dt = Timex.parse!(timestamp, "{ISO:Basic:Z}")
+
+    assert DateTime.diff(scheduled_at, timestamp_dt) == valid_till
   end
 
   defp construct_s3_url(timestamp, valid_till) do
@@ -71,11 +88,11 @@ defmodule Pleroma.Web.RichMedia.Parser.TTL.AwsSignedUrlTest do
 
   defp construct_metadata(timestamp, valid_till, url) do
     %{
-      image: construct_s3_url(timestamp, valid_till),
-      site: "Pleroma",
-      title: "Pleroma",
-      description: "Pleroma",
-      url: url
+      "image" => construct_s3_url(timestamp, valid_till),
+      "site" => "Pleroma",
+      "title" => "Pleroma",
+      "description" => "Pleroma",
+      "url" => url
     }
   end
 end

test/pleroma/web/rich_media/parser/ttl/opengraph_test.exs

@@ -0,0 +1,41 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2024 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.RichMedia.Parser.TTL.OpengraphTest do
+  use Pleroma.DataCase
+  use Oban.Testing, repo: Pleroma.Repo
+
+  import Mox
+
+  alias Pleroma.UnstubbedConfigMock, as: ConfigMock
+  alias Pleroma.Web.RichMedia.Card
+
+  setup do
+    ConfigMock
+    |> stub_with(Pleroma.Test.StaticConfig)
+
+    clear_config([:rich_media, :enabled], true)
+
+    :ok
+  end
+
+  test "OpenGraph TTL value is honored" do
+    url = "https://reddit.com/r/somepost"
+
+    Tesla.Mock.mock(fn
+      %{
+        method: :get,
+        url: ^url
+      } ->
+        %Tesla.Env{status: 200, body: File.read!("test/fixtures/rich_media/reddit.html")}
+
+      %{method: :head} ->
+        %Tesla.Env{status: 200}
+    end)
+
+    Card.get_or_backfill_by_url(url)
+
+    assert_enqueued(worker: Pleroma.Workers.RichMediaExpirationWorker, args: %{"url" => url})
+  end
+end

test/pleroma/web/rich_media/parser_test.exs

@@ -3,7 +3,7 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.RichMedia.ParserTest do
-  use Pleroma.DataCase, async: false
+  use Pleroma.DataCase
 
   alias Pleroma.Web.RichMedia.Parser
 
@@ -104,4 +104,27 @@ defmodule Pleroma.Web.RichMedia.ParserTest do
   test "does a HEAD request to check if the body is html" do
     assert {:error, {:content_type, _}} = Parser.parse("https://example.com/pdf-file")
   end
+
+  test "refuses to crawl incomplete URLs" do
+    url = "example.com/ogp"
+    assert :error == Parser.parse(url)
+  end
+
+  test "refuses to crawl malformed URLs" do
+    url = "example.com[]/ogp"
+    assert :error == Parser.parse(url)
+  end
+
+  test "refuses to crawl URLs of private network from posts" do
+    [
+      "http://127.0.0.1:4000/notice/9kCP7VNyPJXFOXDrgO",
+      "https://10.111.10.1/notice/9kCP7V",
+      "https://172.16.32.40/notice/9kCP7V",
+      "https://192.168.10.40/notice/9kCP7V",
+      "https://pleroma.local/notice/9kCP7V"
+    ]
+    |> Enum.each(fn url ->
+      assert :error == Parser.parse(url)
+    end)
+  end
 end

test/pleroma/web/web_finger_test.exs

@@ -76,15 +76,6 @@ defmodule Pleroma.Web.WebFingerTest do
       {:ok, _data} = WebFinger.finger(user)
     end
 
-    test "returns the ActivityPub actor URI and subscribe address for an ActivityPub user with the ld+json mimetype" do
-      user = "kaniini@gerzilla.de"
-
-      {:ok, data} = WebFinger.finger(user)
-
-      assert data["ap_id"] == "https://gerzilla.de/channel/kaniini"
-      assert data["subscribe_address"] == "https://gerzilla.de/follow?f=&url={uri}"
-    end
-
     test "it work for AP-only user" do
       user = "kpherox@mstdn.jp"
 
@@ -99,12 +90,6 @@ defmodule Pleroma.Web.WebFingerTest do
       assert data["subscribe_address"] == "https://mstdn.jp/authorize_interaction?acct={uri}"
     end
 
-    test "it works for friendica" do
-      user = "lain@squeet.me"
-
-      {:ok, _data} = WebFinger.finger(user)
-    end
-
     test "it gets the xrd endpoint" do
       {:ok, template} = WebFinger.find_lrdd_template("social.heldscal.la")
 
@@ -203,5 +188,44 @@ defmodule Pleroma.Web.WebFingerTest do
 
       assert :error = WebFinger.finger("pekorino@pawoo.net")
     end
+
+    test "prevents spoofing" do
+      Tesla.Mock.mock(fn
+        %{
+          url: "https://gleasonator.com/.well-known/webfinger?resource=acct:alex@gleasonator.com"
+        } ->
+          {:ok,
+           %Tesla.Env{
+             status: 200,
+             body: File.read!("test/fixtures/tesla_mock/webfinger_spoof.json"),
+             headers: [{"content-type", "application/jrd+json"}]
+           }}
+
+        %{url: "https://gleasonator.com/.well-known/host-meta"} ->
+          {:ok,
+           %Tesla.Env{
+             status: 200,
+             body: File.read!("test/fixtures/tesla_mock/gleasonator.com_host_meta")
+           }}
+      end)
+
+      {:error, _data} = WebFinger.finger("alex@gleasonator.com")
+    end
+  end
+
+  @tag capture_log: true
+  test "prevents forgeries" do
+    Tesla.Mock.mock(fn
+      %{url: "https://fba.ryona.agency/.well-known/webfinger?resource=acct:graf@fba.ryona.agency"} ->
+        fake_webfinger =
+          File.read!("test/fixtures/webfinger/graf-imposter-webfinger.json") |> Jason.decode!()
+
+        Tesla.Mock.json(fake_webfinger)
+
+      %{url: "https://fba.ryona.agency/.well-known/host-meta"} ->
+        {:ok, %Tesla.Env{status: 404}}
+    end)
+
+    assert {:error, _} = WebFinger.finger("graf@fba.ryona.agency")
   end
 end