Merge remote-tracking branch 'origin/develop' into shigusegubu

* origin/develop: (55 commits)
  AP UserView: Refactor banner / avatar display code, add test.
  Test for MastoAPI /api/v1/instance response structure (formatting fix).
  Test for MastoAPI /api/v1/instance response structure.
  Credo is upset about me not remembering the alphabet
  Fix formatting
  Remove ActivityRepresenter
  Mastodon 2.7.2 instance attributes (registrations, languages).
  Increment replies_count on replies (MastoAPI)
  Comments split.
  Mastodon-based auth error messages. Defaulted User#auth_active?/1 to `true`.
  Try sending an empty map
  Attempt to fix incorrect federation of default instance avatars
  Add an id index to notifications
  fix up missing announcements with preloads
  Serve non-public activities in /api/v1/favourites
  add overriding truncated_namespace condition for truncating paths for digital ocean
  mrf/keyword_policy.ex: Fix when summary == nil, do not whitelist content == nil
  tests: fix up activity collision test
  activity: use left join instead of inner join when fetching activities that may or may not have a child object
  user: use preloads when deleting accounts
  ...

config/config.exs

@@ -173,7 +173,8 @@ config :pleroma, :instance,
   no_attachment_links: false,
   welcome_user_nickname: nil,
   welcome_message: nil,
-  max_report_comment_size: 1000
+  max_report_comment_size: 1000,
+  safe_dm_mentions: false
 
 config :pleroma, :markup,
   # XXX - unfortunately, inline images must be enabled by default right now, because

docs/Differences-in-MastodonAPI-Responses.md

@@ -19,6 +19,7 @@ Adding the parameter `with_muted=true` to the timeline queries will also return
 Has these additional fields under the `pleroma` object:
 
 - `local`: true if the post was made on the local instance.
+- `conversation_id`: the ID of the conversation the status is associated with (if any)
 
 ## Attachments
 

docs/config.md

@@ -101,7 +101,8 @@ config :pleroma, Pleroma.Mailer,
 * `no_attachment_links`: Set to true to disable automatically adding attachment link text to statuses
 * `welcome_message`: A message that will be send to a newly registered users as a direct message.
 * `welcome_user_nickname`: The nickname of the local user that sends the welcome message.
-* `max_report_size`: The maximum size of the report comment (Default: `1000`)
+* `max_report_comment_size`: The maximum size of the report comment (Default: `1000`)
+* `safe_dm_mentions`: If set to true, only mentions at the beginning of a post will be used to address people in direct messages. This is to prevent accidental mentioning of people when talking about them (e.g. "@friend hey i really don't like @enemy"). (Default: `false`)
 
 ## :logger
 * `backends`: `:console` is used to send logs to stdout, `{ExSyslogger, :ex_syslogger}` to log to syslog

lib/pleroma/activity.ex

@@ -7,6 +7,7 @@ defmodule Pleroma.Activity do
 
   alias Pleroma.Activity
   alias Pleroma.Notification
+  alias Pleroma.Object
   alias Pleroma.Repo
 
   import Ecto.Query
@@ -33,9 +34,42 @@ defmodule Pleroma.Activity do
     field(:recipients, {:array, :string})
     has_many(:notifications, Notification, on_delete: :delete_all)
 
+    # Attention: this is a fake relation, don't try to preload it blindly and expect it to work!
+    # The foreign key is embedded in a jsonb field.
+    #
+    # To use it, you probably want to do an inner join and a preload:
+    #
+    # ```
+    # |> join(:inner, [activity], o in Object,
+    #      on: fragment("(?->>'id') = COALESCE((?)->'object'->> 'id', (?)->>'object')",
+    #        o.data, activity.data, activity.data))
+    # |> preload([activity, object], [object: object])
+    # ```
+    #
+    # As a convenience, Activity.with_preloaded_object() sets up an inner join and preload for the
+    # typical case.
+    has_one(:object, Object, on_delete: :nothing, foreign_key: :id)
+
     timestamps()
   end
 
+  def with_preloaded_object(query) do
+    query
+    |> join(
+      :inner,
+      [activity],
+      o in Object,
+      on:
+        fragment(
+          "(?->>'id') = COALESCE(?->'object'->>'id', ?->>'object')",
+          o.data,
+          activity.data,
+          activity.data
+        )
+    )
+    |> preload([activity, object], object: object)
+  end
+
   def get_by_ap_id(ap_id) do
     Repo.one(
       from(
@@ -45,10 +79,44 @@ defmodule Pleroma.Activity do
     )
   end
 
+  def get_by_ap_id_with_object(ap_id) do
+    Repo.one(
+      from(
+        activity in Activity,
+        where: fragment("(?)->>'id' = ?", activity.data, ^to_string(ap_id)),
+        left_join: o in Object,
+        on:
+          fragment(
+            "(?->>'id') = COALESCE(?->'object'->>'id', ?->>'object')",
+            o.data,
+            activity.data,
+            activity.data
+          ),
+        preload: [object: o]
+      )
+    )
+  end
+
   def get_by_id(id) do
     Repo.get(Activity, id)
   end
 
+  def get_by_id_with_object(id) do
+    from(activity in Activity,
+      where: activity.id == ^id,
+      inner_join: o in Object,
+      on:
+        fragment(
+          "(?->>'id') = COALESCE(?->'object'->>'id', ?->>'object')",
+          o.data,
+          activity.data,
+          activity.data
+        ),
+      preload: [object: o]
+    )
+    |> Repo.one()
+  end
+
   def by_object_ap_id(ap_id) do
     from(
       activity in Activity,
@@ -76,7 +144,7 @@ defmodule Pleroma.Activity do
     )
   end
 
-  def create_by_object_ap_id(ap_id) do
+  def create_by_object_ap_id(ap_id) when is_binary(ap_id) do
     from(
       activity in Activity,
       where:
@@ -90,6 +158,8 @@ defmodule Pleroma.Activity do
     )
   end
 
+  def create_by_object_ap_id(_), do: nil
+
   def get_all_create_by_object_ap_id(ap_id) do
     Repo.all(create_by_object_ap_id(ap_id))
   end
@@ -101,8 +171,39 @@ defmodule Pleroma.Activity do
 
   def get_create_by_object_ap_id(_), do: nil
 
-  def normalize(obj) when is_map(obj), do: Activity.get_by_ap_id(obj["id"])
+  def create_by_object_ap_id_with_object(ap_id) when is_binary(ap_id) do
-  def normalize(ap_id) when is_binary(ap_id), do: Activity.get_by_ap_id(ap_id)
+    from(
+      activity in Activity,
+      where:
+        fragment(
+          "coalesce((?)->'object'->>'id', (?)->>'object') = ?",
+          activity.data,
+          activity.data,
+          ^to_string(ap_id)
+        ),
+      where: fragment("(?)->>'type' = 'Create'", activity.data),
+      inner_join: o in Object,
+      on:
+        fragment(
+          "(?->>'id') = COALESCE(?->'object'->>'id', ?->>'object')",
+          o.data,
+          activity.data,
+          activity.data
+        ),
+      preload: [object: o]
+    )
+  end
+
+  def create_by_object_ap_id_with_object(_), do: nil
+
+  def get_create_by_object_ap_id_with_object(ap_id) do
+    ap_id
+    |> create_by_object_ap_id_with_object()
+    |> Repo.one()
+  end
+
+  def normalize(obj) when is_map(obj), do: get_by_ap_id_with_object(obj["id"])
+  def normalize(ap_id) when is_binary(ap_id), do: get_by_ap_id_with_object(ap_id)
   def normalize(_), do: nil
 
   def get_in_reply_to_activity(%Activity{data: %{"object" => %{"inReplyTo" => ap_id}}}) do
@@ -144,4 +245,50 @@ defmodule Pleroma.Activity do
     |> where([s], s.actor == ^actor)
     |> Repo.all()
   end
+
+  def increase_replies_count(id) do
+    Activity
+    |> where(id: ^id)
+    |> update([a],
+      set: [
+        data:
+          fragment(
+            """
+            jsonb_set(?, '{object, repliesCount}',
+              (coalesce((?->'object'->>'repliesCount')::int, 0) + 1)::varchar::jsonb, true)
+            """,
+            a.data,
+            a.data
+          )
+      ]
+    )
+    |> Repo.update_all([])
+    |> case do
+      {1, [activity]} -> activity
+      _ -> {:error, "Not found"}
+    end
+  end
+
+  def decrease_replies_count(id) do
+    Activity
+    |> where(id: ^id)
+    |> update([a],
+      set: [
+        data:
+          fragment(
+            """
+            jsonb_set(?, '{object, repliesCount}',
+              (greatest(0, (?->'object'->>'repliesCount')::int - 1))::varchar::jsonb, true)
+            """,
+            a.data,
+            a.data
+          )
+      ]
+    )
+    |> Repo.update_all([])
+    |> case do
+      {1, [activity]} -> activity
+      _ -> {:error, "Not found"}
+    end
+  end
 end

lib/pleroma/emails/admin_email.ex

@@ -29,9 +29,13 @@ defmodule Pleroma.AdminEmail do
       if length(statuses) > 0 do
         statuses_list_html =
           statuses
-          |> Enum.map(fn %{id: id} ->
+          |> Enum.map(fn
+            %{id: id} ->
               status_url = Helpers.o_status_url(Pleroma.Web.Endpoint, :notice, id)
               "<li><a href=\"#{status_url}\">#{status_url}</li>"
+
+            id when is_binary(id) ->
+              "<li><a href=\"#{id}\">#{id}</li>"
           end)
           |> Enum.join("\n")
 

lib/pleroma/formatter.ex

@@ -8,6 +8,7 @@ defmodule Pleroma.Formatter do
   alias Pleroma.User
   alias Pleroma.Web.MediaProxy
 
+  @safe_mention_regex ~r/^(\s*(?<mentions>@.+?\s+)+)(?<rest>.*)/
   @markdown_characters_regex ~r/(`|\*|_|{|}|[|]|\(|\)|#|\+|-|\.|!)/
   @link_regex ~r{((?:http(s)?:\/\/)?[\w.-]+(?:\.[\w\.-]+)+[\w\-\._~%:/?#[\]@!\$&'\(\)\*\+,;=.]+)|[0-9a-z+\-\.]+:[0-9a-z$-_.+!*'(),]+}ui
   # credo:disable-for-previous-line Credo.Check.Readability.MaxLineLength
@@ -45,16 +46,29 @@ defmodule Pleroma.Formatter do
 
   @doc """
   Parses a text and replace plain text links with HTML. Returns a tuple with a result text, mentions, and hashtags.
+
+  If the 'safe_mention' option is given, only consecutive mentions at the start the post are actually mentioned.
   """
   @spec linkify(String.t(), keyword()) ::
           {String.t(), [{String.t(), User.t()}], [{String.t(), String.t()}]}
   def linkify(text, options \\ []) do
     options = options ++ @auto_linker_config
+
+    if options[:safe_mention] && Regex.named_captures(@safe_mention_regex, text) do
+      %{"mentions" => mentions, "rest" => rest} = Regex.named_captures(@safe_mention_regex, text)
+      acc = %{mentions: MapSet.new(), tags: MapSet.new()}
+
+      {text_mentions, %{mentions: mentions}} = AutoLinker.link_map(mentions, acc, options)
+      {text_rest, %{tags: tags}} = AutoLinker.link_map(rest, acc, options)
+
+      {text_mentions <> text_rest, MapSet.to_list(mentions), MapSet.to_list(tags)}
+    else
       acc = %{mentions: MapSet.new(), tags: MapSet.new()}
       {text, %{mentions: mentions, tags: tags}} = AutoLinker.link_map(text, acc, options)
 
       {text, MapSet.to_list(mentions), MapSet.to_list(tags)}
     end
+  end
 
   def emojify(text) do
     emojify(text, Emoji.get_all())

lib/pleroma/notification.ex

@@ -7,6 +7,7 @@ defmodule Pleroma.Notification do
 
   alias Pleroma.Activity
   alias Pleroma.Notification
+  alias Pleroma.Object
   alias Pleroma.Pagination
   alias Pleroma.Repo
   alias Pleroma.User
@@ -33,7 +34,15 @@ defmodule Pleroma.Notification do
     Notification
     |> where(user_id: ^user.id)
     |> join(:inner, [n], activity in assoc(n, :activity))
-    |> preload(:activity)
+    |> join(:left, [n, a], object in Object,
+      on:
+        fragment(
+          "(?->>'id') = COALESCE((? -> 'object'::text) ->> 'id'::text)",
+          object.data,
+          a.data
+        )
+    )
+    |> preload([n, a, o], activity: {a, object: o})
   end
 
   def for_user(user, opts \\ %{}) do

lib/pleroma/object.ex

@@ -14,6 +14,8 @@ defmodule Pleroma.Object do
   import Ecto.Query
   import Ecto.Changeset
 
+  require Logger
+
   schema "objects" do
     field(:data, :map)
 
@@ -38,6 +40,33 @@ defmodule Pleroma.Object do
     Repo.one(from(object in Object, where: fragment("(?)->>'id' = ?", object.data, ^ap_id)))
   end
 
+  # If we pass an Activity to Object.normalize(), we can try to use the preloaded object.
+  # Use this whenever possible, especially when walking graphs in an O(N) loop!
+  def normalize(%Activity{object: %Object{} = object}), do: object
+
+  # Catch and log Object.normalize() calls where the Activity's child object is not
+  # preloaded.
+  def normalize(%Activity{data: %{"object" => %{"id" => ap_id}}}) do
+    Logger.debug(
+      "Object.normalize() called without preloaded object (#{ap_id}).  Consider preloading the object!"
+    )
+
+    Logger.debug("Backtrace: #{inspect(Process.info(:erlang.self(), :current_stacktrace))}")
+
+    normalize(ap_id)
+  end
+
+  def normalize(%Activity{data: %{"object" => ap_id}}) do
+    Logger.debug(
+      "Object.normalize() called without preloaded object (#{ap_id}).  Consider preloading the object!"
+    )
+
+    Logger.debug("Backtrace: #{inspect(Process.info(:erlang.self(), :current_stacktrace))}")
+
+    normalize(ap_id)
+  end
+
+  # Old way, try fetching the object through cache.
   def normalize(%{"id" => ap_id}), do: normalize(ap_id)
   def normalize(ap_id) when is_binary(ap_id), do: get_cached_by_ap_id(ap_id)
   def normalize(_), do: nil
@@ -104,4 +133,50 @@ defmodule Pleroma.Object do
       e -> e
     end
   end
+
+  def increase_replies_count(ap_id) do
+    Object
+    |> where([o], fragment("?->>'id' = ?::text", o.data, ^to_string(ap_id)))
+    |> update([o],
+      set: [
+        data:
+          fragment(
+            """
+            jsonb_set(?, '{repliesCount}',
+              (coalesce((?->>'repliesCount')::int, 0) + 1)::varchar::jsonb, true)
+            """,
+            o.data,
+            o.data
+          )
+      ]
+    )
+    |> Repo.update_all([])
+    |> case do
+      {1, [object]} -> set_cache(object)
+      _ -> {:error, "Not found"}
+    end
+  end
+
+  def decrease_replies_count(ap_id) do
+    Object
+    |> where([o], fragment("?->>'id' = ?::text", o.data, ^to_string(ap_id)))
+    |> update([o],
+      set: [
+        data:
+          fragment(
+            """
+            jsonb_set(?, '{repliesCount}',
+              (greatest(0, (?->>'repliesCount')::int - 1))::varchar::jsonb, true)
+            """,
+            o.data,
+            o.data
+          )
+      ]
+    )
+    |> Repo.update_all([])
+    |> case do
+      {1, [object]} -> set_cache(object)
+      _ -> {:error, "Not found"}
+    end
+  end
 end

lib/pleroma/uploaders/s3.ex

@@ -13,9 +13,14 @@ defmodule Pleroma.Uploaders.S3 do
     bucket = Keyword.fetch!(config, :bucket)
 
     bucket_with_namespace =
-      if namespace = Keyword.get(config, :bucket_namespace) do
+      cond do
+        truncated_namespace = Keyword.get(config, :truncated_namespace) ->
+          truncated_namespace
+
+        namespace = Keyword.get(config, :bucket_namespace) ->
           namespace <> ":" <> bucket
-      else
+
+        true ->
           bucket
       end
 

lib/pleroma/user.ex

@@ -50,6 +50,7 @@ defmodule Pleroma.User do
     field(:local, :boolean, default: true)
     field(:follower_address, :string)
     field(:search_rank, :float, virtual: true)
+    field(:search_type, :integer, virtual: true)
     field(:tags, {:array, :string}, default: [])
     field(:bookmarks, {:array, :string}, default: [])
     field(:last_refreshed_at, :naive_datetime_usec)
@@ -59,14 +60,10 @@ defmodule Pleroma.User do
     timestamps()
   end
 
-  def auth_active?(%User{local: false}), do: true
-
-  def auth_active?(%User{info: %User.Info{confirmation_pending: false}}), do: true
-
   def auth_active?(%User{info: %User.Info{confirmation_pending: true}}),
     do: !Pleroma.Config.get([:instance, :account_activation_required])
 
-  def auth_active?(_), do: false
+  def auth_active?(%User{}), do: true
 
   def visible_for?(user, for_user \\ nil)
 
@@ -82,17 +79,17 @@ defmodule Pleroma.User do
   def superuser?(%User{local: true, info: %User.Info{is_moderator: true}}), do: true
   def superuser?(_), do: false
 
-  def avatar_url(user) do
+  def avatar_url(user, options \\ []) do
     case user.avatar do
       %{"url" => [%{"href" => href} | _]} -> href
-      _ -> "#{Web.base_url()}/images/avi.png"
+      _ -> !options[:no_default] && "#{Web.base_url()}/images/avi.png"
     end
   end
 
-  def banner_url(user) do
+  def banner_url(user, options \\ []) do
     case user.info.banner do
       %{"url" => [%{"href" => href} | _]} -> href
-      _ -> "#{Web.base_url()}/images/banner.png"
+      _ -> !options[:no_default] && "#{Web.base_url()}/images/banner.png"
     end
   end
 
@@ -823,31 +820,53 @@ defmodule Pleroma.User do
 
     if resolve, do: get_or_fetch(query)
 
-    fts_results = do_search(fts_search_subquery(query), for_user)
+    {:ok, results} =
-
-    {:ok, trigram_results} =
       Repo.transaction(fn ->
         Ecto.Adapters.SQL.query(Repo, "select set_limit(0.25)", [])
-        do_search(trigram_search_subquery(query), for_user)
+        Repo.all(search_query(query, for_user))
       end)
 
-    Enum.uniq_by(fts_results ++ trigram_results, & &1.id)
+    results
   end
 
-  defp do_search(subquery, for_user, options \\ []) do
+  def search_query(query, for_user) do
-    q =
+    fts_subquery = fts_search_subquery(query)
-      from(
+    trigram_subquery = trigram_search_subquery(query)
-        s in subquery(subquery),
+    union_query = from(s in trigram_subquery, union_all: ^fts_subquery)
+    distinct_query = from(s in subquery(union_query), order_by: s.search_type, distinct: s.id)
+
+    from(s in subquery(boost_search_rank_query(distinct_query, for_user)),
       order_by: [desc: s.search_rank],
-        limit: ^(options[:limit] || 20)
+      limit: 20
     )
+  end
 
-    results =
+  defp boost_search_rank_query(query, nil), do: query
-      q
-      |> Repo.all()
-      |> Enum.filter(&(&1.search_rank > 0))
 
-    boost_search_results(results, for_user)
+  defp boost_search_rank_query(query, for_user) do
+    friends_ids = get_friends_ids(for_user)
+    followers_ids = get_followers_ids(for_user)
+
+    from(u in subquery(query),
+      select_merge: %{
+        search_rank:
+          fragment(
+            """
+             CASE WHEN (?) THEN (?) * 1.3 
+             WHEN (?) THEN (?) * 1.2
+             WHEN (?) THEN (?) * 1.1
+             ELSE (?) END
+            """,
+            u.id in ^friends_ids and u.id in ^followers_ids,
+            u.search_rank,
+            u.id in ^friends_ids,
+            u.search_rank,
+            u.id in ^followers_ids,
+            u.search_rank,
+            u.search_rank
+          )
+      }
+    )
   end
 
   defp fts_search_subquery(term, query \\ User) do
@@ -862,6 +881,7 @@ defmodule Pleroma.User do
     from(
       u in query,
       select_merge: %{
+        search_type: ^0,
         search_rank:
           fragment(
             """
@@ -894,6 +914,8 @@ defmodule Pleroma.User do
     from(
       u in User,
       select_merge: %{
+        # ^1 gives 'Postgrex expected a binary, got 1' for some weird reason
+        search_type: fragment("?", 1),
         search_rank:
           fragment(
             "similarity(?, trim(? || ' ' || coalesce(?, '')))",
@@ -906,33 +928,6 @@ defmodule Pleroma.User do
     )
   end
 
-  defp boost_search_results(results, nil), do: results
-
-  defp boost_search_results(results, for_user) do
-    friends_ids = get_friends_ids(for_user)
-    followers_ids = get_followers_ids(for_user)
-
-    Enum.map(
-      results,
-      fn u ->
-        search_rank_coef =
-          cond do
-            u.id in friends_ids ->
-              1.2
-
-            u.id in followers_ids ->
-              1.1
-
-            true ->
-              1
-          end
-
-        Map.put(u, :search_rank, u.search_rank * search_rank_coef)
-      end
-    )
-    |> Enum.sort_by(&(-&1.search_rank))
-  end
-
   def blocks_import(%User{} = blocker, blocked_identifiers) when is_list(blocked_identifiers) do
     Enum.map(
       blocked_identifiers,
@@ -1111,13 +1106,15 @@ defmodule Pleroma.User do
     friends
     |> Enum.each(fn followed -> User.unfollow(user, followed) end)
 
-    query = from(a in Activity, where: a.actor == ^user.ap_id)
+    query =
+      from(a in Activity, where: a.actor == ^user.ap_id)
+      |> Activity.with_preloaded_object()
 
     Repo.all(query)
     |> Enum.each(fn activity ->
       case activity.data["type"] do
         "Create" ->
-          ActivityPub.delete(Object.normalize(activity.data["object"]))
+          ActivityPub.delete(Object.normalize(activity))
 
         # TODO: Do something with likes, follows, repeats.
         _ ->

lib/pleroma/web/activity_pub/activity_pub.ex

@@ -89,13 +89,37 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     if is_public?(object), do: User.decrease_note_count(actor), else: {:ok, actor}
   end
 
+  def increase_replies_count_if_reply(%{
+        "object" =>
+          %{"inReplyTo" => reply_ap_id, "inReplyToStatusId" => reply_status_id} = object,
+        "type" => "Create"
+      }) do
+    if is_public?(object) do
+      Activity.increase_replies_count(reply_status_id)
+      Object.increase_replies_count(reply_ap_id)
+    end
+  end
+
+  def increase_replies_count_if_reply(_create_data), do: :noop
+
+  def decrease_replies_count_if_reply(%Object{
+        data: %{"inReplyTo" => reply_ap_id, "inReplyToStatusId" => reply_status_id} = object
+      }) do
+    if is_public?(object) do
+      Activity.decrease_replies_count(reply_status_id)
+      Object.decrease_replies_count(reply_ap_id)
+    end
+  end
+
+  def decrease_replies_count_if_reply(_object), do: :noop
+
   def insert(map, local \\ true) when is_map(map) do
     with nil <- Activity.normalize(map),
          map <- lazy_put_activity_defaults(map),
          :ok <- check_actor_is_active(map["actor"]),
          {_, true} <- {:remote_limit_error, check_remote_limit(map)},
          {:ok, map} <- MRF.filter(map),
-         :ok <- insert_full_object(map) do
+         {:ok, object} <- insert_full_object(map) do
       {recipients, _, _} = get_recipients(map)
 
       {:ok, activity} =
@@ -106,6 +130,14 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
           recipients: recipients
         })
 
+      # Splice in the child object if we have one.
+      activity =
+        if !is_nil(object) do
+          Map.put(activity, :object, object)
+        else
+          activity
+        end
+
       Task.start(fn ->
         Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity)
       end)
@@ -170,6 +202,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
              additional
            ),
          {:ok, activity} <- insert(create_data, local),
+         _ <- increase_replies_count_if_reply(create_data),
          # Changing note count prior to enqueuing federation task in order to avoid
          # race conditions on updating user.info
          {:ok, _actor} <- increase_note_count_if_public(actor, activity),
@@ -321,6 +354,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
            "deleted_activity_id" => activity && activity.id
          },
          {:ok, activity} <- insert(data, local),
+         _ <- decrease_replies_count_if_reply(object),
          # Changing note count prior to enqueuing federation task in order to avoid
          # race conditions on updating user.info
          {:ok, _actor} <- decrease_note_count_if_public(user, object),
@@ -430,6 +464,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
           ),
         order_by: [desc: :id]
       )
+      |> Activity.with_preloaded_object()
 
     Repo.all(query)
   end
@@ -709,6 +744,13 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
 
   defp restrict_muted_reblogs(query, _), do: query
 
+  defp maybe_preload_objects(query, %{"skip_preload" => true}), do: query
+
+  defp maybe_preload_objects(query, _) do
+    query
+    |> Activity.with_preloaded_object()
+  end
+
   def fetch_activities_query(recipients, opts \\ %{}) do
     base_query =
       from(
@@ -718,6 +760,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
       )
 
     base_query
+    |> maybe_preload_objects(opts)
     |> restrict_recipients(recipients, opts["user"])
     |> restrict_tag(opts)
     |> restrict_tag_reject(opts)
@@ -940,7 +983,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
            },
            :ok <- Transmogrifier.contain_origin(id, params),
            {:ok, activity} <- Transmogrifier.handle_incoming(params) do
-        {:ok, Object.normalize(activity.data["object"])}
+        {:ok, Object.normalize(activity)}
       else
         {:error, {:reject, nil}} ->
           {:reject, nil}
@@ -952,7 +995,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
           Logger.info("Couldn't get object via AP, trying out OStatus fetching...")
 
           case OStatus.fetch_activity_from_url(id) do
-            {:ok, [activity | _]} -> {:ok, Object.normalize(activity.data["object"])}
+            {:ok, [activity | _]} -> {:ok, Object.normalize(activity)}
             e -> e
           end
       end

lib/pleroma/web/activity_pub/mrf/keyword_policy.ex

@@ -4,6 +4,10 @@
 
 defmodule Pleroma.Web.ActivityPub.MRF.KeywordPolicy do
   @behaviour Pleroma.Web.ActivityPub.MRF
+  defp string_matches?(string, _) when not is_binary(string) do
+    false
+  end
+
   defp string_matches?(string, pattern) when is_binary(pattern) do
     String.contains?(string, pattern)
   end
@@ -44,6 +48,20 @@ defmodule Pleroma.Web.ActivityPub.MRF.KeywordPolicy do
   end
 
   defp check_replace(%{"object" => %{"content" => content, "summary" => summary}} = message) do
+    content =
+      if is_binary(content) do
+        content
+      else
+        ""
+      end
+
+    summary =
+      if is_binary(summary) do
+        summary
+      else
+        ""
+      end
+
     {content, summary} =
       Enum.reduce(
         Pleroma.Config.get([:mrf_keyword, :replace]),
@@ -60,11 +78,6 @@ defmodule Pleroma.Web.ActivityPub.MRF.KeywordPolicy do
      |> put_in(["object", "summary"], summary)}
   end
 
-  @impl true
-  def filter(%{"object" => %{"content" => nil}} = message) do
-    {:ok, message}
-  end
-
   @impl true
   def filter(%{"type" => "Create", "object" => %{"content" => _content}} = message) do
     with {:ok, message} <- check_reject(message),

lib/pleroma/web/activity_pub/relay.ex

@@ -41,7 +41,7 @@ defmodule Pleroma.Web.ActivityPub.Relay do
 
   def publish(%Activity{data: %{"type" => "Create"}} = activity) do
     with %User{} = user <- get_actor(),
-         %Object{} = object <- Object.normalize(activity.data["object"]["id"]) do
+         %Object{} = object <- Object.normalize(activity) do
       ActivityPub.announce(user, object, nil, true, false)
     else
       e -> Logger.error("error: #{inspect(e)}")

lib/pleroma/web/activity_pub/utils.ex

@@ -209,12 +209,12 @@ defmodule Pleroma.Web.ActivityPub.Utils do
   """
   def insert_full_object(%{"object" => %{"type" => type} = object_data})
       when is_map(object_data) and type in @supported_object_types do
-    with {:ok, _} <- Object.create(object_data) do
+    with {:ok, object} <- Object.create(object_data) do
-      :ok
+      {:ok, object}
     end
   end
 
-  def insert_full_object(_), do: :ok
+  def insert_full_object(_), do: {:ok, nil}
 
   def update_object_in_activities(%{data: %{"id" => id}} = object) do
     # TODO

lib/pleroma/web/activity_pub/views/object_view.ex

@@ -17,7 +17,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectView do
 
   def render("object.json", %{object: %Activity{data: %{"type" => "Create"}} = activity}) do
     base = Pleroma.Web.ActivityPub.Utils.make_json_ld_header()
-    object = Object.normalize(activity.data["object"])
+    object = Object.normalize(activity)
 
     additional =
       Transmogrifier.prepare_object(activity.data)
@@ -28,7 +28,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectView do
 
   def render("object.json", %{object: %Activity{} = activity}) do
     base = Pleroma.Web.ActivityPub.Utils.make_json_ld_header()
-    object = Object.normalize(activity.data["object"])
+    object = Object.normalize(activity)
 
     additional =
       Transmogrifier.prepare_object(activity.data)

lib/pleroma/web/activity_pub/views/user_view.ex

@@ -87,16 +87,10 @@ defmodule Pleroma.Web.ActivityPub.UserView do
         "publicKeyPem" => public_key
       },
       "endpoints" => endpoints,
-      "icon" => %{
-        "type" => "Image",
-        "url" => User.avatar_url(user)
-      },
-      "image" => %{
-        "type" => "Image",
-        "url" => User.banner_url(user)
-      },
       "tag" => user.info.source_data["tag"] || []
     }
+    |> Map.merge(maybe_make_image(&User.avatar_url/2, "icon", user))
+    |> Map.merge(maybe_make_image(&User.banner_url/2, "image", user))
     |> Map.merge(Utils.make_json_ld_header())
   end
 
@@ -294,4 +288,17 @@ defmodule Pleroma.Web.ActivityPub.UserView do
       map
     end
   end
+
+  defp maybe_make_image(func, key, user) do
+    if image = func.(user, no_default: true) do
+      %{
+        key => %{
+          "type" => "Image",
+          "url" => image
+        }
+      }
+    else
+      %{}
+    end
+  end
 end

lib/pleroma/web/common_api/common_api.ex

@@ -6,7 +6,6 @@ defmodule Pleroma.Web.CommonAPI do
   alias Pleroma.Activity
   alias Pleroma.Formatter
   alias Pleroma.Object
-  alias Pleroma.Repo
   alias Pleroma.ThreadMute
   alias Pleroma.User
   alias Pleroma.Web.ActivityPub.ActivityPub
@@ -64,8 +63,9 @@ defmodule Pleroma.Web.CommonAPI do
   end
 
   def delete(activity_id, user) do
-    with %Activity{data: %{"object" => %{"id" => object_id}}} <- Repo.get(Activity, activity_id),
+    with %Activity{data: %{"object" => _}} = activity <-
-         %Object{} = object <- Object.normalize(object_id),
+           Activity.get_by_id_with_object(activity_id),
+         %Object{} = object <- Object.normalize(activity),
          true <- User.superuser?(user) || user.ap_id == object.data["actor"],
          {:ok, _} <- unpin(activity_id, user),
          {:ok, delete} <- ActivityPub.delete(object) do
@@ -75,7 +75,7 @@ defmodule Pleroma.Web.CommonAPI do
 
   def repeat(id_or_ap_id, user) do
     with %Activity{} = activity <- get_by_id_or_ap_id(id_or_ap_id),
-         object <- Object.normalize(activity.data["object"]["id"]),
+         object <- Object.normalize(activity),
          nil <- Utils.get_existing_announce(user.ap_id, object) do
       ActivityPub.announce(user, object)
     else
@@ -86,7 +86,7 @@ defmodule Pleroma.Web.CommonAPI do
 
   def unrepeat(id_or_ap_id, user) do
     with %Activity{} = activity <- get_by_id_or_ap_id(id_or_ap_id),
-         object <- Object.normalize(activity.data["object"]["id"]) do
+         object <- Object.normalize(activity) do
       ActivityPub.unannounce(user, object)
     else
       _ ->
@@ -96,7 +96,7 @@ defmodule Pleroma.Web.CommonAPI do
 
   def favorite(id_or_ap_id, user) do
     with %Activity{} = activity <- get_by_id_or_ap_id(id_or_ap_id),
-         object <- Object.normalize(activity.data["object"]["id"]),
+         object <- Object.normalize(activity),
          nil <- Utils.get_existing_like(user.ap_id, object) do
       ActivityPub.like(user, object)
     else
@@ -107,7 +107,7 @@ defmodule Pleroma.Web.CommonAPI do
 
   def unfavorite(id_or_ap_id, user) do
     with %Activity{} = activity <- get_by_id_or_ap_id(id_or_ap_id),
-         object <- Object.normalize(activity.data["object"]["id"]) do
+         object <- Object.normalize(activity) do
       ActivityPub.unlike(user, object)
     else
       _ ->
@@ -142,7 +142,8 @@ defmodule Pleroma.Web.CommonAPI do
            make_content_html(
              status,
              attachments,
-             data
+             data,
+             visibility
            ),
          {to, cc} <- to_for_user_and_mentions(user, mentions, in_reply_to, visibility),
          context <- make_context(in_reply_to),

lib/pleroma/web/common_api/utils.ex

@@ -17,13 +17,14 @@ defmodule Pleroma.Web.CommonAPI.Utils do
 
   # This is a hack for twidere.
   def get_by_id_or_ap_id(id) do
-    activity = Repo.get(Activity, id) || Activity.get_create_by_object_ap_id(id)
+    activity =
+      Activity.get_by_id_with_object(id) || Activity.get_create_by_object_ap_id_with_object(id)
 
     activity &&
       if activity.data["type"] == "Create" do
         activity
       else
-        Activity.get_create_by_object_ap_id(activity.data["object"])
+        Activity.get_create_by_object_ap_id_with_object(activity.data["object"])
       end
   end
 
@@ -101,7 +102,8 @@ defmodule Pleroma.Web.CommonAPI.Utils do
   def make_content_html(
         status,
         attachments,
-        data
+        data,
+        visibility
       ) do
     no_attachment_links =
       data
@@ -110,8 +112,15 @@ defmodule Pleroma.Web.CommonAPI.Utils do
 
     content_type = get_content_type(data["content_type"])
 
+    options =
+      if visibility == "direct" && Config.get([:instance, :safe_dm_mentions]) do
+        [safe_mention: true]
+      else
+        []
+      end
+
     status
-    |> format_input(content_type)
+    |> format_input(content_type, options)
     |> maybe_add_attachments(attachments, no_attachment_links)
     |> maybe_add_nsfw_tag(data)
   end
@@ -294,10 +303,10 @@ defmodule Pleroma.Web.CommonAPI.Utils do
 
   def maybe_notify_mentioned_recipients(
         recipients,
-        %Activity{data: %{"to" => _to, "type" => type} = data} = _activity
+        %Activity{data: %{"to" => _to, "type" => type} = data} = activity
       )
       when type == "Create" do
-    object = Object.normalize(data["object"])
+    object = Object.normalize(activity)
 
     object_data =
       cond do
@@ -344,4 +353,33 @@ defmodule Pleroma.Web.CommonAPI.Utils do
   end
 
   def get_report_statuses(_, _), do: {:ok, nil}
+
+  # DEPRECATED mostly, context objects are now created at insertion time.
+  def context_to_conversation_id(context) do
+    with %Object{id: id} <- Object.get_cached_by_ap_id(context) do
+      id
+    else
+      _e ->
+        changeset = Object.context_mapping(context)
+
+        case Repo.insert(changeset) do
+          {:ok, %{id: id}} ->
+            id
+
+          # This should be solved by an upsert, but it seems ecto
+          # has problems accessing the constraint inside the jsonb.
+          {:error, _} ->
+            Object.get_cached_by_ap_id(context).id
+        end
+    end
+  end
+
+  def conversation_id_to_context(id) do
+    with %Object{data: %{"id" => context}} <- Repo.get(Object, id) do
+      context
+    else
+      _e ->
+        {:error, "No such conversation"}
+    end
+  end
 end

lib/pleroma/web/mastodon_api/mastodon_api_controller.ex

@@ -161,6 +161,9 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
       },
       stats: Stats.get_stats(),
       thumbnail: Web.base_url() <> "/instance/thumbnail.jpeg",
+      languages: ["en"],
+      registrations: Pleroma.Config.get([:instance, :registrations_open]),
+      # Extra (not present in Mastodon):
       max_toot_chars: Keyword.get(instance, :limit)
     }
 
@@ -944,12 +947,14 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
   end
 
   def favourites(%{assigns: %{user: user}} = conn, params) do
-    activities =
+    params =
       params
       |> Map.put("type", "Create")
       |> Map.put("favorited_by", user.ap_id)
       |> Map.put("blocking_user", user)
-      |> ActivityPub.fetch_public_activities()
+
+    activities =
+      ActivityPub.fetch_activities([], params)
       |> Enum.reverse()
 
     conn

lib/pleroma/web/mastodon_api/views/status_view.ex

@@ -46,6 +46,14 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
     end
   end
 
+  defp get_context_id(%{data: %{"context_id" => context_id}}) when not is_nil(context_id),
+    do: context_id
+
+  defp get_context_id(%{data: %{"context" => context}}) when is_binary(context),
+    do: Utils.context_to_conversation_id(context)
+
+  defp get_context_id(_), do: nil
+
   def render("index.json", opts) do
     replied_to_activities = get_replied_to_activities(opts.activities)
 
@@ -166,7 +174,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
       content: content,
       created_at: created_at,
       reblogs_count: announcement_count,
-      replies_count: 0,
+      replies_count: object["repliesCount"] || 0,
       favourites_count: like_count,
       reblogged: present?(repeated),
       favourited: present?(favorited),
@@ -186,7 +194,8 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
       language: nil,
       emojis: build_emojis(activity.data["object"]["emoji"]),
       pleroma: %{
-        local: activity.local
+        local: activity.local,
+        conversation_id: get_context_id(activity)
       }
     }
   end

lib/pleroma/web/nodeinfo/nodeinfo_controller.ex

@@ -124,6 +124,9 @@ defmodule Pleroma.Web.Nodeinfo.NodeinfoController do
         end,
         if Keyword.get(instance, :allow_relay) do
           "relay"
+        end,
+        if Keyword.get(instance, :safe_dm_mentions) do
+          "safe_dm_mentions"
         end
       ]
       |> Enum.filter(& &1)

lib/pleroma/web/oauth/oauth_controller.ex

@@ -83,14 +83,18 @@ defmodule Pleroma.Web.OAuth.OAuthController do
       end
     else
       {scopes_issue, _} when scopes_issue in [:unsupported_scopes, :missing_scopes] ->
+        # Per https://github.com/tootsuite/mastodon/blob/
+        #   51e154f5e87968d6bb115e053689767ab33e80cd/app/controllers/api/base_controller.rb#L39
         conn
-        |> put_flash(:error, "Permissions not specified.")
+        |> put_flash(:error, "This action is outside the authorized scopes")
         |> put_status(:unauthorized)
         |> authorize(auth_params)
 
       {:auth_active, false} ->
+        # Per https://github.com/tootsuite/mastodon/blob/
+        #   51e154f5e87968d6bb115e053689767ab33e80cd/app/controllers/api/base_controller.rb#L76
         conn
-        |> put_flash(:error, "Account confirmation pending.")
+        |> put_flash(:error, "Your login is missing a confirmed e-mail address")
         |> put_status(:forbidden)
         |> authorize(auth_params)
 
@@ -149,9 +153,11 @@ defmodule Pleroma.Web.OAuth.OAuthController do
       json(conn, response)
     else
       {:auth_active, false} ->
+        # Per https://github.com/tootsuite/mastodon/blob/
+        #   51e154f5e87968d6bb115e053689767ab33e80cd/app/controllers/api/base_controller.rb#L76
         conn
         |> put_status(:forbidden)
-        |> json(%{error: "Account confirmation pending"})
+        |> json(%{error: "Your login is missing a confirmed e-mail address"})
 
       _error ->
         put_status(conn, 400)

lib/pleroma/web/ostatus/handlers/note_handler.ex

@@ -106,7 +106,7 @@ defmodule Pleroma.Web.OStatus.NoteHandler do
   # TODO: Clean this up a bit.
   def handle_note(entry, doc \\ nil) do
     with id <- XML.string_from_xpath("//id", entry),
-         activity when is_nil(activity) <- Activity.get_create_by_object_ap_id(id),
+         activity when is_nil(activity) <- Activity.get_create_by_object_ap_id_with_object(id),
          [author] <- :xmerl_xpath.string('//author[1]', doc),
          {:ok, actor} <- OStatus.find_make_or_update_user(author),
          content_html <- OStatus.get_content(entry),

lib/pleroma/web/ostatus/ostatus.ex

@@ -23,8 +23,8 @@ defmodule Pleroma.Web.OStatus do
   alias Pleroma.Web.WebFinger
   alias Pleroma.Web.Websub
 
-  def is_representable?(%Activity{data: data}) do
+  def is_representable?(%Activity{} = activity) do
-    object = Object.normalize(data["object"])
+    object = Object.normalize(activity)
 
     cond do
       is_nil(object) ->
@@ -119,7 +119,7 @@ defmodule Pleroma.Web.OStatus do
 
   def make_share(entry, doc, retweeted_activity) do
     with {:ok, actor} <- find_make_or_update_user(doc),
-         %Object{} = object <- Object.normalize(retweeted_activity.data["object"]),
+         %Object{} = object <- Object.normalize(retweeted_activity),
          id when not is_nil(id) <- string_from_xpath("/entry/id", entry),
          {:ok, activity, _object} = ActivityPub.announce(actor, object, id, false) do
       {:ok, activity}
@@ -137,7 +137,7 @@ defmodule Pleroma.Web.OStatus do
 
   def make_favorite(entry, doc, favorited_activity) do
     with {:ok, actor} <- find_make_or_update_user(doc),
-         %Object{} = object <- Object.normalize(favorited_activity.data["object"]),
+         %Object{} = object <- Object.normalize(favorited_activity),
          id when not is_nil(id) <- string_from_xpath("/entry/id", entry),
          {:ok, activity, _object} = ActivityPub.like(actor, object, id, false) do
       {:ok, activity}
@@ -159,7 +159,7 @@ defmodule Pleroma.Web.OStatus do
     Logger.debug("Trying to get entry from db")
 
     with id when not is_nil(id) <- string_from_xpath("//activity:object[1]/id", entry),
-         %Activity{} = activity <- Activity.get_create_by_object_ap_id(id) do
+         %Activity{} = activity <- Activity.get_create_by_object_ap_id_with_object(id) do
       {:ok, activity}
     else
       _ ->

lib/pleroma/web/ostatus/ostatus_controller.ex

@@ -102,7 +102,8 @@ defmodule Pleroma.Web.OStatus.OStatusController do
       ActivityPubController.call(conn, :object)
     else
       with id <- o_status_url(conn, :object, uuid),
-           {_, %Activity{} = activity} <- {:activity, Activity.get_create_by_object_ap_id(id)},
+           {_, %Activity{} = activity} <-
+             {:activity, Activity.get_create_by_object_ap_id_with_object(id)},
            {_, true} <- {:public?, Visibility.is_public?(activity)},
            %User{} = user <- User.get_cached_by_ap_id(activity.data["actor"]) do
         case get_format(conn) do
@@ -148,13 +149,13 @@ defmodule Pleroma.Web.OStatus.OStatusController do
   end
 
   def notice(conn, %{"id" => id}) do
-    with {_, %Activity{} = activity} <- {:activity, Activity.get_by_id(id)},
+    with {_, %Activity{} = activity} <- {:activity, Activity.get_by_id_with_object(id)},
          {_, true} <- {:public?, Visibility.is_public?(activity)},
          %User{} = user <- User.get_cached_by_ap_id(activity.data["actor"]) do
       case format = get_format(conn) do
         "html" ->
           if activity.data["type"] == "Create" do
-            %Object{} = object = Object.normalize(activity.data["object"])
+            %Object{} = object = Object.normalize(activity)
 
             Fallback.RedirectController.redirector_with_meta(conn, %{
               activity_id: activity.id,
@@ -191,9 +192,9 @@ defmodule Pleroma.Web.OStatus.OStatusController do
 
   # Returns an HTML embedded <audio> or <video> player suitable for embed iframes.
   def notice_player(conn, %{"id" => id}) do
-    with %Activity{data: %{"type" => "Create"}} = activity <- Activity.get_by_id(id),
+    with %Activity{data: %{"type" => "Create"}} = activity <- Activity.get_by_id_with_object(id),
          true <- Visibility.is_public?(activity),
-         %Object{} = object <- Object.normalize(activity.data["object"]),
+         %Object{} = object <- Object.normalize(activity),
          %{data: %{"attachment" => [%{"url" => [url | _]} | _]}} <- object,
          true <- String.starts_with?(url["mediaType"], ["audio", "video"]) do
       conn
@@ -219,7 +220,7 @@ defmodule Pleroma.Web.OStatus.OStatusController do
          %Activity{data: %{"type" => "Create"}} = activity,
          _user
        ) do
-    object = Object.normalize(activity.data["object"])
+    object = Object.normalize(activity)
 
     conn
     |> put_resp_header("content-type", "application/activity+json")

lib/pleroma/web/rich_media/helpers.ex

@@ -21,9 +21,9 @@ defmodule Pleroma.Web.RichMedia.Helpers do
   defp validate_page_url(%URI{}), do: :ok
   defp validate_page_url(_), do: :error
 
-  def fetch_data_for_activity(%Activity{} = activity) do
+  def fetch_data_for_activity(%Activity{data: %{"type" => "Create"}} = activity) do
     with true <- Pleroma.Config.get([:rich_media, :enabled]),
-         %Object{} = object <- Object.normalize(activity.data["object"]),
+         %Object{} = object <- Object.normalize(activity),
          {:ok, page_url} <- HTML.extract_first_external_url(object, object.data["content"]),
          :ok <- validate_page_url(page_url),
          {:ok, rich_media} <- Parser.parse(page_url) do
@@ -32,4 +32,6 @@ defmodule Pleroma.Web.RichMedia.Helpers do
       _ -> %{}
     end
   end
+
+  def fetch_data_for_activity(_), do: %{}
 end

lib/pleroma/web/streamer.ex

@@ -202,7 +202,7 @@ defmodule Pleroma.Web.Streamer do
         mutes = user.info.mutes || []
         reblog_mutes = user.info.muted_reblogs || []
 
-        parent = Object.normalize(item.data["object"])
+        parent = Object.normalize(item)
 
         unless is_nil(parent) or item.actor in blocks or item.actor in mutes or
                  item.actor in reblog_mutes or not ActivityPub.contain_activity(item, user) or

lib/pleroma/web/twitter_api/controllers/util_controller.ex

@@ -197,7 +197,9 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do
           vapidPublicKey: vapid_public_key,
           accountActivationRequired:
             if(Keyword.get(instance, :account_activation_required, false), do: "1", else: "0"),
-          invitesEnabled: if(Keyword.get(instance, :invites_enabled, false), do: "1", else: "0")
+          invitesEnabled: if(Keyword.get(instance, :invites_enabled, false), do: "1", else: "0"),
+          safeDMMentionsEnabled:
+            if(Pleroma.Config.get([:instance, :safe_dm_mentions]), do: "1", else: "0")
         }
 
         pleroma_fe =

lib/pleroma/web/twitter_api/representers/activity_representer.ex

@@ -1,15 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-# FIXME: Remove this module?
-# THIS MODULE IS DEPRECATED! DON'T USE IT!
-# USE THE Pleroma.Web.TwitterAPI.Views.ActivityView MODULE!
-defmodule Pleroma.Web.TwitterAPI.Representers.ActivityRepresenter do
-  def to_map(activity, opts) do
-    Pleroma.Web.TwitterAPI.ActivityView.render(
-      "activity.json",
-      Map.put(opts, :activity, activity)
-    )
-  end
-end

lib/pleroma/web/twitter_api/twitter_api.ex

@@ -5,7 +5,6 @@
 defmodule Pleroma.Web.TwitterAPI.TwitterAPI do
   alias Pleroma.Activity
   alias Pleroma.Mailer
-  alias Pleroma.Object
   alias Pleroma.Repo
   alias Pleroma.User
   alias Pleroma.UserEmail
@@ -282,35 +281,6 @@ defmodule Pleroma.Web.TwitterAPI.TwitterAPI do
     _activities = Repo.all(q)
   end
 
-  # DEPRECATED mostly, context objects are now created at insertion time.
-  def context_to_conversation_id(context) do
-    with %Object{id: id} <- Object.get_cached_by_ap_id(context) do
-      id
-    else
-      _e ->
-        changeset = Object.context_mapping(context)
-
-        case Repo.insert(changeset) do
-          {:ok, %{id: id}} ->
-            id
-
-          # This should be solved by an upsert, but it seems ecto
-          # has problems accessing the constraint inside the jsonb.
-          {:error, _} ->
-            Object.get_cached_by_ap_id(context).id
-        end
-    end
-  end
-
-  def conversation_id_to_context(id) do
-    with %Object{data: %{"id" => context}} <- Repo.get(Object, id) do
-      context
-    else
-      _e ->
-        {:error, "No such conversation"}
-    end
-  end
-
   def get_external_profile(for_user, uri) do
     with %User{} = user <- User.get_or_fetch(uri) do
       {:ok, UserView.render("show.json", %{user: user, for: for_user})}

lib/pleroma/web/twitter_api/twitter_api_controller.ex

@@ -16,6 +16,7 @@ defmodule Pleroma.Web.TwitterAPI.Controller do
   alias Pleroma.Web.ActivityPub.ActivityPub
   alias Pleroma.Web.ActivityPub.Visibility
   alias Pleroma.Web.CommonAPI
+  alias Pleroma.Web.CommonAPI.Utils
   alias Pleroma.Web.OAuth.Token
   alias Pleroma.Web.TwitterAPI.ActivityView
   alias Pleroma.Web.TwitterAPI.NotificationView
@@ -278,7 +279,7 @@ defmodule Pleroma.Web.TwitterAPI.Controller do
   end
 
   def fetch_conversation(%{assigns: %{user: user}} = conn, %{"id" => id}) do
-    with context when is_binary(context) <- TwitterAPI.conversation_id_to_context(id),
+    with context when is_binary(context) <- Utils.conversation_id_to_context(id),
          activities <-
            ActivityPub.fetch_activities_for_context(context, %{
              "blocking_user" => user,

lib/pleroma/web/twitter_api/views/activity_view.ex

@@ -15,7 +15,6 @@ defmodule Pleroma.Web.TwitterAPI.ActivityView do
   alias Pleroma.Web.MastodonAPI.StatusView
   alias Pleroma.Web.TwitterAPI.ActivityView
   alias Pleroma.Web.TwitterAPI.Representers.ObjectRepresenter
-  alias Pleroma.Web.TwitterAPI.TwitterAPI
   alias Pleroma.Web.TwitterAPI.UserView
 
   import Ecto.Query
@@ -78,7 +77,7 @@ defmodule Pleroma.Web.TwitterAPI.ActivityView do
   defp get_context_id(%{data: %{"context" => context}}, options) do
     cond do
       id = options[:context_ids][context] -> id
-      true -> TwitterAPI.context_to_conversation_id(context)
+      true -> Utils.context_to_conversation_id(context)
     end
   end
 
@@ -267,6 +266,8 @@ defmodule Pleroma.Web.TwitterAPI.ActivityView do
         content
         |> String.replace(~r/<br\s?\/?>/, "\n")
         |> HTML.get_cached_stripped_html_for_object(activity, __MODULE__)
+      else
+        ""
       end
 
     reply_parent = Activity.get_in_reply_to_activity(activity)

mix.exs

@@ -9,6 +9,7 @@ defmodule Pleroma.Mixfile do
       elixirc_paths: elixirc_paths(Mix.env()),
       compilers: [:phoenix, :gettext] ++ Mix.compilers(),
       elixirc_options: [warnings_as_errors: true],
+      xref: [exclude: [:eldap]],
       start_permanent: Mix.env() == :prod,
       aliases: aliases(),
       deps: deps(),

priv/repo/migrations/20190325185009_create_notification_id_index.exs

@@ -0,0 +1,7 @@
+defmodule Pleroma.Repo.Migrations.CreateNotificationIdIndex do
+  use Ecto.Migration
+
+  def change do
+  create index(:notifications, ["id desc nulls last"])
+  end
+end

priv/repo/migrations/20190325215156_update_status_reply_count.exs

@@ -0,0 +1,48 @@
+defmodule Pleroma.Repo.Migrations.UpdateStatusReplyCount do
+  use Ecto.Migration
+
+  @public "https://www.w3.org/ns/activitystreams#Public"
+
+  def up do
+    execute("""
+      WITH reply_count AS (
+        SELECT count(*) AS count, data->>'inReplyTo' AS ap_id
+        FROM objects
+        WHERE
+          data->>'inReplyTo' IS NOT NULL AND
+          data->>'type' = 'Note' AND (
+            data->'cc' ? '#{@public}' OR
+            data->'to' ? '#{@public}')
+        GROUP BY data->>'inReplyTo'
+      )
+      UPDATE objects AS o
+      SET "data" = jsonb_set(o.data, '{repliesCount}', reply_count.count::varchar::jsonb, true)
+      FROM reply_count
+      WHERE reply_count.ap_id = o.data->>'id';
+    """)
+
+    execute("""
+      WITH reply_count AS (SELECT
+          count(*) as count,
+          data->'object'->>'inReplyTo' AS ap_id
+        FROM
+          activities
+        WHERE
+          data->'object'->>'inReplyTo' IS NOT NULL AND
+          data->'object'->>'type' = 'Note' AND (
+            data->'object'->'cc' ? '#{@public}' OR
+            data->'object'->'to' ? '#{@public}')
+        GROUP BY
+          data->'object'->>'inReplyTo'
+      )
+      UPDATE activities AS a
+      SET "data" = jsonb_set(a.data, '{object, repliesCount}', reply_count.count::varchar::jsonb, true)
+      FROM reply_count
+      WHERE reply_count.ap_id = a.data->'object'->>'id';
+    """)
+  end
+
+  def down do
+    :noop
+  end
+end

test/formatter_test.exs

@@ -181,6 +181,31 @@ defmodule Pleroma.FormatterTest do
       expected_text = "@a hi"
       assert {^expected_text, [] = _mentions, [] = _tags} = Formatter.linkify(text)
     end
+
+    test "given the 'safe_mention' option, it will only mention people in the beginning" do
+      user = insert(:user)
+      _other_user = insert(:user)
+      third_user = insert(:user)
+      text = " @#{user.nickname} hey dude i hate @#{third_user.nickname}"
+      {expected_text, mentions, [] = _tags} = Formatter.linkify(text, safe_mention: true)
+
+      assert mentions == [{"@#{user.nickname}", user}]
+
+      assert expected_text ==
+               "<span class='h-card'><a data-user='#{user.id}' class='u-url mention' href='#{
+                 user.ap_id
+               }'>@<span>#{user.nickname}</span></a></span> hey dude i hate <span class='h-card'><a data-user='#{
+                 third_user.id
+               }' class='u-url mention' href='#{third_user.ap_id}'>@<span>#{third_user.nickname}</span></a></span>"
+    end
+
+    test "given the 'safe_mention' option, it will still work without any mention" do
+      text = "A post without any mention"
+      {expected_text, mentions, [] = _tags} = Formatter.linkify(text, safe_mention: true)
+
+      assert mentions == []
+      assert expected_text == text
+    end
   end
 
   describe ".parse_tags" do

test/tasks/relay_test.exs

@@ -60,7 +60,8 @@ defmodule Mix.Tasks.Pleroma.RelayTest do
         ActivityPub.fetch_activities([], %{
           "type" => "Undo",
           "actor_id" => follower_id,
-          "limit" => 1
+          "limit" => 1,
+          "skip_preload" => true
         })
 
       assert undo_activity.data["type"] == "Undo"

test/user_test.exs

@@ -879,7 +879,11 @@ defmodule Pleroma.UserTest do
       user = insert(:user, %{nickname: "john"})
 
       Enum.each(["john", "jo", "j"], fn query ->
-        assert user == User.search(query) |> List.first() |> Map.put(:search_rank, nil)
+        assert user ==
+                 User.search(query)
+                 |> List.first()
+                 |> Map.put(:search_rank, nil)
+                 |> Map.put(:search_type, nil)
       end)
     end
 
@@ -887,7 +891,11 @@ defmodule Pleroma.UserTest do
       user = insert(:user, %{name: "John Doe"})
 
       Enum.each(["John Doe", "JOHN", "doe", "j d", "j", "d"], fn query ->
-        assert user == User.search(query) |> List.first() |> Map.put(:search_rank, nil)
+        assert user ==
+                 User.search(query)
+                 |> List.first()
+                 |> Map.put(:search_rank, nil)
+                 |> Map.put(:search_type, nil)
       end)
     end
 
@@ -941,6 +949,7 @@ defmodule Pleroma.UserTest do
                User.search("lain@pleroma.soykaf.com")
                |> List.first()
                |> Map.put(:search_rank, nil)
+               |> Map.put(:search_type, nil)
     end
 
     test "does not yield false-positive matches" do
@@ -958,7 +967,7 @@ defmodule Pleroma.UserTest do
       user = User.get_by_ap_id("http://mastodon.example.org/users/admin")
 
       assert length(results) == 1
-      assert user == result |> Map.put(:search_rank, nil)
+      assert user == result |> Map.put(:search_rank, nil) |> Map.put(:search_type, nil)
     end
   end
 

test/web/activity_pub/activity_pub_test.exs

@@ -140,7 +140,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do
       activity = insert(:note_activity)
       {:ok, new_activity} = ActivityPub.insert(activity.data)
 
-      assert activity == new_activity
+      assert activity.id == new_activity.id
     end
 
     test "inserts a given map into the activity database, giving it an id if it has none." do
@@ -232,6 +232,39 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do
       user = Repo.get(User, user.id)
       assert user.info.note_count == 2
     end
+
+    test "increases replies count" do
+      user = insert(:user)
+      user2 = insert(:user)
+
+      {:ok, activity} = CommonAPI.post(user, %{"status" => "1", "visibility" => "public"})
+      ap_id = activity.data["id"]
+      reply_data = %{"status" => "1", "in_reply_to_status_id" => activity.id}
+
+      # public
+      {:ok, _} = CommonAPI.post(user2, Map.put(reply_data, "visibility", "public"))
+      assert %{data: data, object: object} = Activity.get_by_ap_id_with_object(ap_id)
+      assert data["object"]["repliesCount"] == 1
+      assert object.data["repliesCount"] == 1
+
+      # unlisted
+      {:ok, _} = CommonAPI.post(user2, Map.put(reply_data, "visibility", "unlisted"))
+      assert %{data: data, object: object} = Activity.get_by_ap_id_with_object(ap_id)
+      assert data["object"]["repliesCount"] == 2
+      assert object.data["repliesCount"] == 2
+
+      # private
+      {:ok, _} = CommonAPI.post(user2, Map.put(reply_data, "visibility", "private"))
+      assert %{data: data, object: object} = Activity.get_by_ap_id_with_object(ap_id)
+      assert data["object"]["repliesCount"] == 2
+      assert object.data["repliesCount"] == 2
+
+      # direct
+      {:ok, _} = CommonAPI.post(user2, Map.put(reply_data, "visibility", "direct"))
+      assert %{data: data, object: object} = Activity.get_by_ap_id_with_object(ap_id)
+      assert data["object"]["repliesCount"] == 2
+      assert object.data["repliesCount"] == 2
+    end
   end
 
   describe "fetch activities for recipients" do
@@ -270,7 +303,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do
     booster = insert(:user)
     {:ok, user} = User.block(user, %{ap_id: activity_one.data["actor"]})
 
-    activities = ActivityPub.fetch_activities([], %{"blocking_user" => user})
+    activities =
+      ActivityPub.fetch_activities([], %{"blocking_user" => user, "skip_preload" => true})
 
     assert Enum.member?(activities, activity_two)
     assert Enum.member?(activities, activity_three)
@@ -278,7 +312,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do
 
     {:ok, user} = User.unblock(user, %{ap_id: activity_one.data["actor"]})
 
-    activities = ActivityPub.fetch_activities([], %{"blocking_user" => user})
+    activities =
+      ActivityPub.fetch_activities([], %{"blocking_user" => user, "skip_preload" => true})
 
     assert Enum.member?(activities, activity_two)
     assert Enum.member?(activities, activity_three)
@@ -289,14 +324,16 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do
     %Activity{} = boost_activity = Activity.get_create_by_object_ap_id(id)
     activity_three = Repo.get(Activity, activity_three.id)
 
-    activities = ActivityPub.fetch_activities([], %{"blocking_user" => user})
+    activities =
+      ActivityPub.fetch_activities([], %{"blocking_user" => user, "skip_preload" => true})
 
     assert Enum.member?(activities, activity_two)
     refute Enum.member?(activities, activity_three)
     refute Enum.member?(activities, boost_activity)
     assert Enum.member?(activities, activity_one)
 
-    activities = ActivityPub.fetch_activities([], %{"blocking_user" => nil})
+    activities =
+      ActivityPub.fetch_activities([], %{"blocking_user" => nil, "skip_preload" => true})
 
     assert Enum.member?(activities, activity_two)
     assert Enum.member?(activities, activity_three)
@@ -312,14 +349,20 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do
     booster = insert(:user)
     {:ok, user} = User.mute(user, %User{ap_id: activity_one.data["actor"]})
 
-    activities = ActivityPub.fetch_activities([], %{"muting_user" => user})
+    activities =
+      ActivityPub.fetch_activities([], %{"muting_user" => user, "skip_preload" => true})
 
     assert Enum.member?(activities, activity_two)
     assert Enum.member?(activities, activity_three)
     refute Enum.member?(activities, activity_one)
 
     # Calling with 'with_muted' will deliver muted activities, too.
-    activities = ActivityPub.fetch_activities([], %{"muting_user" => user, "with_muted" => true})
+    activities =
+      ActivityPub.fetch_activities([], %{
+        "muting_user" => user,
+        "with_muted" => true,
+        "skip_preload" => true
+      })
 
     assert Enum.member?(activities, activity_two)
     assert Enum.member?(activities, activity_three)
@@ -327,7 +370,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do
 
     {:ok, user} = User.unmute(user, %User{ap_id: activity_one.data["actor"]})
 
-    activities = ActivityPub.fetch_activities([], %{"muting_user" => user})
+    activities =
+      ActivityPub.fetch_activities([], %{"muting_user" => user, "skip_preload" => true})
 
     assert Enum.member?(activities, activity_two)
     assert Enum.member?(activities, activity_three)
@@ -338,14 +382,15 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do
     %Activity{} = boost_activity = Activity.get_create_by_object_ap_id(id)
     activity_three = Repo.get(Activity, activity_three.id)
 
-    activities = ActivityPub.fetch_activities([], %{"muting_user" => user})
+    activities =
+      ActivityPub.fetch_activities([], %{"muting_user" => user, "skip_preload" => true})
 
     assert Enum.member?(activities, activity_two)
     refute Enum.member?(activities, activity_three)
     refute Enum.member?(activities, boost_activity)
     assert Enum.member?(activities, activity_one)
 
-    activities = ActivityPub.fetch_activities([], %{"muting_user" => nil})
+    activities = ActivityPub.fetch_activities([], %{"muting_user" => nil, "skip_preload" => true})
 
     assert Enum.member?(activities, activity_two)
     assert Enum.member?(activities, activity_three)
@@ -353,6 +398,20 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do
     assert Enum.member?(activities, activity_one)
   end
 
+  test "does include announces on request" do
+    activity_three = insert(:note_activity)
+    user = insert(:user)
+    booster = insert(:user)
+
+    {:ok, user} = User.follow(user, booster)
+
+    {:ok, announce, _object} = CommonAPI.repeat(activity_three.id, booster)
+
+    [announce_activity] = ActivityPub.fetch_activities([user.ap_id | user.following])
+
+    assert announce_activity.id == announce.id
+  end
+
   test "excludes reblogs on request" do
     user = insert(:user)
     {:ok, expected_activity} = ActivityBuilder.insert(%{"type" => "Create"}, %{:user => user})
@@ -725,6 +784,40 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do
 
       assert user.ap_id in delete.data["to"]
     end
+
+    test "decreases reply count" do
+      user = insert(:user)
+      user2 = insert(:user)
+
+      {:ok, activity} = CommonAPI.post(user, %{"status" => "1", "visibility" => "public"})
+      reply_data = %{"status" => "1", "in_reply_to_status_id" => activity.id}
+      ap_id = activity.data["id"]
+
+      {:ok, public_reply} = CommonAPI.post(user2, Map.put(reply_data, "visibility", "public"))
+      {:ok, unlisted_reply} = CommonAPI.post(user2, Map.put(reply_data, "visibility", "unlisted"))
+      {:ok, private_reply} = CommonAPI.post(user2, Map.put(reply_data, "visibility", "private"))
+      {:ok, direct_reply} = CommonAPI.post(user2, Map.put(reply_data, "visibility", "direct"))
+
+      _ = CommonAPI.delete(direct_reply.id, user2)
+      assert %{data: data, object: object} = Activity.get_by_ap_id_with_object(ap_id)
+      assert data["object"]["repliesCount"] == 2
+      assert object.data["repliesCount"] == 2
+
+      _ = CommonAPI.delete(private_reply.id, user2)
+      assert %{data: data, object: object} = Activity.get_by_ap_id_with_object(ap_id)
+      assert data["object"]["repliesCount"] == 2
+      assert object.data["repliesCount"] == 2
+
+      _ = CommonAPI.delete(public_reply.id, user2)
+      assert %{data: data, object: object} = Activity.get_by_ap_id_with_object(ap_id)
+      assert data["object"]["repliesCount"] == 1
+      assert object.data["repliesCount"] == 1
+
+      _ = CommonAPI.delete(unlisted_reply.id, user2)
+      assert %{data: data, object: object} = Activity.get_by_ap_id_with_object(ap_id)
+      assert data["object"]["repliesCount"] == 0
+      assert object.data["repliesCount"] == 0
+    end
   end
 
   describe "timeline post-processing" do
@@ -763,6 +856,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do
 
       activities = ActivityPub.fetch_activities([user1.ap_id | user1.following])
 
+      private_activity_1 = Activity.get_by_ap_id_with_object(private_activity_1.data["id"])
       assert [public_activity, private_activity_1, private_activity_3] == activities
       assert length(activities) == 3
 

test/web/activity_pub/views/user_view_test.exs

@@ -16,6 +16,29 @@ defmodule Pleroma.Web.ActivityPub.UserViewTest do
     assert String.contains?(result["publicKey"]["publicKeyPem"], "BEGIN PUBLIC KEY")
   end
 
+  test "Does not add an avatar image if the user hasn't set one" do
+    user = insert(:user)
+    {:ok, user} = Pleroma.Web.WebFinger.ensure_keys_present(user)
+
+    result = UserView.render("user.json", %{user: user})
+    refute result["icon"]
+    refute result["image"]
+
+    user =
+      insert(:user,
+        avatar: %{"url" => [%{"href" => "https://someurl"}]},
+        info: %{
+          banner: %{"url" => [%{"href" => "https://somebanner"}]}
+        }
+      )
+
+    {:ok, user} = Pleroma.Web.WebFinger.ensure_keys_present(user)
+
+    result = UserView.render("user.json", %{user: user})
+    assert result["icon"]["url"] == "https://someurl"
+    assert result["image"]["url"] == "https://somebanner"
+  end
+
   describe "endpoints" do
     test "local users have a usable endpoints structure" do
       user = insert(:user)

test/web/common_api/common_api_test.exs

@@ -10,6 +10,24 @@ defmodule Pleroma.Web.CommonAPITest do
 
   import Pleroma.Factory
 
+  test "with the safe_dm_mention option set, it does not mention people beyond the initial tags" do
+    har = insert(:user)
+    jafnhar = insert(:user)
+    tridi = insert(:user)
+    option = Pleroma.Config.get([:instance, :safe_dm_mentions])
+    Pleroma.Config.put([:instance, :safe_dm_mentions], true)
+
+    {:ok, activity} =
+      CommonAPI.post(har, %{
+        "status" => "@#{jafnhar.nickname} hey, i never want to see @#{tridi.nickname} again",
+        "visibility" => "direct"
+      })
+
+    refute tridi.ap_id in activity.recipients
+    assert jafnhar.ap_id in activity.recipients
+    Pleroma.Config.put([:instance, :safe_dm_mentions], option)
+  end
+
   test "it de-duplicates tags" do
     user = insert(:user)
     {:ok, activity} = CommonAPI.post(user, %{"status" => "#2hu #2HU"})

test/web/common_api/common_api_utils_test.exs

@@ -4,6 +4,7 @@
 
 defmodule Pleroma.Web.CommonAPI.UtilsTest do
   alias Pleroma.Builders.UserBuilder
+  alias Pleroma.Object
   alias Pleroma.Web.CommonAPI.Utils
   alias Pleroma.Web.Endpoint
   use Pleroma.DataCase
@@ -136,4 +137,20 @@ defmodule Pleroma.Web.CommonAPI.UtilsTest do
       assert output == expected
     end
   end
+
+  describe "context_to_conversation_id" do
+    test "creates a mapping object" do
+      conversation_id = Utils.context_to_conversation_id("random context")
+      object = Object.get_by_ap_id("random context")
+
+      assert conversation_id == object.id
+    end
+
+    test "returns an existing mapping for an existing object" do
+      {:ok, object} = Object.context_mapping("random context") |> Repo.insert()
+      conversation_id = Utils.context_to_conversation_id("random context")
+
+      assert conversation_id == object.id
+    end
+  end
 end

test/web/mastodon_api/mastodon_api_controller_test.exs

@@ -1808,6 +1808,27 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do
   end
 
   test "get instance information", %{conn: conn} do
+    conn = get(conn, "/api/v1/instance")
+    assert result = json_response(conn, 200)
+
+    # Note: not checking for "max_toot_chars" since it's optional
+    assert %{
+             "uri" => _,
+             "title" => _,
+             "description" => _,
+             "version" => _,
+             "email" => _,
+             "urls" => %{
+               "streaming_api" => _
+             },
+             "stats" => _,
+             "thumbnail" => _,
+             "languages" => _,
+             "registrations" => _
+           } = result
+  end
+
+  test "get instance stats", %{conn: conn} do
     user = insert(:user, %{local: true})
 
     user2 = insert(:user, %{local: true})

test/web/mastodon_api/status_view_test.exs

@@ -9,6 +9,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusViewTest do
   alias Pleroma.User
   alias Pleroma.Web.ActivityPub.ActivityPub
   alias Pleroma.Web.CommonAPI
+  alias Pleroma.Web.CommonAPI.Utils
   alias Pleroma.Web.MastodonAPI.AccountView
   alias Pleroma.Web.MastodonAPI.StatusView
   alias Pleroma.Web.OStatus
@@ -72,6 +73,8 @@ defmodule Pleroma.Web.MastodonAPI.StatusViewTest do
     note = insert(:note_activity)
     user = User.get_cached_by_ap_id(note.data["actor"])
 
+    convo_id = Utils.context_to_conversation_id(note.data["object"]["context"])
+
     status = StatusView.render("status.json", %{activity: note})
 
     created_at =
@@ -122,7 +125,8 @@ defmodule Pleroma.Web.MastodonAPI.StatusViewTest do
         }
       ],
       pleroma: %{
-        local: true
+        local: true,
+        conversation_id: convo_id
       }
     }
 

test/web/node_info_test.exs

@@ -108,4 +108,27 @@ defmodule Pleroma.Web.NodeInfoTest do
     assert result = json_response(conn, 200)
     assert Pleroma.Application.repository() == result["software"]["repository"]
   end
+
+  test "it returns the safe_dm_mentions feature if enabled", %{conn: conn} do
+    option = Pleroma.Config.get([:instance, :safe_dm_mentions])
+    Pleroma.Config.put([:instance, :safe_dm_mentions], true)
+
+    response =
+      conn
+      |> get("/nodeinfo/2.1.json")
+      |> json_response(:ok)
+
+    assert "safe_dm_mentions" in response["metadata"]["features"]
+
+    Pleroma.Config.put([:instance, :safe_dm_mentions], false)
+
+    response =
+      conn
+      |> get("/nodeinfo/2.1.json")
+      |> json_response(:ok)
+
+    refute "safe_dm_mentions" in response["metadata"]["features"]
+
+    Pleroma.Config.put([:instance, :safe_dm_mentions], option)
+  end
 end

test/web/oauth/ldap_authorization_test.exs

@@ -10,6 +10,8 @@ defmodule Pleroma.Web.OAuth.LDAPAuthorizationTest do
   import ExUnit.CaptureLog
   import Mock
 
+  @skip if !Code.ensure_loaded?(:eldap), do: :skip
+
   setup_all do
     ldap_authenticator =
       Pleroma.Config.get(Pleroma.Web.Auth.Authenticator, Pleroma.Web.Auth.PleromaAuthenticator)
@@ -27,6 +29,7 @@ defmodule Pleroma.Web.OAuth.LDAPAuthorizationTest do
     :ok
   end
 
+  @tag @skip
   test "authorizes the existing user using LDAP credentials" do
     password = "testpassword"
     user = insert(:user, password_hash: Comeonin.Pbkdf2.hashpwsalt(password))
@@ -65,6 +68,7 @@ defmodule Pleroma.Web.OAuth.LDAPAuthorizationTest do
     end
   end
 
+  @tag @skip
   test "creates a new user after successful LDAP authorization" do
     password = "testpassword"
     user = build(:user)
@@ -110,6 +114,7 @@ defmodule Pleroma.Web.OAuth.LDAPAuthorizationTest do
     end
   end
 
+  @tag @skip
   test "falls back to the default authorization when LDAP is unavailable" do
     password = "testpassword"
     user = insert(:user, password_hash: Comeonin.Pbkdf2.hashpwsalt(password))
@@ -153,6 +158,7 @@ defmodule Pleroma.Web.OAuth.LDAPAuthorizationTest do
     end
   end
 
+  @tag @skip
   test "disallow authorization for wrong LDAP credentials" do
     password = "testpassword"
     user = insert(:user, password_hash: Comeonin.Pbkdf2.hashpwsalt(password))

test/web/oauth/oauth_controller_test.exs

@@ -87,7 +87,7 @@ defmodule Pleroma.Web.OAuth.OAuthControllerTest do
     assert result =~ app.redirect_uris
 
     # Error message
-    assert result =~ "Permissions not specified"
+    assert result =~ "This action is outside the authorized scopes"
   end
 
   test "returns 401 for scopes beyond app scopes", %{conn: conn} do
@@ -113,7 +113,7 @@ defmodule Pleroma.Web.OAuth.OAuthControllerTest do
     assert result =~ app.redirect_uris
 
     # Error message
-    assert result =~ "Permissions not specified"
+    assert result =~ "This action is outside the authorized scopes"
   end
 
   test "issues a token for an all-body request" do

test/web/twitter_api/representers/activity_representer_test.exs

@@ -1,170 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Pleroma.Web.TwitterAPI.Representers.ActivityRepresenterTest do
-  use Pleroma.DataCase
-  alias Pleroma.Activity
-  alias Pleroma.Object
-  alias Pleroma.User
-  alias Pleroma.Web.ActivityPub.ActivityPub
-  alias Pleroma.Web.TwitterAPI.Representers.ActivityRepresenter
-  alias Pleroma.Web.TwitterAPI.Representers.ObjectRepresenter
-  alias Pleroma.Web.TwitterAPI.UserView
-  import Pleroma.Factory
-
-  test "a like activity" do
-    user = insert(:user)
-    note_activity = insert(:note_activity)
-    object = Object.get_by_ap_id(note_activity.data["object"]["id"])
-
-    {:ok, like_activity, _object} = ActivityPub.like(user, object)
-
-    status =
-      ActivityRepresenter.to_map(like_activity, %{user: user, liked_activity: note_activity})
-
-    assert status["id"] == like_activity.id
-    assert status["in_reply_to_status_id"] == note_activity.id
-
-    note_activity = Activity.get_by_ap_id(note_activity.data["id"])
-    activity_actor = Repo.get_by(User, ap_id: note_activity.data["actor"])
-    liked_status = ActivityRepresenter.to_map(note_activity, %{user: activity_actor, for: user})
-    assert liked_status["favorited"] == true
-    assert status["activity_type"] == "like"
-  end
-
-  test "an activity" do
-    user = insert(:user)
-    #   {:ok, mentioned_user } = UserBuilder.insert(%{nickname: "shp", ap_id: "shp"})
-    mentioned_user = insert(:user, %{nickname: "shp"})
-
-    # {:ok, follower} = UserBuilder.insert(%{following: [User.ap_followers(user)]})
-    follower = insert(:user, %{following: [User.ap_followers(user)]})
-
-    object = %Object{
-      data: %{
-        "type" => "Image",
-        "url" => [
-          %{
-            "type" => "Link",
-            "mediaType" => "image/jpg",
-            "href" => "http://example.org/image.jpg"
-          }
-        ],
-        "uuid" => 1
-      }
-    }
-
-    content_html =
-      "<script>alert('YAY')</script>Some :2hu: content mentioning <a href='#{mentioned_user.ap_id}'>@shp</shp>"
-
-    content = HtmlSanitizeEx.strip_tags(content_html)
-    date = DateTime.from_naive!(~N[2016-05-24 13:26:08.003], "Etc/UTC") |> DateTime.to_iso8601()
-
-    {:ok, convo_object} = Object.context_mapping("2hu") |> Repo.insert()
-
-    to = [
-      User.ap_followers(user),
-      "https://www.w3.org/ns/activitystreams#Public",
-      mentioned_user.ap_id
-    ]
-
-    activity = %Activity{
-      id: 1,
-      data: %{
-        "type" => "Create",
-        "id" => "id",
-        "to" => to,
-        "actor" => User.ap_id(user),
-        "object" => %{
-          "published" => date,
-          "type" => "Note",
-          "content" => content_html,
-          "summary" => "2hu :2hu:",
-          "inReplyToStatusId" => 213_123,
-          "attachment" => [
-            object
-          ],
-          "external_url" => "some url",
-          "like_count" => 5,
-          "announcement_count" => 3,
-          "context" => "2hu",
-          "tag" => ["content", "mentioning", "nsfw"],
-          "emoji" => %{
-            "2hu" => "corndog.png"
-          }
-        },
-        "published" => date,
-        "context" => "2hu"
-      },
-      local: false,
-      recipients: to
-    }
-
-    corndog_emojo = ~s(<img height="32px" width="32px" alt="2hu" title="2hu" src="corndog.png" />)
-
-    expected_html =
-      ~s(<p>2hu ) <>
-        corndog_emojo <>
-        ~s(</p>alert\('YAY'\)Some ) <>
-        corndog_emojo <>
-        ~s( content mentioning <a href=") <> mentioned_user.ap_id <> ~s(">@shp</a>)
-
-    expected_status = %{
-      "id" => activity.id,
-      "user" => UserView.render("show.json", %{user: user, for: follower}),
-      "is_local" => false,
-      "statusnet_html" => expected_html,
-      "text" => "2hu :2hu:" <> content,
-      "is_post_verb" => true,
-      "created_at" => "Tue May 24 13:26:08 +0000 2016",
-      "in_reply_to_status_id" => 213_123,
-      "in_reply_to_screen_name" => nil,
-      "in_reply_to_user_id" => nil,
-      "in_reply_to_profileurl" => nil,
-      "in_reply_to_ostatus_uri" => nil,
-      "statusnet_conversation_id" => convo_object.id,
-      "attachments" => [
-        ObjectRepresenter.to_map(object)
-      ],
-      "attentions" => [
-        UserView.render("show.json", %{user: mentioned_user, for: follower})
-      ],
-      "fave_num" => 5,
-      "repeat_num" => 3,
-      "favorited" => false,
-      "repeated" => false,
-      "pinned" => false,
-      "external_url" => "some url",
-      "tags" => ["nsfw", "content", "mentioning"],
-      "activity_type" => "post",
-      "possibly_sensitive" => true,
-      "uri" => activity.data["object"]["id"],
-      "visibility" => "direct",
-      "card" => nil,
-      "muted" => false,
-      "summary" => "2hu :2hu:",
-      "summary_html" =>
-        "2hu <img height=\"32px\" width=\"32px\" alt=\"2hu\" title=\"2hu\" src=\"corndog.png\" />"
-    }
-
-    assert ActivityRepresenter.to_map(activity, %{
-             user: user,
-             for: follower,
-             mentioned: [mentioned_user]
-           }) == expected_status
-  end
-
-  test "a delete activity" do
-    object = insert(:note)
-    user = User.get_by_ap_id(object.data["actor"])
-
-    {:ok, delete} = ActivityPub.delete(object)
-
-    map = ActivityRepresenter.to_map(delete, %{user: user})
-
-    assert map["is_post_verb"] == false
-    assert map["activity_type"] == "delete"
-    assert map["uri"] == object.data["id"]
-  end
-end

test/web/twitter_api/twitter_api_controller_test.exs

@@ -16,9 +16,9 @@ defmodule Pleroma.Web.TwitterAPI.ControllerTest do
   alias Pleroma.Web.ActivityPub.ActivityPub
   alias Pleroma.Web.CommonAPI
   alias Pleroma.Web.OAuth.Token
+  alias Pleroma.Web.TwitterAPI.ActivityView
   alias Pleroma.Web.TwitterAPI.Controller
   alias Pleroma.Web.TwitterAPI.NotificationView
-  alias Pleroma.Web.TwitterAPI.Representers.ActivityRepresenter
   alias Pleroma.Web.TwitterAPI.TwitterAPI
   alias Pleroma.Web.TwitterAPI.UserView
 
@@ -116,7 +116,11 @@ defmodule Pleroma.Web.TwitterAPI.ControllerTest do
         |> post(request_path, %{status: "Nice meme.", visibility: "private"})
 
       assert json_response(conn, 200) ==
-               ActivityRepresenter.to_map(Repo.one(Activity), %{user: user, for: user})
+               ActivityView.render("activity.json", %{
+                 activity: Repo.one(Activity),
+                 user: user,
+                 for: user
+               })
     end
   end
 
@@ -273,7 +277,7 @@ defmodule Pleroma.Web.TwitterAPI.ControllerTest do
 
       response = json_response(conn, 200)
 
-      assert response == ActivityRepresenter.to_map(activity, %{user: actor})
+      assert response == ActivityView.render("activity.json", %{activity: activity, user: actor})
     end
   end
 
@@ -372,7 +376,8 @@ defmodule Pleroma.Web.TwitterAPI.ControllerTest do
 
       assert response ==
                Enum.map(returned_activities, fn activity ->
-                 ActivityRepresenter.to_map(activity, %{
+                 ActivityView.render("activity.json", %{
+                   activity: activity,
                    user: User.get_cached_by_ap_id(activity.data["actor"]),
                    for: current_user
                  })
@@ -469,10 +474,10 @@ defmodule Pleroma.Web.TwitterAPI.ControllerTest do
       assert length(response) == 1
 
       assert Enum.at(response, 0) ==
-               ActivityRepresenter.to_map(activity, %{
+               ActivityView.render("activity.json", %{
                  user: current_user,
                  for: current_user,
-                 mentioned: [current_user]
+                 activity: activity
                })
     end
 
@@ -594,7 +599,9 @@ defmodule Pleroma.Web.TwitterAPI.ControllerTest do
       conn = get(conn, "/api/statuses/user_timeline.json", %{"user_id" => user.id})
       response = json_response(conn, 200)
       assert length(response) == 1
-      assert Enum.at(response, 0) == ActivityRepresenter.to_map(activity, %{user: user})
+
+      assert Enum.at(response, 0) ==
+               ActivityView.render("activity.json", %{user: user, activity: activity})
     end
 
     test "with screen_name", %{conn: conn} do
@@ -604,7 +611,9 @@ defmodule Pleroma.Web.TwitterAPI.ControllerTest do
       conn = get(conn, "/api/statuses/user_timeline.json", %{"screen_name" => user.nickname})
       response = json_response(conn, 200)
       assert length(response) == 1
-      assert Enum.at(response, 0) == ActivityRepresenter.to_map(activity, %{user: user})
+
+      assert Enum.at(response, 0) ==
+               ActivityView.render("activity.json", %{user: user, activity: activity})
     end
 
     test "with credentials", %{conn: conn, user: current_user} do
@@ -620,7 +629,11 @@ defmodule Pleroma.Web.TwitterAPI.ControllerTest do
       assert length(response) == 1
 
       assert Enum.at(response, 0) ==
-               ActivityRepresenter.to_map(activity, %{user: current_user, for: current_user})
+               ActivityView.render("activity.json", %{
+                 user: current_user,
+                 for: current_user,
+                 activity: activity
+               })
     end
 
     test "with credentials with user_id", %{conn: conn, user: current_user} do
@@ -635,7 +648,9 @@ defmodule Pleroma.Web.TwitterAPI.ControllerTest do
       response = json_response(conn, 200)
 
       assert length(response) == 1
-      assert Enum.at(response, 0) == ActivityRepresenter.to_map(activity, %{user: user})
+
+      assert Enum.at(response, 0) ==
+               ActivityView.render("activity.json", %{user: user, activity: activity})
     end
 
     test "with credentials screen_name", %{conn: conn, user: current_user} do
@@ -650,7 +665,9 @@ defmodule Pleroma.Web.TwitterAPI.ControllerTest do
       response = json_response(conn, 200)
 
       assert length(response) == 1
-      assert Enum.at(response, 0) == ActivityRepresenter.to_map(activity, %{user: user})
+
+      assert Enum.at(response, 0) ==
+               ActivityView.render("activity.json", %{user: user, activity: activity})
     end
 
     test "with credentials with user_id, excluding RTs", %{conn: conn, user: current_user} do
@@ -669,7 +686,9 @@ defmodule Pleroma.Web.TwitterAPI.ControllerTest do
       response = json_response(conn, 200)
 
       assert length(response) == 1
-      assert Enum.at(response, 0) == ActivityRepresenter.to_map(activity, %{user: user})
+
+      assert Enum.at(response, 0) ==
+               ActivityView.render("activity.json", %{user: user, activity: activity})
 
       conn =
         conn
@@ -678,7 +697,9 @@ defmodule Pleroma.Web.TwitterAPI.ControllerTest do
       response = json_response(conn, 200)
 
       assert length(response) == 1
-      assert Enum.at(response, 0) == ActivityRepresenter.to_map(activity, %{user: user})
+
+      assert Enum.at(response, 0) ==
+               ActivityView.render("activity.json", %{user: user, activity: activity})
     end
   end
 
@@ -937,7 +958,11 @@ defmodule Pleroma.Web.TwitterAPI.ControllerTest do
       activity_user = Repo.get_by(User, ap_id: note_activity.data["actor"])
 
       assert json_response(response, 200) ==
-               ActivityRepresenter.to_map(activity, %{user: activity_user, for: current_user})
+               ActivityView.render("activity.json", %{
+                 user: activity_user,
+                 for: current_user,
+                 activity: activity
+               })
     end
   end
 
@@ -971,7 +996,11 @@ defmodule Pleroma.Web.TwitterAPI.ControllerTest do
       activity_user = Repo.get_by(User, ap_id: note_activity.data["actor"])
 
       assert json_response(response, 200) ==
-               ActivityRepresenter.to_map(activity, %{user: activity_user, for: current_user})
+               ActivityView.render("activity.json", %{
+                 user: activity_user,
+                 for: current_user,
+                 activity: activity
+               })
     end
   end
 
@@ -1955,7 +1984,7 @@ defmodule Pleroma.Web.TwitterAPI.ControllerTest do
       user = refresh_record(user)
 
       assert json_response(response, 200) ==
-               ActivityRepresenter.to_map(activity, %{user: user, for: user})
+               ActivityView.render("activity.json", %{user: user, for: user, activity: activity})
     end
   end
 
@@ -1985,7 +2014,7 @@ defmodule Pleroma.Web.TwitterAPI.ControllerTest do
       user = refresh_record(user)
 
       assert json_response(response, 200) ==
-               ActivityRepresenter.to_map(activity, %{user: user, for: user})
+               ActivityView.render("activity.json", %{user: user, for: user, activity: activity})
     end
   end
 

test/web/twitter_api/twitter_api_test.exs

@@ -445,22 +445,6 @@ defmodule Pleroma.Web.TwitterAPI.TwitterAPITest do
     :ok
   end
 
-  describe "context_to_conversation_id" do
-    test "creates a mapping object" do
-      conversation_id = TwitterAPI.context_to_conversation_id("random context")
-      object = Object.get_by_ap_id("random context")
-
-      assert conversation_id == object.id
-    end
-
-    test "returns an existing mapping for an existing object" do
-      {:ok, object} = Object.context_mapping("random context") |> Repo.insert()
-      conversation_id = TwitterAPI.context_to_conversation_id("random context")
-
-      assert conversation_id == object.id
-    end
-  end
-
   describe "fetching a user by uri" do
     test "fetches a user by uri" do
       id = "https://mastodon.social/users/lambadalambda"

test/web/twitter_api/util_controller_test.exs

@@ -75,6 +75,29 @@ defmodule Pleroma.Web.TwitterAPI.UtilControllerTest do
   end
 
   describe "GET /api/statusnet/config.json" do
+    test "returns the state of safe_dm_mentions flag", %{conn: conn} do
+      option = Pleroma.Config.get([:instance, :safe_dm_mentions])
+      Pleroma.Config.put([:instance, :safe_dm_mentions], true)
+
+      response =
+        conn
+        |> get("/api/statusnet/config.json")
+        |> json_response(:ok)
+
+      assert response["site"]["safeDMMentionsEnabled"] == "1"
+
+      Pleroma.Config.put([:instance, :safe_dm_mentions], false)
+
+      response =
+        conn
+        |> get("/api/statusnet/config.json")
+        |> json_response(:ok)
+
+      assert response["site"]["safeDMMentionsEnabled"] == "0"
+
+      Pleroma.Config.put([:instance, :safe_dm_mentions], option)
+    end
+
     test "it returns the managed config", %{conn: conn} do
       Pleroma.Config.put([:instance, :managed_config], false)
       Pleroma.Config.put([:fe], theme: "rei-ayanami-towel")

test/web/twitter_api/views/activity_view_test.exs

@@ -12,7 +12,6 @@ defmodule Pleroma.Web.TwitterAPI.ActivityViewTest do
   alias Pleroma.Web.CommonAPI
   alias Pleroma.Web.CommonAPI.Utils
   alias Pleroma.Web.TwitterAPI.ActivityView
-  alias Pleroma.Web.TwitterAPI.TwitterAPI
   alias Pleroma.Web.TwitterAPI.UserView
 
   import Pleroma.Factory
@@ -129,7 +128,7 @@ defmodule Pleroma.Web.TwitterAPI.ActivityViewTest do
 
     result = ActivityView.render("activity.json", activity: activity)
 
-    convo_id = TwitterAPI.context_to_conversation_id(activity.data["object"]["context"])
+    convo_id = Utils.context_to_conversation_id(activity.data["object"]["context"])
 
     expected = %{
       "activity_type" => "post",
@@ -177,12 +176,12 @@ defmodule Pleroma.Web.TwitterAPI.ActivityViewTest do
     other_user = insert(:user, %{nickname: "shp"})
     {:ok, activity} = CommonAPI.post(user, %{"status" => "Hey @shp!"})
 
-    convo_id = TwitterAPI.context_to_conversation_id(activity.data["object"]["context"])
+    convo_id = Utils.context_to_conversation_id(activity.data["object"]["context"])
 
     mocks = [
       {
-        TwitterAPI,
+        Utils,
-        [],
+        [:passthrough],
         [context_to_conversation_id: fn _ -> false end]
       },
       {
@@ -197,7 +196,7 @@ defmodule Pleroma.Web.TwitterAPI.ActivityViewTest do
 
       assert result["statusnet_conversation_id"] == convo_id
       assert result["user"]
-      refute called(TwitterAPI.context_to_conversation_id(:_))
+      refute called(Utils.context_to_conversation_id(:_))
       refute called(User.get_cached_by_ap_id(user.ap_id))
       refute called(User.get_cached_by_ap_id(other_user.ap_id))
     end
@@ -280,7 +279,7 @@ defmodule Pleroma.Web.TwitterAPI.ActivityViewTest do
     {:ok, activity} = CommonAPI.post(user, %{"status" => "Hey @shp!"})
     {:ok, announce, _object} = CommonAPI.repeat(activity.id, other_user)
 
-    convo_id = TwitterAPI.context_to_conversation_id(activity.data["object"]["context"])
+    convo_id = Utils.context_to_conversation_id(activity.data["object"]["context"])
 
     activity = Repo.get(Activity, activity.id)