Merge remote-tracking branch 'pleroma/develop' into remove-user-activities

lib/mix/tasks/pleroma/relay.ex

@@ -4,8 +4,8 @@
 
 defmodule Mix.Tasks.Pleroma.Relay do
   use Mix.Task
-  alias Pleroma.Web.ActivityPub.Relay
   alias Mix.Tasks.Pleroma.Common
+  alias Pleroma.Web.ActivityPub.Relay
 
   @shortdoc "Manages remote relays"
   @moduledoc """

lib/mix/tasks/pleroma/robotstxt.ex

@@ -0,0 +1,32 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Mix.Tasks.Pleroma.RobotsTxt do
+  use Mix.Task
+
+  @shortdoc "Generate robots.txt"
+  @moduledoc """
+  Generates robots.txt
+
+  ## Overwrite robots.txt to disallow all
+
+      mix pleroma.robots_txt disallow_all
+
+  This will write a robots.txt that will hide all paths on your instance
+  from search engines and other robots that obey robots.txt
+
+  """
+  def run(["disallow_all"]) do
+    static_dir = Pleroma.Config.get([:instance, :static_dir], "instance/static/")
+
+    if !File.exists?(static_dir) do
+      File.mkdir_p!(static_dir)
+    end
+
+    robots_txt_path = Path.join(static_dir, "robots.txt")
+    robots_txt_content = "User-Agent: *\nDisallow: /\n"
+
+    File.write!(robots_txt_path, robots_txt_content, [:write])
+  end
+end

lib/mix/tasks/pleroma/uploads.ex

@@ -4,9 +4,9 @@
 
 defmodule Mix.Tasks.Pleroma.Uploads do
   use Mix.Task
+  alias Mix.Tasks.Pleroma.Common
   alias Pleroma.Upload
   alias Pleroma.Uploaders.Local
-  alias Mix.Tasks.Pleroma.Common
   require Logger
 
   @log_every 50
@@ -20,7 +20,6 @@ defmodule Mix.Tasks.Pleroma.Uploads do
    Options:
    - `--delete` - delete local uploads after migrating them to the target uploader
 
-
    A list of available uploaders can be seen in config.exs
   """
   def run(["migrate_local", target_uploader | args]) do

lib/mix/tasks/pleroma/user.ex

@@ -5,9 +5,9 @@
 defmodule Mix.Tasks.Pleroma.User do
   use Mix.Task
   import Ecto.Changeset
+  alias Mix.Tasks.Pleroma.Common
   alias Pleroma.Repo
   alias Pleroma.User
-  alias Mix.Tasks.Pleroma.Common
 
   @shortdoc "Manages Pleroma users"
   @moduledoc """

lib/pleroma/PasswordResetToken.ex

@@ -7,9 +7,9 @@ defmodule Pleroma.PasswordResetToken do
 
   import Ecto.Changeset
 
-  alias Pleroma.User
-  alias Pleroma.Repo
   alias Pleroma.PasswordResetToken
+  alias Pleroma.Repo
+  alias Pleroma.User
 
   schema "password_reset_tokens" do
     belongs_to(:user, User, type: Pleroma.FlakeId)

lib/pleroma/activity.ex

@@ -5,9 +5,10 @@
 defmodule Pleroma.Activity do
   use Ecto.Schema
 
-  alias Pleroma.Repo
   alias Pleroma.Activity
   alias Pleroma.Notification
+  alias Pleroma.Object
+  alias Pleroma.Repo
 
   import Ecto.Query
 
@@ -22,6 +23,10 @@ defmodule Pleroma.Activity do
     "Like" => "favourite"
   }
 
+  @mastodon_to_ap_notification_types for {k, v} <- @mastodon_notification_types,
+                                         into: %{},
+                                         do: {v, k}
+
   schema "activities" do
     field(:data, :map)
     field(:local, :boolean, default: true)
@@ -29,9 +34,42 @@ defmodule Pleroma.Activity do
     field(:recipients, {:array, :string})
     has_many(:notifications, Notification, on_delete: :delete_all)
 
+    # Attention: this is a fake relation, don't try to preload it blindly and expect it to work!
+    # The foreign key is embedded in a jsonb field.
+    #
+    # To use it, you probably want to do an inner join and a preload:
+    #
+    # ```
+    # |> join(:inner, [activity], o in Object,
+    #      on: fragment("(?->>'id') = COALESCE((?)->'object'->> 'id', (?)->>'object')",
+    #        o.data, activity.data, activity.data))
+    # |> preload([activity, object], [object: object])
+    # ```
+    #
+    # As a convenience, Activity.with_preloaded_object() sets up an inner join and preload for the
+    # typical case.
+    has_one(:object, Object, on_delete: :nothing, foreign_key: :id)
+
     timestamps()
   end
 
+  def with_preloaded_object(query) do
+    query
+    |> join(
+      :inner,
+      [activity],
+      o in Object,
+      on:
+        fragment(
+          "(?->>'id') = COALESCE(?->'object'->>'id', ?->>'object')",
+          o.data,
+          activity.data,
+          activity.data
+        )
+    )
+    |> preload([activity, object], object: object)
+  end
+
   def get_by_ap_id(ap_id) do
     Repo.one(
       from(
@@ -41,10 +79,44 @@ defmodule Pleroma.Activity do
     )
   end
 
+  def get_by_ap_id_with_object(ap_id) do
+    Repo.one(
+      from(
+        activity in Activity,
+        where: fragment("(?)->>'id' = ?", activity.data, ^to_string(ap_id)),
+        left_join: o in Object,
+        on:
+          fragment(
+            "(?->>'id') = COALESCE(?->'object'->>'id', ?->>'object')",
+            o.data,
+            activity.data,
+            activity.data
+          ),
+        preload: [object: o]
+      )
+    )
+  end
+
   def get_by_id(id) do
     Repo.get(Activity, id)
   end
 
+  def get_by_id_with_object(id) do
+    from(activity in Activity,
+      where: activity.id == ^id,
+      inner_join: o in Object,
+      on:
+        fragment(
+          "(?->>'id') = COALESCE(?->'object'->>'id', ?->>'object')",
+          o.data,
+          activity.data,
+          activity.data
+        ),
+      preload: [object: o]
+    )
+    |> Repo.one()
+  end
+
   def by_object_ap_id(ap_id) do
     from(
       activity in Activity,
@@ -72,7 +144,7 @@ defmodule Pleroma.Activity do
     )
   end
 
-  def create_by_object_ap_id(ap_id) do
+  def create_by_object_ap_id(ap_id) when is_binary(ap_id) do
     from(
       activity in Activity,
       where:
@@ -86,6 +158,8 @@ defmodule Pleroma.Activity do
     )
   end
 
+  def create_by_object_ap_id(_), do: nil
+
   def get_all_create_by_object_ap_id(ap_id) do
     Repo.all(create_by_object_ap_id(ap_id))
   end
@@ -97,8 +171,39 @@ defmodule Pleroma.Activity do
 
   def get_create_by_object_ap_id(_), do: nil
 
-  def normalize(obj) when is_map(obj), do: Activity.get_by_ap_id(obj["id"])
-  def normalize(ap_id) when is_binary(ap_id), do: Activity.get_by_ap_id(ap_id)
+  def create_by_object_ap_id_with_object(ap_id) when is_binary(ap_id) do
+    from(
+      activity in Activity,
+      where:
+        fragment(
+          "coalesce((?)->'object'->>'id', (?)->>'object') = ?",
+          activity.data,
+          activity.data,
+          ^to_string(ap_id)
+        ),
+      where: fragment("(?)->>'type' = 'Create'", activity.data),
+      inner_join: o in Object,
+      on:
+        fragment(
+          "(?->>'id') = COALESCE(?->'object'->>'id', ?->>'object')",
+          o.data,
+          activity.data,
+          activity.data
+        ),
+      preload: [object: o]
+    )
+  end
+
+  def create_by_object_ap_id_with_object(_), do: nil
+
+  def get_create_by_object_ap_id_with_object(ap_id) do
+    ap_id
+    |> create_by_object_ap_id_with_object()
+    |> Repo.one()
+  end
+
+  def normalize(obj) when is_map(obj), do: get_by_ap_id_with_object(obj["id"])
+  def normalize(ap_id) when is_binary(ap_id), do: get_by_ap_id_with_object(ap_id)
   def normalize(_), do: nil
 
   def get_in_reply_to_activity(%Activity{data: %{"object" => %{"inReplyTo" => ap_id}}}) do
@@ -107,6 +212,19 @@ defmodule Pleroma.Activity do
 
   def get_in_reply_to_activity(_), do: nil
 
+  def delete_by_ap_id(id) when is_binary(id) do
+    by_object_ap_id(id)
+    |> select([u], u)
+    |> Repo.delete_all()
+    |> elem(1)
+    |> Enum.find(fn
+      %{data: %{"type" => "Create", "object" => %{"id" => ap_id}}} -> ap_id == id
+      _ -> nil
+    end)
+  end
+
+  def delete_by_ap_id(_), do: nil
+
   for {ap_type, type} <- @mastodon_notification_types do
     def mastodon_notification_type(%Activity{data: %{"type" => unquote(ap_type)}}),
       do: unquote(type)
@@ -114,6 +232,10 @@ defmodule Pleroma.Activity do
 
   def mastodon_notification_type(%Activity{}), do: nil
 
+  def from_mastodon_notification_type(type) do
+    Map.get(@mastodon_to_ap_notification_types, type)
+  end
+
   def all_by_actor_and_id(actor, status_ids \\ [])
   def all_by_actor_and_id(_actor, []), do: []
 
@@ -123,4 +245,50 @@ defmodule Pleroma.Activity do
     |> where([s], s.actor == ^actor)
     |> Repo.all()
   end
+
+  def increase_replies_count(id) do
+    Activity
+    |> where(id: ^id)
+    |> update([a],
+      set: [
+        data:
+          fragment(
+            """
+            jsonb_set(?, '{object, repliesCount}',
+              (coalesce((?->'object'->>'repliesCount')::int, 0) + 1)::varchar::jsonb, true)
+            """,
+            a.data,
+            a.data
+          )
+      ]
+    )
+    |> Repo.update_all([])
+    |> case do
+      {1, [activity]} -> activity
+      _ -> {:error, "Not found"}
+    end
+  end
+
+  def decrease_replies_count(id) do
+    Activity
+    |> where(id: ^id)
+    |> update([a],
+      set: [
+        data:
+          fragment(
+            """
+            jsonb_set(?, '{object, repliesCount}',
+              (greatest(0, (?->'object'->>'repliesCount')::int - 1))::varchar::jsonb, true)
+            """,
+            a.data,
+            a.data
+          )
+      ]
+    )
+    |> Repo.update_all([])
+    |> case do
+      {1, [activity]} -> activity
+      _ -> {:error, "Not found"}
+    end
+  end
 end

lib/pleroma/application.ex

@@ -11,10 +11,10 @@ defmodule Pleroma.Application do
   @repository Mix.Project.config()[:source_url]
   def name, do: @name
   def version, do: @version
-  def named_version(), do: @name <> " " <> @version
+  def named_version, do: @name <> " " <> @version
   def repository, do: @repository
 
-  def user_agent() do
+  def user_agent do
     info = "#{Pleroma.Web.base_url()} <#{Pleroma.Config.get([:instance, :email], "")}>"
     named_version() <> "; " <> info
   end
@@ -48,7 +48,7 @@ defmodule Pleroma.Application do
           [
             :user_cache,
             [
-              default_ttl: 25000,
+              default_ttl: 25_000,
               ttl_interval: 1000,
               limit: 2500
             ]
@@ -60,7 +60,7 @@ defmodule Pleroma.Application do
           [
             :object_cache,
             [
-              default_ttl: 25000,
+              default_ttl: 25_000,
               ttl_interval: 1000,
               limit: 2500
             ]
@@ -110,7 +110,6 @@ defmodule Pleroma.Application do
           worker(Pleroma.Web.Federator.RetryQueue, []),
           worker(Pleroma.Stats, []),
           worker(Pleroma.Web.Push, []),
-          worker(Pleroma.Jobs, []),
           worker(Task, [&Pleroma.Web.Federator.init/0], restart: :temporary)
         ] ++
         streamer_child() ++
@@ -127,7 +126,7 @@ defmodule Pleroma.Application do
     Supervisor.start_link(children, opts)
   end
 
-  def enabled_hackney_pools() do
+  def enabled_hackney_pools do
     [:media] ++
       if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Hackney do
         [:federation]
@@ -142,14 +141,14 @@ defmodule Pleroma.Application do
   end
 
   if Mix.env() == :test do
-    defp streamer_child(), do: []
-    defp chat_child(), do: []
+    defp streamer_child, do: []
+    defp chat_child, do: []
   else
-    defp streamer_child() do
+    defp streamer_child do
       [worker(Pleroma.Web.Streamer, [])]
     end
 
-    defp chat_child() do
+    defp chat_child do
       if Pleroma.Config.get([:chat, :enabled]) do
         [worker(Pleroma.Web.ChatChannel.ChatChannelState, [])]
       else
@@ -158,7 +157,7 @@ defmodule Pleroma.Application do
     end
   end
 
-  defp hackney_pool_children() do
+  defp hackney_pool_children do
     for pool <- enabled_hackney_pools() do
       options = Pleroma.Config.get([:hackney_pools, pool])
       :hackney_pool.child_spec(pool, options)

lib/pleroma/captcha/captcha.ex

@@ -10,7 +10,7 @@ defmodule Pleroma.Captcha do
   use GenServer
 
   @doc false
-  def start_link() do
+  def start_link do
     GenServer.start_link(__MODULE__, [], name: __MODULE__)
   end
 
@@ -22,7 +22,7 @@ defmodule Pleroma.Captcha do
   @doc """
   Ask the configured captcha service for a new captcha
   """
-  def new() do
+  def new do
     GenServer.call(__MODULE__, :new)
   end
 
@@ -73,7 +73,7 @@ defmodule Pleroma.Captcha do
     secret = KeyGenerator.generate(secret_key_base, token <> "_encrypt")
     sign_secret = KeyGenerator.generate(secret_key_base, token <> "_sign")
 
-    # If the time found is less than (current_time - seconds_valid), then the time has already passed.
+    # If the time found is less than (current_time-seconds_valid) then the time has already passed
     # Later we check that the time found is more than the presumed invalidatation time, that means
     # that the data is still valid and the captcha can be checked
     seconds_valid = Pleroma.Config.get!([Pleroma.Captcha, :seconds_valid])

lib/pleroma/captcha/kocaptcha.ex

@@ -7,7 +7,7 @@ defmodule Pleroma.Captcha.Kocaptcha do
   @behaviour Service
 
   @impl Service
-  def new() do
+  def new do
     endpoint = Pleroma.Config.get!([__MODULE__, :endpoint])
 
     case Tesla.get(endpoint <> "/new") do

lib/pleroma/clippy.ex

@@ -7,13 +7,13 @@ defmodule Pleroma.Clippy do
   # No software is complete until they have a Clippy implementation.
   # A ballmer peak _may_ be required to change this module.
 
-  def tip() do
+  def tip do
     tips()
     |> Enum.random()
     |> puts()
   end
 
-  def tips() do
+  def tips do
     host = Pleroma.Config.get([Pleroma.Web.Endpoint, :url, :host])
 
     [
@@ -92,8 +92,8 @@ defmodule Pleroma.Clippy do
 
     # surrond one/five line clippy with blank lines around to not fuck up the layout
     #
-    # yes this fix sucks but it's good enough, have you ever seen a release of windows wihtout some butched
-    # features anyway?
+    # yes this fix sucks but it's good enough, have you ever seen a release of windows
+    # without some butched features anyway?
     lines =
       if length(lines) == 1 or length(lines) == 5 do
         [""] ++ lines ++ [""]

lib/pleroma/config/deprecation_warnings.ex

@@ -5,7 +5,7 @@
 defmodule Pleroma.Config.DeprecationWarnings do
   require Logger
 
-  def check_frontend_config_mechanism() do
+  def check_frontend_config_mechanism do
     if Pleroma.Config.get(:fe) do
       Logger.warn("""
       !!!DEPRECATION WARNING!!!

lib/pleroma/emails/admin_email.ex

@@ -29,9 +29,13 @@ defmodule Pleroma.AdminEmail do
       if length(statuses) > 0 do
         statuses_list_html =
           statuses
-          |> Enum.map(fn %{id: id} ->
-            status_url = Helpers.o_status_url(Pleroma.Web.Endpoint, :notice, id)
-            "<li><a href=\"#{status_url}\">#{status_url}</li>"
+          |> Enum.map(fn
+            %{id: id} ->
+              status_url = Helpers.o_status_url(Pleroma.Web.Endpoint, :notice, id)
+              "<li><a href=\"#{status_url}\">#{status_url}</li>"
+
+            id when is_binary(id) ->
+              "<li><a href=\"#{id}\">#{id}</li>"
           end)
           |> Enum.join("\n")
 

lib/pleroma/emails/mailer.ex

@@ -6,7 +6,7 @@ defmodule Pleroma.Mailer do
   use Swoosh.Mailer, otp_app: :pleroma
 
   def deliver_async(email, config \\ []) do
-    Pleroma.Jobs.enqueue(:mailer, __MODULE__, [:deliver_async, email, config])
+    PleromaJobQueue.enqueue(:mailer, __MODULE__, [:deliver_async, email, config])
   end
 
   def perform(:deliver_async, email, config), do: deliver(email, config)

lib/pleroma/emoji.ex

@@ -17,13 +17,13 @@ defmodule Pleroma.Emoji do
   @ets_options [:ordered_set, :protected, :named_table, {:read_concurrency, true}]
 
   @doc false
-  def start_link() do
+  def start_link do
     GenServer.start_link(__MODULE__, [], name: __MODULE__)
   end
 
   @doc "Reloads the emojis from disk."
   @spec reload() :: :ok
-  def reload() do
+  def reload do
     GenServer.call(__MODULE__, :reload)
   end
 
@@ -38,7 +38,7 @@ defmodule Pleroma.Emoji do
 
   @doc "Returns all the emojos!!"
   @spec get_all() :: [{String.t(), String.t()}, ...]
-  def get_all() do
+  def get_all do
     :ets.tab2list(@ets)
   end
 
@@ -72,7 +72,7 @@ defmodule Pleroma.Emoji do
     {:ok, state}
   end
 
-  defp load() do
+  defp load do
     emojis =
       (load_finmoji(Keyword.get(Application.get_env(:pleroma, :instance), :finmoji_enabled)) ++
          load_from_file("config/emoji.txt") ++

lib/pleroma/filter.ex

@@ -8,8 +8,8 @@ defmodule Pleroma.Filter do
   import Ecto.Changeset
   import Ecto.Query
 
-  alias Pleroma.User
   alias Pleroma.Repo
+  alias Pleroma.User
 
   schema "filters" do
     belongs_to(:user, User, type: Pleroma.FlakeId)

lib/pleroma/flake_id.ex

@@ -85,7 +85,7 @@ defmodule Pleroma.FlakeId do
     {:ok, FlakeId.from_string(value)}
   end
 
-  def autogenerate(), do: get()
+  def autogenerate, do: get()
 
   # -- GenServer API
   def start_link do
@@ -165,7 +165,7 @@ defmodule Pleroma.FlakeId do
     1_000_000_000 * mega_seconds + seconds * 1000 + :erlang.trunc(micro_seconds / 1000)
   end
 
-  defp worker_id() do
+  defp worker_id do
     <<worker::integer-size(48)>> = :crypto.strong_rand_bytes(6)
     worker
   end

lib/pleroma/formatter.ex

@@ -8,8 +8,10 @@ defmodule Pleroma.Formatter do
   alias Pleroma.User
   alias Pleroma.Web.MediaProxy
 
+  @safe_mention_regex ~r/^(\s*(?<mentions>@.+?\s+)+)(?<rest>.*)/
   @markdown_characters_regex ~r/(`|\*|_|{|}|[|]|\(|\)|#|\+|-|\.|!)/
   @link_regex ~r{((?:http(s)?:\/\/)?[\w.-]+(?:\.[\w\.-]+)+[\w\-\._~%:/?#[\]@!\$&'\(\)\*\+,;=.]+)|[0-9a-z+\-\.]+:[0-9a-z$-_.+!*'(),]+}ui
+  # credo:disable-for-previous-line Credo.Check.Readability.MaxLineLength
 
   @auto_linker_config hashtag: true,
                       hashtag_handler: &Pleroma.Formatter.hashtag_handler/4,
@@ -44,15 +46,28 @@ defmodule Pleroma.Formatter do
 
   @doc """
   Parses a text and replace plain text links with HTML. Returns a tuple with a result text, mentions, and hashtags.
+
+  If the 'safe_mention' option is given, only consecutive mentions at the start the post are actually mentioned.
   """
   @spec linkify(String.t(), keyword()) ::
           {String.t(), [{String.t(), User.t()}], [{String.t(), String.t()}]}
   def linkify(text, options \\ []) do
     options = options ++ @auto_linker_config
-    acc = %{mentions: MapSet.new(), tags: MapSet.new()}
-    {text, %{mentions: mentions, tags: tags}} = AutoLinker.link_map(text, acc, options)
 
-    {text, MapSet.to_list(mentions), MapSet.to_list(tags)}
+    if options[:safe_mention] && Regex.named_captures(@safe_mention_regex, text) do
+      %{"mentions" => mentions, "rest" => rest} = Regex.named_captures(@safe_mention_regex, text)
+      acc = %{mentions: MapSet.new(), tags: MapSet.new()}
+
+      {text_mentions, %{mentions: mentions}} = AutoLinker.link_map(mentions, acc, options)
+      {text_rest, %{tags: tags}} = AutoLinker.link_map(rest, acc, options)
+
+      {text_mentions <> text_rest, MapSet.to_list(mentions), MapSet.to_list(tags)}
+    else
+      acc = %{mentions: MapSet.new(), tags: MapSet.new()}
+      {text, %{mentions: mentions, tags: tags}} = AutoLinker.link_map(text, acc, options)
+
+      {text, MapSet.to_list(mentions), MapSet.to_list(tags)}
+    end
   end
 
   def emojify(text) do

lib/pleroma/gopher/server.ex

@@ -6,7 +6,7 @@ defmodule Pleroma.Gopher.Server do
   use GenServer
   require Logger
 
-  def start_link() do
+  def start_link do
     config = Pleroma.Config.get(:gopher, [])
     ip = Keyword.get(config, :ip, {0, 0, 0, 0})
     port = Keyword.get(config, :port, 1234)
@@ -36,12 +36,12 @@ defmodule Pleroma.Gopher.Server do
 end
 
 defmodule Pleroma.Gopher.Server.ProtocolHandler do
-  alias Pleroma.Web.ActivityPub.ActivityPub
-  alias Pleroma.Web.ActivityPub.Visibility
   alias Pleroma.Activity
   alias Pleroma.HTML
-  alias Pleroma.User
   alias Pleroma.Repo
+  alias Pleroma.User
+  alias Pleroma.Web.ActivityPub.ActivityPub
+  alias Pleroma.Web.ActivityPub.Visibility
 
   def start_link(ref, socket, transport, opts) do
     pid = spawn_link(__MODULE__, :init, [ref, socket, transport, opts])
@@ -66,7 +66,8 @@ defmodule Pleroma.Gopher.Server.ProtocolHandler do
   def link(name, selector, type \\ 1) do
     address = Pleroma.Web.Endpoint.host()
     port = Pleroma.Config.get([:gopher, :port], 1234)
-    "#{type}#{name}\t#{selector}\t#{address}\t#{port}\r\n"
+    dstport = Pleroma.Config.get([:gopher, :dstport], port)
+    "#{type}#{name}\t#{selector}\t#{address}\t#{dstport}\r\n"
   end
 
   def render_activities(activities) do

lib/pleroma/html.ex

@@ -9,7 +9,7 @@ defmodule Pleroma.HTML do
   defp get_scrubbers(scrubbers) when is_list(scrubbers), do: scrubbers
   defp get_scrubbers(_), do: [Pleroma.HTML.Scrubber.Default]
 
-  def get_scrubbers() do
+  def get_scrubbers do
     Pleroma.Config.get([:markup, :scrub_policy])
     |> get_scrubbers
   end
@@ -95,6 +95,13 @@ defmodule Pleroma.HTML.Scrubber.TwitterText do
   Meta.allow_tag_with_uri_attributes("a", ["href", "data-user", "data-tag"], @valid_schemes)
   Meta.allow_tag_with_these_attributes("a", ["name", "title", "class"])
 
+  Meta.allow_tag_with_this_attribute_values("a", "rel", [
+    "tag",
+    "nofollow",
+    "noopener",
+    "noreferrer"
+  ])
+
   # paragraphs and linebreaks
   Meta.allow_tag_with_these_attributes("br", [])
   Meta.allow_tag_with_these_attributes("p", [])
@@ -137,6 +144,13 @@ defmodule Pleroma.HTML.Scrubber.Default do
   Meta.allow_tag_with_uri_attributes("a", ["href", "data-user", "data-tag"], @valid_schemes)
   Meta.allow_tag_with_these_attributes("a", ["name", "title", "class"])
 
+  Meta.allow_tag_with_this_attribute_values("a", "rel", [
+    "tag",
+    "nofollow",
+    "noopener",
+    "noreferrer"
+  ])
+
   Meta.allow_tag_with_these_attributes("abbr", ["title"])
 
   Meta.allow_tag_with_these_attributes("b", [])

lib/pleroma/http/connection.ex

@@ -8,8 +8,8 @@ defmodule Pleroma.HTTP.Connection do
   """
 
   @hackney_options [
-    timeout: 10000,
-    recv_timeout: 20000,
+    connect_timeout: 2_000,
+    recv_timeout: 20_000,
     follow_redirect: true,
     pool: :federation
   ]
@@ -31,6 +31,10 @@ defmodule Pleroma.HTTP.Connection do
   #
   defp hackney_options(opts) do
     options = Keyword.get(opts, :adapter, [])
-    @hackney_options ++ options
+    adapter_options = Pleroma.Config.get([:http, :adapter], [])
+
+    @hackney_options
+    |> Keyword.merge(adapter_options)
+    |> Keyword.merge(options)
   end
 end

lib/pleroma/http/http.ex

@@ -27,21 +27,29 @@ defmodule Pleroma.HTTP do
 
   """
   def request(method, url, body \\ "", headers \\ [], options \\ []) do
-    options =
-      process_request_options(options)
-      |> process_sni_options(url)
+    try do
+      options =
+        process_request_options(options)
+        |> process_sni_options(url)
 
-    params = Keyword.get(options, :params, [])
+      params = Keyword.get(options, :params, [])
 
-    %{}
-    |> Builder.method(method)
-    |> Builder.headers(headers)
-    |> Builder.opts(options)
-    |> Builder.url(url)
-    |> Builder.add_param(:body, :body, body)
-    |> Builder.add_param(:query, :query, params)
-    |> Enum.into([])
-    |> (&Tesla.request(Connection.new(), &1)).()
+      %{}
+      |> Builder.method(method)
+      |> Builder.headers(headers)
+      |> Builder.opts(options)
+      |> Builder.url(url)
+      |> Builder.add_param(:body, :body, body)
+      |> Builder.add_param(:query, :query, params)
+      |> Enum.into([])
+      |> (&Tesla.request(Connection.new(options), &1)).()
+    rescue
+      e ->
+        {:error, e}
+    catch
+      :exit, e ->
+        {:error, e}
+    end
   end
 
   defp process_sni_options(options, nil), do: options

lib/pleroma/instances/instance.ex

@@ -2,8 +2,8 @@ defmodule Pleroma.Instances.Instance do
   @moduledoc "Instance."
 
   alias Pleroma.Instances
-  alias Pleroma.Repo
   alias Pleroma.Instances.Instance
+  alias Pleroma.Repo
 
   use Ecto.Schema
 
@@ -12,7 +12,7 @@ defmodule Pleroma.Instances.Instance do
 
   schema "instances" do
     field(:host, :string)
-    field(:unreachable_since, :naive_datetime)
+    field(:unreachable_since, :naive_datetime_usec)
 
     timestamps()
   end

lib/pleroma/jobs.ex

@@ -1,152 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Pleroma.Jobs do
-  @moduledoc """
-  A basic job queue
-  """
-  use GenServer
-
-  require Logger
-
-  def init(args) do
-    {:ok, args}
-  end
-
-  def start_link do
-    queues =
-      Pleroma.Config.get(Pleroma.Jobs)
-      |> Enum.map(fn {name, _} -> create_queue(name) end)
-      |> Enum.into(%{})
-
-    state = %{
-      queues: queues,
-      refs: %{}
-    }
-
-    GenServer.start_link(__MODULE__, state, name: __MODULE__)
-  end
-
-  def create_queue(name) do
-    {name, {:sets.new(), []}}
-  end
-
-  @doc """
-  Enqueues a job.
-
-  Returns `:ok`.
-
-  ## Arguments
-
-  - `queue_name` - a queue name(must be specified in the config).
-  - `mod` - a worker module (must have `perform` function).
-  - `args` - a list of arguments for the `perform` function of the worker module.
-  - `priority` - a job priority (`0` by default).
-
-  ## Examples
-
-  Enqueue `Module.perform/0` with `priority=1`:
-
-      iex> Pleroma.Jobs.enqueue(:example_queue, Module, [])
-      :ok
-
-  Enqueue `Module.perform(:job_name)` with `priority=5`:
-
-      iex> Pleroma.Jobs.enqueue(:example_queue, Module, [:job_name], 5)
-      :ok
-
-  Enqueue `Module.perform(:another_job, data)` with `priority=1`:
-
-      iex> data = "foobar"
-      iex> Pleroma.Jobs.enqueue(:example_queue, Module, [:another_job, data])
-      :ok
-
-  Enqueue `Module.perform(:foobar_job, :foo, :bar, 42)` with `priority=1`:
-
-      iex> Pleroma.Jobs.enqueue(:example_queue, Module, [:foobar_job, :foo, :bar, 42])
-      :ok
-
-  """
-
-  def enqueue(queue_name, mod, args, priority \\ 1)
-
-  if Mix.env() == :test do
-    def enqueue(_queue_name, mod, args, _priority) do
-      apply(mod, :perform, args)
-    end
-  else
-    @spec enqueue(atom(), atom(), [any()], integer()) :: :ok
-    def enqueue(queue_name, mod, args, priority) do
-      GenServer.cast(__MODULE__, {:enqueue, queue_name, mod, args, priority})
-    end
-  end
-
-  def handle_cast({:enqueue, queue_name, mod, args, priority}, state) do
-    {running_jobs, queue} = state[:queues][queue_name]
-
-    queue = enqueue_sorted(queue, {mod, args}, priority)
-
-    state =
-      state
-      |> update_queue(queue_name, {running_jobs, queue})
-      |> maybe_start_job(queue_name, running_jobs, queue)
-
-    {:noreply, state}
-  end
-
-  def handle_info({:DOWN, ref, :process, _pid, _reason}, state) do
-    queue_name = state.refs[ref]
-
-    {running_jobs, queue} = state[:queues][queue_name]
-
-    running_jobs = :sets.del_element(ref, running_jobs)
-
-    state =
-      state
-      |> remove_ref(ref)
-      |> update_queue(queue_name, {running_jobs, queue})
-      |> maybe_start_job(queue_name, running_jobs, queue)
-
-    {:noreply, state}
-  end
-
-  def maybe_start_job(state, queue_name, running_jobs, queue) do
-    if :sets.size(running_jobs) < Pleroma.Config.get([__MODULE__, queue_name, :max_jobs]) &&
-         queue != [] do
-      {{mod, args}, queue} = queue_pop(queue)
-      {:ok, pid} = Task.start(fn -> apply(mod, :perform, args) end)
-      mref = Process.monitor(pid)
-
-      state
-      |> add_ref(queue_name, mref)
-      |> update_queue(queue_name, {:sets.add_element(mref, running_jobs), queue})
-    else
-      state
-    end
-  end
-
-  def enqueue_sorted(queue, element, priority) do
-    [%{item: element, priority: priority} | queue]
-    |> Enum.sort_by(fn %{priority: priority} -> priority end)
-  end
-
-  def queue_pop([%{item: element} | queue]) do
-    {element, queue}
-  end
-
-  defp add_ref(state, queue_name, ref) do
-    refs = Map.put(state[:refs], ref, queue_name)
-    Map.put(state, :refs, refs)
-  end
-
-  defp remove_ref(state, ref) do
-    refs = Map.delete(state[:refs], ref)
-    Map.put(state, :refs, refs)
-  end
-
-  defp update_queue(state, queue_name, data) do
-    queues = Map.put(state[:queues], queue_name, data)
-    Map.put(state, :queues, queues)
-  end
-end

lib/pleroma/notification.ex

@@ -5,14 +5,17 @@
 defmodule Pleroma.Notification do
   use Ecto.Schema
 
-  alias Pleroma.User
   alias Pleroma.Activity
   alias Pleroma.Notification
+  alias Pleroma.Object
+  alias Pleroma.Pagination
   alias Pleroma.Repo
-  alias Pleroma.Web.CommonAPI.Utils
+  alias Pleroma.User
   alias Pleroma.Web.CommonAPI
+  alias Pleroma.Web.CommonAPI.Utils
 
   import Ecto.Query
+  import Ecto.Changeset
 
   schema "notifications" do
     field(:seen, :boolean, default: false)
@@ -22,36 +25,30 @@ defmodule Pleroma.Notification do
     timestamps()
   end
 
-  # TODO: Make generic and unify (see activity_pub.ex)
-  defp restrict_max(query, %{"max_id" => max_id}) do
-    from(activity in query, where: activity.id < ^max_id)
+  def changeset(%Notification{} = notification, attrs) do
+    notification
+    |> cast(attrs, [:seen])
   end
 
-  defp restrict_max(query, _), do: query
-
-  defp restrict_since(query, %{"since_id" => since_id}) do
-    from(activity in query, where: activity.id > ^since_id)
+  def for_user_query(user) do
+    Notification
+    |> where(user_id: ^user.id)
+    |> join(:inner, [n], activity in assoc(n, :activity))
+    |> join(:left, [n, a], object in Object,
+      on:
+        fragment(
+          "(?->>'id') = COALESCE((? -> 'object'::text) ->> 'id'::text)",
+          object.data,
+          a.data
+        )
+    )
+    |> preload([n, a, o], activity: {a, object: o})
   end
 
-  defp restrict_since(query, _), do: query
-
   def for_user(user, opts \\ %{}) do
-    query =
-      from(
-        n in Notification,
-        where: n.user_id == ^user.id,
-        order_by: [desc: n.id],
-        join: activity in assoc(n, :activity),
-        preload: [activity: activity],
-        limit: 20
-      )
-
-    query =
-      query
-      |> restrict_since(opts)
-      |> restrict_max(opts)
-
-    Repo.all(query)
+    user
+    |> for_user_query()
+    |> Pagination.fetch_paginated(opts)
   end
 
   def set_read_up_to(%{id: user_id} = _user, id) do
@@ -68,6 +65,14 @@ defmodule Pleroma.Notification do
     Repo.update_all(query, [])
   end
 
+  def read_one(%User{} = user, notification_id) do
+    with {:ok, %Notification{} = notification} <- get(user, notification_id) do
+      notification
+      |> changeset(%{seen: true})
+      |> Repo.update()
+    end
+  end
+
   def get(%{id: user_id} = _user, id) do
     query =
       from(

lib/pleroma/object.ex

@@ -5,15 +5,17 @@
 defmodule Pleroma.Object do
   use Ecto.Schema
 
-  alias Pleroma.Repo
-  alias Pleroma.Object
-  alias Pleroma.User
   alias Pleroma.Activity
+  alias Pleroma.Object
   alias Pleroma.ObjectTombstone
+  alias Pleroma.Repo
+  alias Pleroma.User
 
   import Ecto.Query
   import Ecto.Changeset
 
+  require Logger
+
   schema "objects" do
     field(:data, :map)
 
@@ -38,6 +40,33 @@ defmodule Pleroma.Object do
     Repo.one(from(object in Object, where: fragment("(?)->>'id' = ?", object.data, ^ap_id)))
   end
 
+  # If we pass an Activity to Object.normalize(), we can try to use the preloaded object.
+  # Use this whenever possible, especially when walking graphs in an O(N) loop!
+  def normalize(%Activity{object: %Object{} = object}), do: object
+
+  # Catch and log Object.normalize() calls where the Activity's child object is not
+  # preloaded.
+  def normalize(%Activity{data: %{"object" => %{"id" => ap_id}}}) do
+    Logger.debug(
+      "Object.normalize() called without preloaded object (#{ap_id}).  Consider preloading the object!"
+    )
+
+    Logger.debug("Backtrace: #{inspect(Process.info(:erlang.self(), :current_stacktrace))}")
+
+    normalize(ap_id)
+  end
+
+  def normalize(%Activity{data: %{"object" => ap_id}}) do
+    Logger.debug(
+      "Object.normalize() called without preloaded object (#{ap_id}).  Consider preloading the object!"
+    )
+
+    Logger.debug("Backtrace: #{inspect(Process.info(:erlang.self(), :current_stacktrace))}")
+
+    normalize(ap_id)
+  end
+
+  # Old way, try fetching the object through cache.
   def normalize(%{"id" => ap_id}), do: normalize(ap_id)
   def normalize(ap_id) when is_binary(ap_id), do: get_cached_by_ap_id(ap_id)
   def normalize(_), do: nil
@@ -86,9 +115,9 @@ defmodule Pleroma.Object do
 
   def delete(%Object{data: %{"id" => id}} = object) do
     with {:ok, _obj} = swap_object_with_tombstone(object),
-         Repo.delete_all(Activity.by_object_ap_id(id)),
+         deleted_activity = Activity.delete_by_ap_id(id),
          {:ok, true} <- Cachex.del(:object_cache, "object:#{id}") do
-      {:ok, object}
+      {:ok, object, deleted_activity}
     end
   end
 
@@ -104,4 +133,50 @@ defmodule Pleroma.Object do
       e -> e
     end
   end
+
+  def increase_replies_count(ap_id) do
+    Object
+    |> where([o], fragment("?->>'id' = ?::text", o.data, ^to_string(ap_id)))
+    |> update([o],
+      set: [
+        data:
+          fragment(
+            """
+            jsonb_set(?, '{repliesCount}',
+              (coalesce((?->>'repliesCount')::int, 0) + 1)::varchar::jsonb, true)
+            """,
+            o.data,
+            o.data
+          )
+      ]
+    )
+    |> Repo.update_all([])
+    |> case do
+      {1, [object]} -> set_cache(object)
+      _ -> {:error, "Not found"}
+    end
+  end
+
+  def decrease_replies_count(ap_id) do
+    Object
+    |> where([o], fragment("?->>'id' = ?::text", o.data, ^to_string(ap_id)))
+    |> update([o],
+      set: [
+        data:
+          fragment(
+            """
+            jsonb_set(?, '{repliesCount}',
+              (greatest(0, (?->>'repliesCount')::int - 1))::varchar::jsonb, true)
+            """,
+            o.data,
+            o.data
+          )
+      ]
+    )
+    |> Repo.update_all([])
+    |> case do
+      {1, [object]} -> set_cache(object)
+      _ -> {:error, "Not found"}
+    end
+  end
 end

lib/pleroma/pagination.ex

@@ -0,0 +1,78 @@
+defmodule Pleroma.Pagination do
+  @moduledoc """
+  Implements Mastodon-compatible pagination.
+  """
+
+  import Ecto.Query
+  import Ecto.Changeset
+
+  alias Pleroma.Repo
+
+  @default_limit 20
+
+  def fetch_paginated(query, params) do
+    options = cast_params(params)
+
+    query
+    |> paginate(options)
+    |> Repo.all()
+    |> enforce_order(options)
+  end
+
+  def paginate(query, options) do
+    query
+    |> restrict(:min_id, options)
+    |> restrict(:since_id, options)
+    |> restrict(:max_id, options)
+    |> restrict(:order, options)
+    |> restrict(:limit, options)
+  end
+
+  defp cast_params(params) do
+    param_types = %{
+      min_id: :string,
+      since_id: :string,
+      max_id: :string,
+      limit: :integer
+    }
+
+    changeset = cast({%{}, param_types}, params, Map.keys(param_types))
+    changeset.changes
+  end
+
+  defp restrict(query, :min_id, %{min_id: min_id}) do
+    where(query, [q], q.id > ^min_id)
+  end
+
+  defp restrict(query, :since_id, %{since_id: since_id}) do
+    where(query, [q], q.id > ^since_id)
+  end
+
+  defp restrict(query, :max_id, %{max_id: max_id}) do
+    where(query, [q], q.id < ^max_id)
+  end
+
+  defp restrict(query, :order, %{min_id: _}) do
+    order_by(query, [u], fragment("? asc nulls last", u.id))
+  end
+
+  defp restrict(query, :order, _options) do
+    order_by(query, [u], fragment("? desc nulls last", u.id))
+  end
+
+  defp restrict(query, :limit, options) do
+    limit = Map.get(options, :limit, @default_limit)
+
+    query
+    |> limit(^limit)
+  end
+
+  defp restrict(query, _, _), do: query
+
+  defp enforce_order(result, %{min_id: _}) do
+    result
+    |> Enum.reverse()
+  end
+
+  defp enforce_order(result, _), do: result
+end

lib/pleroma/plugs/http_security_plug.ex

@@ -34,13 +34,16 @@ defmodule Pleroma.Plugs.HTTPSecurityPlug do
 
   defp csp_string do
     scheme = Config.get([Pleroma.Web.Endpoint, :url])[:scheme]
-    websocket_url = String.replace(Pleroma.Web.Endpoint.static_url(), "http", "ws")
+    static_url = Pleroma.Web.Endpoint.static_url()
+    websocket_url = String.replace(static_url, "http", "ws")
+
+    connect_src = "connect-src 'self' #{static_url} #{websocket_url}"
 
     connect_src =
       if Mix.env() == :dev do
-        "connect-src 'self' http://localhost:3035/ " <> websocket_url
+        connect_src <> " http://localhost:3035/"
       else
-        "connect-src 'self' " <> websocket_url
+        connect_src
       end
 
     script_src =

lib/pleroma/plugs/http_signature.ex

@@ -3,8 +3,8 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.Plugs.HTTPSignaturePlug do
-  alias Pleroma.Web.HTTPSignatures
   alias Pleroma.Web.ActivityPub.Utils
+  alias Pleroma.Web.HTTPSignatures
   import Plug.Conn
   require Logger
 

lib/pleroma/plugs/instance_static.ex

@@ -21,7 +21,8 @@ defmodule Pleroma.Plugs.InstanceStatic do
     end
   end
 
-  @only ~w(index.html static emoji packs sounds images instance favicon.png sw.js sw-pleroma.js)
+  @only ~w(index.html robots.txt static emoji packs sounds images instance favicon.png sw.js
+  sw-pleroma.js)
 
   def init(opts) do
     opts

lib/pleroma/plugs/oauth_plug.ex

@@ -6,8 +6,8 @@ defmodule Pleroma.Plugs.OAuthPlug do
   import Plug.Conn
   import Ecto.Query
 
-  alias Pleroma.User
   alias Pleroma.Repo
+  alias Pleroma.User
   alias Pleroma.Web.OAuth.Token
 
   @realm_reg Regex.compile!("Bearer\:?\s+(.*)$", "i")
@@ -38,6 +38,7 @@ defmodule Pleroma.Plugs.OAuthPlug do
         preload: [user: user]
       )
 
+    # credo:disable-for-next-line Credo.Check.Readability.MaxLineLength
     with %Token{user: %{info: %{deactivated: false} = _} = user} = token_record <- Repo.one(query) do
       {:ok, user, token_record}
     end

lib/pleroma/plugs/uploaded_media.ex

@@ -24,6 +24,18 @@ defmodule Pleroma.Plugs.UploadedMedia do
   end
 
   def call(%{request_path: <<"/", @path, "/", file::binary>>} = conn, opts) do
+    conn =
+      case fetch_query_params(conn) do
+        %{query_params: %{"name" => name}} = conn ->
+          name = String.replace(name, "\"", "\\\"")
+
+          conn
+          |> put_resp_header("content-disposition", "filename=\"#{name}\"")
+
+        conn ->
+          conn
+      end
+
     config = Pleroma.Config.get([Pleroma.Upload])
 
     with uploader <- Keyword.fetch!(config, :uploader),

lib/pleroma/plugs/user_fetcher_plug.ex

@@ -3,8 +3,8 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Plugs.UserFetcherPlug do
-  alias Pleroma.User
   alias Pleroma.Repo
+  alias Pleroma.User
 
   import Plug.Conn
 

lib/pleroma/repo.ex

@@ -3,7 +3,10 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Repo do
-  use Ecto.Repo, otp_app: :pleroma
+  use Ecto.Repo,
+    otp_app: :pleroma,
+    adapter: Ecto.Adapters.Postgres,
+    migration_timestamps: [type: :naive_datetime_usec]
 
   @doc """
   Dynamically loads the repository url from the

lib/pleroma/reverse_proxy.ex

@@ -3,10 +3,12 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.ReverseProxy do
-  @keep_req_headers ~w(accept user-agent accept-encoding cache-control if-modified-since if-unmodified-since if-none-match if-range range)
+  @keep_req_headers ~w(accept user-agent accept-encoding cache-control if-modified-since) ++
+                      ~w(if-unmodified-since if-none-match if-range range)
   @resp_cache_headers ~w(etag date last-modified cache-control)
   @keep_resp_headers @resp_cache_headers ++
-                       ~w(content-type content-disposition content-encoding content-range accept-ranges vary)
+                       ~w(content-type content-disposition content-encoding content-range) ++
+                       ~w(accept-ranges vary)
   @default_cache_control_header "public, max-age=1209600"
   @valid_resp_codes [200, 206, 304]
   @max_read_duration :timer.seconds(30)
@@ -282,8 +284,8 @@ defmodule Pleroma.ReverseProxy do
         headers
 
       has_cache? ->
-        # There's caching header present but no cache-control -- we need to explicitely override it to public
-        # as Plug defaults to "max-age=0, private, must-revalidate"
+        # There's caching header present but no cache-control -- we need to explicitely override it
+        # to public as Plug defaults to "max-age=0, private, must-revalidate"
         List.keystore(headers, "cache-control", 0, {"cache-control", "public"})
 
       true ->
@@ -309,7 +311,25 @@ defmodule Pleroma.ReverseProxy do
       end
 
     if attachment? do
-      disposition = "attachment; filename=" <> Keyword.get(opts, :attachment_name, "attachment")
+      name =
+        try do
+          {{"content-disposition", content_disposition_string}, _} =
+            List.keytake(headers, "content-disposition", 0)
+
+          [name | _] =
+            Regex.run(
+              ~r/filename="((?:[^"\\]|\\.)*)"/u,
+              content_disposition_string || "",
+              capture: :all_but_first
+            )
+
+          name
+        rescue
+          MatchError -> Keyword.get(opts, :attachment_name, "attachment")
+        end
+
+      disposition = "attachment; filename=\"#{name}\""
+
       List.keystore(headers, "content-disposition", 0, {"content-disposition", disposition})
     else
       headers

lib/pleroma/stats.ex

@@ -4,8 +4,8 @@
 
 defmodule Pleroma.Stats do
   import Ecto.Query
-  alias Pleroma.User
   alias Pleroma.Repo
+  alias Pleroma.User
 
   def start_link do
     agent = Agent.start_link(fn -> {[], %{}} end, name: __MODULE__)

lib/pleroma/thread_mute.ex

@@ -4,7 +4,11 @@
 
 defmodule Pleroma.ThreadMute do
   use Ecto.Schema
-  alias Pleroma.{Repo, User, ThreadMute}
+
+  alias Pleroma.Repo
+  alias Pleroma.ThreadMute
+  alias Pleroma.User
+
   require Ecto.Query
 
   schema "thread_mutes" do

lib/pleroma/upload.ex

@@ -70,7 +70,7 @@ defmodule Pleroma.Upload do
            %{
              "type" => "Link",
              "mediaType" => upload.content_type,
-             "href" => url_from_spec(opts.base_url, url_spec)
+             "href" => url_from_spec(upload, opts.base_url, url_spec)
            }
          ],
          "name" => Map.get(opts, :description) || upload.name
@@ -85,6 +85,10 @@ defmodule Pleroma.Upload do
     end
   end
 
+  def char_unescaped?(char) do
+    URI.char_unreserved?(char) or char == ?/
+  end
+
   defp get_opts(opts) do
     {size_limit, activity_type} =
       case Keyword.get(opts, :type) do
@@ -215,16 +219,18 @@ defmodule Pleroma.Upload do
     tmp_path
   end
 
-  defp url_from_spec(base_url, {:file, path}) do
+  defp url_from_spec(%__MODULE__{name: name}, base_url, {:file, path}) do
     path =
-      path
-      |> URI.encode()
-      |> String.replace("?", "%3F")
-      |> String.replace(":", "%3A")
+      URI.encode(path, &char_unescaped?/1) <>
+        if Pleroma.Config.get([__MODULE__, :link_name], false) do
+          "?name=#{URI.encode(name, &char_unescaped?/1)}"
+        else
+          ""
+        end
 
     [base_url, "media", path]
     |> Path.join()
   end
 
-  defp url_from_spec(_base_url, {:url, url}), do: url
+  defp url_from_spec(_upload, _base_url, {:url, url}), do: url
 end

lib/pleroma/uploaders/s3.ex

@@ -6,16 +6,22 @@ defmodule Pleroma.Uploaders.S3 do
   @behaviour Pleroma.Uploaders.Uploader
   require Logger
 
-  # The file name is re-encoded with S3's constraints here to comply with previous links with less strict filenames
+  # The file name is re-encoded with S3's constraints here to comply with previous
+  # links with less strict filenames
   def get_file(file) do
     config = Pleroma.Config.get([__MODULE__])
     bucket = Keyword.fetch!(config, :bucket)
 
     bucket_with_namespace =
-      if namespace = Keyword.get(config, :bucket_namespace) do
-        namespace <> ":" <> bucket
-      else
-        bucket
+      cond do
+        truncated_namespace = Keyword.get(config, :truncated_namespace) ->
+          truncated_namespace
+
+        namespace = Keyword.get(config, :bucket_namespace) ->
+          namespace <> ":" <> bucket
+
+        true ->
+          bucket
       end
 
     {:ok,

lib/pleroma/uploaders/swift/keystone.ex

@@ -17,7 +17,7 @@ defmodule Pleroma.Uploaders.Swift.Keystone do
     |> Poison.decode!()
   end
 
-  def get_token() do
+  def get_token do
     settings = Pleroma.Config.get(Pleroma.Uploaders.Swift)
     username = Keyword.fetch!(settings, :username)
     password = Keyword.fetch!(settings, :password)

lib/pleroma/uploaders/uploader.ex

@@ -29,7 +29,6 @@ defmodule Pleroma.Uploaders.Uploader do
   * `{:error, String.t}` error information if the file failed to be saved to the backend.
   * `:wait_callback` will wait for an http post request at `/api/pleroma/upload_callback/:upload_path` and call the uploader's `http_callback/3` method.
 
-
   """
   @type file_spec :: {:file | :url, String.t()}
   @callback put_file(Pleroma.Upload.t()) ::

lib/pleroma/user.ex

@@ -8,21 +8,21 @@ defmodule Pleroma.User do
   import Ecto.Changeset
   import Ecto.Query
 
+  alias Comeonin.Pbkdf2
+  alias Pleroma.Activity
+  alias Pleroma.Formatter
+  alias Pleroma.Notification
+  alias Pleroma.Object
   alias Pleroma.Repo
   alias Pleroma.User
-  alias Pleroma.Object
   alias Pleroma.Web
-  alias Pleroma.Activity
-  alias Pleroma.Notification
-  alias Comeonin.Pbkdf2
-  alias Pleroma.Formatter
-  alias Pleroma.Web.CommonAPI.Utils, as: CommonUtils
-  alias Pleroma.Web.OStatus
-  alias Pleroma.Web.Websub
-  alias Pleroma.Web.OAuth
-  alias Pleroma.Web.ActivityPub.Utils
   alias Pleroma.Web.ActivityPub.ActivityPub
+  alias Pleroma.Web.ActivityPub.Utils
+  alias Pleroma.Web.CommonAPI.Utils, as: CommonUtils
+  alias Pleroma.Web.OAuth
+  alias Pleroma.Web.OStatus
   alias Pleroma.Web.RelMe
+  alias Pleroma.Web.Websub
 
   require Logger
 
@@ -30,6 +30,7 @@ defmodule Pleroma.User do
 
   @primary_key {:id, Pleroma.FlakeId, autogenerate: true}
 
+  # credo:disable-for-next-line Credo.Check.Readability.MaxLineLength
   @email_regex ~r/^[a-zA-Z0-9.!#$%&'*+\/=?^_`{|}~-]+@[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*$/
 
   @strict_local_nickname_regex ~r/^[a-zA-Z\d]+$/
@@ -49,23 +50,20 @@ defmodule Pleroma.User do
     field(:local, :boolean, default: true)
     field(:follower_address, :string)
     field(:search_rank, :float, virtual: true)
+    field(:search_type, :integer, virtual: true)
     field(:tags, {:array, :string}, default: [])
     field(:bookmarks, {:array, :string}, default: [])
-    field(:last_refreshed_at, :naive_datetime)
+    field(:last_refreshed_at, :naive_datetime_usec)
     has_many(:notifications, Notification)
     embeds_one(:info, Pleroma.User.Info)
 
     timestamps()
   end
 
-  def auth_active?(%User{local: false}), do: true
-
-  def auth_active?(%User{info: %User.Info{confirmation_pending: false}}), do: true
-
   def auth_active?(%User{info: %User.Info{confirmation_pending: true}}),
     do: !Pleroma.Config.get([:instance, :account_activation_required])
 
-  def auth_active?(_), do: false
+  def auth_active?(%User{}), do: true
 
   def visible_for?(user, for_user \\ nil)
 
@@ -81,17 +79,17 @@ defmodule Pleroma.User do
   def superuser?(%User{local: true, info: %User.Info{is_moderator: true}}), do: true
   def superuser?(_), do: false
 
-  def avatar_url(user) do
+  def avatar_url(user, options \\ []) do
     case user.avatar do
       %{"url" => [%{"href" => href} | _]} -> href
-      _ -> "#{Web.base_url()}/images/avi.png"
+      _ -> !options[:no_default] && "#{Web.base_url()}/images/avi.png"
     end
   end
 
-  def banner_url(user) do
+  def banner_url(user, options \\ []) do
     case user.info.banner do
       %{"url" => [%{"href" => href} | _]} -> href
-      _ -> "#{Web.base_url()}/images/banner.png"
+      _ -> !options[:no_default] && "#{Web.base_url()}/images/banner.png"
     end
   end
 
@@ -103,9 +101,8 @@ defmodule Pleroma.User do
     "#{Web.base_url()}/users/#{nickname}"
   end
 
-  def ap_followers(%User{} = user) do
-    "#{ap_id(user)}/followers"
-  end
+  def ap_followers(%User{follower_address: fa}) when is_binary(fa), do: fa
+  def ap_followers(%User{} = user), do: "#{ap_id(user)}/followers"
 
   def user_info(%User{} = user) do
     oneself = if user.local, do: 1, else: 0
@@ -285,7 +282,7 @@ defmodule Pleroma.User do
   def needs_update?(%User{local: false, last_refreshed_at: nil}), do: true
 
   def needs_update?(%User{local: false} = user) do
-    NaiveDateTime.diff(NaiveDateTime.utc_now(), user.last_refreshed_at) >= 86400
+    NaiveDateTime.diff(NaiveDateTime.utc_now(), user.last_refreshed_at) >= 86_400
   end
 
   def needs_update?(_), do: true
@@ -334,10 +331,11 @@ defmodule Pleroma.User do
                 ^followed_addresses
               )
           ]
-        ]
+        ],
+        select: u
       )
 
-    {1, [follower]} = Repo.update_all(q, [], returning: true)
+    {1, [follower]} = Repo.update_all(q, [])
 
     Enum.each(followeds, fn followed ->
       update_follower_count(followed)
@@ -367,10 +365,11 @@ defmodule Pleroma.User do
         q =
           from(u in User,
             where: u.id == ^follower.id,
-            update: [push: [following: ^ap_followers]]
+            update: [push: [following: ^ap_followers]],
+            select: u
           )
 
-        {1, [follower]} = Repo.update_all(q, [], returning: true)
+        {1, [follower]} = Repo.update_all(q, [])
 
         {:ok, _} = update_follower_count(followed)
 
@@ -385,10 +384,11 @@ defmodule Pleroma.User do
       q =
         from(u in User,
           where: u.id == ^follower.id,
-          update: [pull: [following: ^ap_followers]]
+          update: [pull: [following: ^ap_followers]],
+          select: u
         )
 
-      {1, [follower]} = Repo.update_all(q, [], returning: true)
+      {1, [follower]} = Repo.update_all(q, [])
 
       {:ok, followed} = update_follower_count(followed)
 
@@ -435,7 +435,8 @@ defmodule Pleroma.User do
     Repo.get_by(User, ap_id: ap_id)
   end
 
-  # This is mostly an SPC migration fix. This guesses the user nickname (by taking the last part of the ap_id and the domain) and tries to get that user
+  # This is mostly an SPC migration fix. This guesses the user nickname by taking the last part
+  # of the ap_id and the domain and tries to get that user
   def get_by_guessed_nickname(ap_id) do
     domain = URI.parse(ap_id).host
     name = List.last(String.split(ap_id, "/"))
@@ -532,6 +533,10 @@ defmodule Pleroma.User do
       _e ->
         with [_nick, _domain] <- String.split(nickname, "@"),
              {:ok, user} <- fetch_by_nickname(nickname) do
+          if Pleroma.Config.get([:fetch_initial_posts, :enabled]) do
+            {:ok, _} = Task.start(__MODULE__, :fetch_initial_posts, [user])
+          end
+
           user
         else
           _e -> nil
@@ -539,6 +544,17 @@ defmodule Pleroma.User do
     end
   end
 
+  @doc "Fetch some posts when the user has just been federated with"
+  def fetch_initial_posts(user) do
+    pages = Pleroma.Config.get!([:fetch_initial_posts, :pages])
+
+    Enum.each(
+      # Insert all the posts in reverse order, so they're in the right order on the timeline
+      Enum.reverse(Utils.fetch_ordered_collection(user.info.source_data["outbox"], pages)),
+      &Pleroma.Web.Federator.incoming_ap_doc/1
+    )
+  end
+
   def get_followers_query(%User{id: id, follower_address: follower_address}, nil) do
     from(
       u in User,
@@ -620,7 +636,7 @@ defmodule Pleroma.User do
     users =
       user
       |> User.get_follow_requests_query()
-      |> join(:inner, [a], u in User, a.actor == u.ap_id)
+      |> join(:inner, [a], u in User, on: a.actor == u.ap_id)
       |> where([a, u], not fragment("? @> ?", u.following, ^[user.follower_address]))
       |> group_by([a, u], u.id)
       |> select([a, u], u)
@@ -642,7 +658,8 @@ defmodule Pleroma.User do
           )
       ]
     )
-    |> Repo.update_all([], returning: true)
+    |> select([u], u)
+    |> Repo.update_all([])
     |> case do
       {1, [user]} -> set_cache(user)
       _ -> {:error, user}
@@ -662,7 +679,8 @@ defmodule Pleroma.User do
           )
       ]
     )
-    |> Repo.update_all([], returning: true)
+    |> select([u], u)
+    |> Repo.update_all([])
     |> case do
       {1, [user]} -> set_cache(user)
       _ -> {:error, user}
@@ -708,7 +726,8 @@ defmodule Pleroma.User do
           )
       ]
     )
-    |> Repo.update_all([], returning: true)
+    |> select([u], u)
+    |> Repo.update_all([])
     |> case do
       {1, [user]} -> set_cache(user)
       _ -> {:error, user}
@@ -749,77 +768,59 @@ defmodule Pleroma.User do
     Repo.all(query)
   end
 
-  @spec search_for_admin(binary(), %{
-          admin: Pleroma.User.t(),
-          local: boolean(),
-          page: number(),
-          page_size: number()
-        }) :: {:ok, [Pleroma.User.t()], number()}
-  def search_for_admin(term, %{admin: admin, local: local, page: page, page_size: page_size}) do
-    term = String.trim_leading(term, "@")
-
-    local_paginated_query =
-      User
-      |> maybe_local_user_query(local)
-      |> paginate(page, page_size)
-
-    search_query = fts_search_subquery(term, local_paginated_query)
-
-    count =
-      term
-      |> fts_search_subquery()
-      |> maybe_local_user_query(local)
-      |> Repo.aggregate(:count, :id)
-
-    {:ok, do_search(search_query, admin), count}
-  end
-
-  @spec all_for_admin(number(), number()) :: {:ok, [Pleroma.User.t()], number()}
-  def all_for_admin(page, page_size) do
-    query = from(u in User, order_by: u.id)
-
-    paginated_query =
-      query
-      |> paginate(page, page_size)
-
-    count =
-      query
-      |> Repo.aggregate(:count, :id)
-
-    {:ok, Repo.all(paginated_query), count}
-  end
-
   def search(query, resolve \\ false, for_user \\ nil) do
     # Strip the beginning @ off if there is a query
     query = String.trim_leading(query, "@")
 
     if resolve, do: get_or_fetch(query)
 
-    fts_results = do_search(fts_search_subquery(query), for_user)
-
-    {:ok, trigram_results} =
+    {:ok, results} =
       Repo.transaction(fn ->
         Ecto.Adapters.SQL.query(Repo, "select set_limit(0.25)", [])
-        do_search(trigram_search_subquery(query), for_user)
+        Repo.all(search_query(query, for_user))
       end)
 
-    Enum.uniq_by(fts_results ++ trigram_results, & &1.id)
+    results
   end
 
-  defp do_search(subquery, for_user, options \\ []) do
-    q =
-      from(
-        s in subquery(subquery),
-        order_by: [desc: s.search_rank],
-        limit: ^(options[:limit] || 20)
-      )
+  def search_query(query, for_user) do
+    fts_subquery = fts_search_subquery(query)
+    trigram_subquery = trigram_search_subquery(query)
+    union_query = from(s in trigram_subquery, union_all: ^fts_subquery)
+    distinct_query = from(s in subquery(union_query), order_by: s.search_type, distinct: s.id)
 
-    results =
-      q
-      |> Repo.all()
-      |> Enum.filter(&(&1.search_rank > 0))
+    from(s in subquery(boost_search_rank_query(distinct_query, for_user)),
+      order_by: [desc: s.search_rank],
+      limit: 20
+    )
+  end
 
-    boost_search_results(results, for_user)
+  defp boost_search_rank_query(query, nil), do: query
+
+  defp boost_search_rank_query(query, for_user) do
+    friends_ids = get_friends_ids(for_user)
+    followers_ids = get_followers_ids(for_user)
+
+    from(u in subquery(query),
+      select_merge: %{
+        search_rank:
+          fragment(
+            """
+             CASE WHEN (?) THEN (?) * 1.3
+             WHEN (?) THEN (?) * 1.2
+             WHEN (?) THEN (?) * 1.1
+             ELSE (?) END
+            """,
+            u.id in ^friends_ids and u.id in ^followers_ids,
+            u.search_rank,
+            u.id in ^friends_ids,
+            u.search_rank,
+            u.id in ^followers_ids,
+            u.search_rank,
+            u.search_rank
+          )
+      }
+    )
   end
 
   defp fts_search_subquery(term, query \\ User) do
@@ -834,6 +835,7 @@ defmodule Pleroma.User do
     from(
       u in query,
       select_merge: %{
+        search_type: ^0,
         search_rank:
           fragment(
             """
@@ -866,6 +868,8 @@ defmodule Pleroma.User do
     from(
       u in User,
       select_merge: %{
+        # ^1 gives 'Postgrex expected a binary, got 1' for some weird reason
+        search_type: fragment("?", 1),
         search_rank:
           fragment(
             "similarity(?, trim(? || ' ' || coalesce(?, '')))",
@@ -878,33 +882,6 @@ defmodule Pleroma.User do
     )
   end
 
-  defp boost_search_results(results, nil), do: results
-
-  defp boost_search_results(results, for_user) do
-    friends_ids = get_friends_ids(for_user)
-    followers_ids = get_followers_ids(for_user)
-
-    Enum.map(
-      results,
-      fn u ->
-        search_rank_coef =
-          cond do
-            u.id in friends_ids ->
-              1.2
-
-            u.id in followers_ids ->
-              1.1
-
-            true ->
-              1
-          end
-
-        Map.put(u, :search_rank, u.search_rank * search_rank_coef)
-      end
-    )
-    |> Enum.sort_by(&(-&1.search_rank))
-  end
-
   def blocks_import(%User{} = blocker, blocked_identifiers) when is_list(blocked_identifiers) do
     Enum.map(
       blocked_identifiers,
@@ -1044,6 +1021,42 @@ defmodule Pleroma.User do
     )
   end
 
+  def maybe_external_user_query(query, external) do
+    if external, do: external_user_query(query), else: query
+  end
+
+  def external_user_query(query \\ User) do
+    from(
+      u in query,
+      where: u.local == false,
+      where: not is_nil(u.nickname)
+    )
+  end
+
+  def maybe_active_user_query(query, active) do
+    if active, do: active_user_query(query), else: query
+  end
+
+  def active_user_query(query \\ User) do
+    from(
+      u in query,
+      where: fragment("not (?->'deactivated' @> 'true')", u.info),
+      where: not is_nil(u.nickname)
+    )
+  end
+
+  def maybe_deactivated_user_query(query, deactivated) do
+    if deactivated, do: deactivated_user_query(query), else: query
+  end
+
+  def deactivated_user_query(query \\ User) do
+    from(
+      u in query,
+      where: fragment("(?->'deactivated' @> 'true')", u.info),
+      where: not is_nil(u.nickname)
+    )
+  end
+
   def active_local_user_query do
     from(
       u in local_user_query(),
@@ -1083,19 +1096,15 @@ defmodule Pleroma.User do
     friends
     |> Enum.each(fn followed -> User.unfollow(user, followed) end)
 
-    delete_user_activities(user)
-
-    {:ok, user}
-  end
-
-  def delete_user_activities(user) do
-    query = from(a in Activity, where: a.actor == ^user.ap_id)
+    query =
+      from(a in Activity, where: a.actor == ^user.ap_id)
+      |> Activity.with_preloaded_object()
 
     Repo.all(query)
     |> Enum.each(fn activity ->
       case activity.data["type"] do
         "Create" ->
-          ActivityPub.delete(Object.normalize(activity.data["object"]))
+          ActivityPub.delete(Object.normalize(activity))
 
         # TODO: Do something with likes, follows, repeats.
         _ ->
@@ -1112,24 +1121,39 @@ defmodule Pleroma.User do
 
   def html_filter_policy(_), do: @default_scrubbers
 
+  def fetch_by_ap_id(ap_id) do
+    ap_try = ActivityPub.make_user_from_ap_id(ap_id)
+
+    case ap_try do
+      {:ok, user} ->
+        user
+
+      _ ->
+        case OStatus.make_user(ap_id) do
+          {:ok, user} -> user
+          _ -> {:error, "Could not fetch by AP id"}
+        end
+    end
+  end
+
   def get_or_fetch_by_ap_id(ap_id) do
     user = get_by_ap_id(ap_id)
 
     if !is_nil(user) and !User.needs_update?(user) do
       user
     else
-      ap_try = ActivityPub.make_user_from_ap_id(ap_id)
+      # Whether to fetch initial posts for the user (if it's a new user & the fetching is enabled)
+      should_fetch_initial = is_nil(user) and Pleroma.Config.get([:fetch_initial_posts, :enabled])
 
-      case ap_try do
-        {:ok, user} ->
-          user
+      user = fetch_by_ap_id(ap_id)
 
-        _ ->
-          case OStatus.make_user(ap_id) do
-            {:ok, user} -> user
-            _ -> {:error, "Could not fetch by AP id"}
-          end
+      if should_fetch_initial do
+        with %User{} = user do
+          {:ok, _} = Task.start(__MODULE__, :fetch_initial_posts, [user])
+        end
       end
+
+      user
     end
   end
 
@@ -1304,7 +1328,7 @@ defmodule Pleroma.User do
     |> Enum.map(&String.downcase(&1))
   end
 
-  defp local_nickname_regex() do
+  defp local_nickname_regex do
     if Pleroma.Config.get([:instance, :extended_nickname_format]) do
       @extended_local_nickname_regex
     else
@@ -1347,4 +1371,8 @@ defmodule Pleroma.User do
       offset: ^((page - 1) * page_size)
     )
   end
+
+  def showing_reblogs?(%User{} = user, %User{} = target) do
+    target.ap_id not in user.info.muted_reblogs
+  end
 end

lib/pleroma/user/info.ex

@@ -6,6 +6,8 @@ defmodule Pleroma.User.Info do
   use Ecto.Schema
   import Ecto.Changeset
 
+  alias Pleroma.User.Info
+
   embedded_schema do
     field(:banner, :map, default: %{})
     field(:background, :map, default: %{})
@@ -19,6 +21,7 @@ defmodule Pleroma.User.Info do
     field(:blocks, {:array, :string}, default: [])
     field(:domain_blocks, {:array, :string}, default: [])
     field(:mutes, {:array, :string}, default: [])
+    field(:muted_reblogs, {:array, :string}, default: [])
     field(:deactivated, :boolean, default: false)
     field(:no_rich_text, :boolean, default: false)
     field(:ap_enabled, :boolean, default: false)
@@ -250,4 +253,23 @@ defmodule Pleroma.User.Info do
 
     cast(info, params, [:pinned_activities])
   end
+
+  def roles(%Info{is_moderator: is_moderator, is_admin: is_admin}) do
+    %{
+      admin: is_admin,
+      moderator: is_moderator
+    }
+  end
+
+  def add_reblog_mute(info, ap_id) do
+    params = %{muted_reblogs: info.muted_reblogs ++ [ap_id]}
+
+    cast(info, params, [:muted_reblogs])
+  end
+
+  def remove_reblog_mute(info, ap_id) do
+    params = %{muted_reblogs: List.delete(info.muted_reblogs, ap_id)}
+
+    cast(info, params, [:muted_reblogs])
+  end
 end

lib/pleroma/user/welcome_message.ex

@@ -14,7 +14,7 @@ defmodule Pleroma.User.WelcomeMessage do
     end
   end
 
-  defp welcome_user() do
+  defp welcome_user do
     with nickname when is_binary(nickname) <-
            Pleroma.Config.get([:instance, :welcome_user_nickname]),
          %User{local: true} = user <- User.get_cached_by_nickname(nickname) do
@@ -24,7 +24,7 @@ defmodule Pleroma.User.WelcomeMessage do
     end
   end
 
-  defp welcome_message() do
+  defp welcome_message do
     Pleroma.Config.get([:instance, :welcome_message])
   end
 end

lib/pleroma/user_invite_token.ex

@@ -7,8 +7,8 @@ defmodule Pleroma.UserInviteToken do
 
   import Ecto.Changeset
 
-  alias Pleroma.UserInviteToken
   alias Pleroma.Repo
+  alias Pleroma.UserInviteToken
 
   schema "user_invite_tokens" do
     field(:token, :string)

lib/pleroma/web/activity_pub/activity_pub.ex

@@ -4,17 +4,17 @@
 
 defmodule Pleroma.Web.ActivityPub.ActivityPub do
   alias Pleroma.Activity
-  alias Pleroma.Repo
+  alias Pleroma.Instances
+  alias Pleroma.Notification
   alias Pleroma.Object
+  alias Pleroma.Repo
   alias Pleroma.Upload
   alias Pleroma.User
-  alias Pleroma.Notification
-  alias Pleroma.Instances
-  alias Pleroma.Web.ActivityPub.Transmogrifier
   alias Pleroma.Web.ActivityPub.MRF
-  alias Pleroma.Web.WebFinger
+  alias Pleroma.Web.ActivityPub.Transmogrifier
   alias Pleroma.Web.Federator
   alias Pleroma.Web.OStatus
+  alias Pleroma.Web.WebFinger
 
   import Ecto.Query
   import Pleroma.Web.ActivityPub.Utils
@@ -89,13 +89,37 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     if is_public?(object), do: User.decrease_note_count(actor), else: {:ok, actor}
   end
 
+  def increase_replies_count_if_reply(%{
+        "object" =>
+          %{"inReplyTo" => reply_ap_id, "inReplyToStatusId" => reply_status_id} = object,
+        "type" => "Create"
+      }) do
+    if is_public?(object) do
+      Activity.increase_replies_count(reply_status_id)
+      Object.increase_replies_count(reply_ap_id)
+    end
+  end
+
+  def increase_replies_count_if_reply(_create_data), do: :noop
+
+  def decrease_replies_count_if_reply(%Object{
+        data: %{"inReplyTo" => reply_ap_id, "inReplyToStatusId" => reply_status_id} = object
+      }) do
+    if is_public?(object) do
+      Activity.decrease_replies_count(reply_status_id)
+      Object.decrease_replies_count(reply_ap_id)
+    end
+  end
+
+  def decrease_replies_count_if_reply(_object), do: :noop
+
   def insert(map, local \\ true) when is_map(map) do
     with nil <- Activity.normalize(map),
          map <- lazy_put_activity_defaults(map),
          :ok <- check_actor_is_active(map["actor"]),
          {_, true} <- {:remote_limit_error, check_remote_limit(map)},
          {:ok, map} <- MRF.filter(map),
-         :ok <- insert_full_object(map) do
+         {:ok, object} <- insert_full_object(map) do
       {recipients, _, _} = get_recipients(map)
 
       {:ok, activity} =
@@ -106,6 +130,14 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
           recipients: recipients
         })
 
+      # Splice in the child object if we have one.
+      activity =
+        if !is_nil(object) do
+          Map.put(activity, :object, object)
+        else
+          activity
+        end
+
       Task.start(fn ->
         Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity)
       end)
@@ -170,7 +202,9 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
              additional
            ),
          {:ok, activity} <- insert(create_data, local),
-         # Changing note count prior to enqueuing federation task in order to avoid race conditions on updating user.info
+         _ <- increase_replies_count_if_reply(create_data),
+         # Changing note count prior to enqueuing federation task in order to avoid
+         # race conditions on updating user.info
          {:ok, _actor} <- increase_note_count_if_public(actor, activity),
          :ok <- maybe_federate(activity) do
       {:ok, activity}
@@ -309,17 +343,20 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
 
   def delete(%Object{data: %{"id" => id, "actor" => actor}} = object, local \\ true) do
     user = User.get_cached_by_ap_id(actor)
+    to = (object.data["to"] || []) ++ (object.data["cc"] || [])
 
-    data = %{
-      "type" => "Delete",
-      "actor" => actor,
-      "object" => id,
-      "to" => [user.follower_address, "https://www.w3.org/ns/activitystreams#Public"]
-    }
-
-    with {:ok, _} <- Object.delete(object),
+    with {:ok, object, activity} <- Object.delete(object),
+         data <- %{
+           "type" => "Delete",
+           "actor" => actor,
+           "object" => id,
+           "to" => to,
+           "deleted_activity_id" => activity && activity.id
+         },
          {:ok, activity} <- insert(data, local),
-         # Changing note count prior to enqueuing federation task in order to avoid race conditions on updating user.info
+         _ <- decrease_replies_count_if_reply(object),
+         # Changing note count prior to enqueuing federation task in order to avoid
+         # race conditions on updating user.info
          {:ok, _actor} <- decrease_note_count_if_public(user, object),
          :ok <- maybe_federate(activity) do
       {:ok, activity}
@@ -367,20 +404,38 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
           content: content
         } = params
       ) do
-    additional = params[:additional] || %{}
-
     # only accept false as false value
     local = !(params[:local] == false)
+    forward = !(params[:forward] == false)
 
-    %{
+    additional = params[:additional] || %{}
+
+    params = %{
       actor: actor,
       context: context,
       account: account,
       statuses: statuses,
       content: content
     }
-    |> make_flag_data(additional)
-    |> insert(local)
+
+    additional =
+      if forward do
+        Map.merge(additional, %{"to" => [], "cc" => [account.ap_id]})
+      else
+        Map.merge(additional, %{"to" => [], "cc" => []})
+      end
+
+    with flag_data <- make_flag_data(params, additional),
+         {:ok, activity} <- insert(flag_data, local),
+         :ok <- maybe_federate(activity) do
+      Enum.each(User.all_superusers(), fn superuser ->
+        superuser
+        |> Pleroma.AdminEmail.report(actor, account, statuses, content)
+        |> Pleroma.Mailer.deliver_async()
+      end)
+
+      {:ok, activity}
+    end
   end
 
   def fetch_activities_for_context(context, opts \\ %{}) do
@@ -409,6 +464,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
           ),
         order_by: [desc: :id]
       )
+      |> Activity.with_preloaded_object()
 
     Repo.all(query)
   end
@@ -501,7 +557,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
        when is_list(tag_reject) and tag_reject != [] do
     from(
       activity in query,
-      where: fragment("(not (? #> '{\"object\",\"tag\"}') \\?| ?)", activity.data, ^tag_reject)
+      where: fragment(~s(\(not \(? #> '{"object","tag"}'\) \\?| ?\)), activity.data, ^tag_reject)
     )
   end
 
@@ -511,7 +567,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
        when is_list(tag_all) and tag_all != [] do
     from(
       activity in query,
-      where: fragment("(? #> '{\"object\",\"tag\"}') \\?& ?", activity.data, ^tag_all)
+      where: fragment(~s(\(? #> '{"object","tag"}'\) \\?& ?), activity.data, ^tag_all)
     )
   end
 
@@ -520,14 +576,14 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
   defp restrict_tag(query, %{"tag" => tag}) when is_list(tag) do
     from(
       activity in query,
-      where: fragment("(? #> '{\"object\",\"tag\"}') \\?| ?", activity.data, ^tag)
+      where: fragment(~s(\(? #> '{"object","tag"}'\) \\?| ?), activity.data, ^tag)
     )
   end
 
   defp restrict_tag(query, %{"tag" => tag}) when is_binary(tag) do
     from(
       activity in query,
-      where: fragment("? <@ (? #> '{\"object\",\"tag\"}')", ^tag, activity.data)
+      where: fragment(~s(? <@ (? #> '{"object","tag"}'\)), ^tag, activity.data)
     )
   end
 
@@ -600,7 +656,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
   defp restrict_favorited_by(query, %{"favorited_by" => ap_id}) do
     from(
       activity in query,
-      where: fragment("? <@ (? #> '{\"object\",\"likes\"}')", ^ap_id, activity.data)
+      where: fragment(~s(? <@ (? #> '{"object","likes"}'\)), ^ap_id, activity.data)
     )
   end
 
@@ -609,7 +665,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
   defp restrict_media(query, %{"only_media" => val}) when val == "true" or val == "1" do
     from(
       activity in query,
-      where: fragment("not (? #> '{\"object\",\"attachment\"}' = ?)", activity.data, ^[])
+      where: fragment(~s(not (? #> '{"object","attachment"}' = ?\)), activity.data, ^[])
     )
   end
 
@@ -676,6 +732,30 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
 
   defp restrict_pinned(query, _), do: query
 
+  defp restrict_muted_reblogs(query, %{"muting_user" => %User{info: info}}) do
+    muted_reblogs = info.muted_reblogs || []
+
+    from(
+      activity in query,
+      where:
+        fragment(
+          "not ( ?->>'type' = 'Announce' and ? = ANY(?))",
+          activity.data,
+          activity.actor,
+          ^muted_reblogs
+        )
+    )
+  end
+
+  defp restrict_muted_reblogs(query, _), do: query
+
+  defp maybe_preload_objects(query, %{"skip_preload" => true}), do: query
+
+  defp maybe_preload_objects(query, _) do
+    query
+    |> Activity.with_preloaded_object()
+  end
+
   def fetch_activities_query(recipients, opts \\ %{}) do
     base_query =
       from(
@@ -685,6 +765,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
       )
 
     base_query
+    |> maybe_preload_objects(opts)
     |> restrict_recipients(recipients, opts["user"])
     |> restrict_tag(opts)
     |> restrict_tag_reject(opts)
@@ -703,6 +784,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     |> restrict_replies(opts)
     |> restrict_reblogs(opts)
     |> restrict_pinned(opts)
+    |> restrict_muted_reblogs(opts)
   end
 
   def fetch_activities(recipients, opts \\ %{}) do
@@ -906,7 +988,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
            },
            :ok <- Transmogrifier.contain_origin(id, params),
            {:ok, activity} <- Transmogrifier.handle_incoming(params) do
-        {:ok, Object.normalize(activity.data["object"])}
+        {:ok, Object.normalize(activity)}
       else
         {:error, {:reject, nil}} ->
           {:reject, nil}
@@ -918,7 +1000,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
           Logger.info("Couldn't get object via AP, trying out OStatus fetching...")
 
           case OStatus.fetch_activity_from_url(id) do
-            {:ok, [activity | _]} -> {:ok, Object.normalize(activity.data["object"])}
+            {:ok, [activity | _]} -> {:ok, Object.normalize(activity)}
             e -> e
           end
       end

lib/pleroma/web/activity_pub/activity_pub_controller.ex

@@ -6,15 +6,15 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
   use Pleroma.Web, :controller
 
   alias Pleroma.Activity
-  alias Pleroma.User
   alias Pleroma.Object
-  alias Pleroma.Web.ActivityPub.ObjectView
-  alias Pleroma.Web.ActivityPub.UserView
+  alias Pleroma.User
   alias Pleroma.Web.ActivityPub.ActivityPub
-  alias Pleroma.Web.ActivityPub.Visibility
+  alias Pleroma.Web.ActivityPub.ObjectView
   alias Pleroma.Web.ActivityPub.Relay
   alias Pleroma.Web.ActivityPub.Transmogrifier
+  alias Pleroma.Web.ActivityPub.UserView
   alias Pleroma.Web.ActivityPub.Utils
+  alias Pleroma.Web.ActivityPub.Visibility
   alias Pleroma.Web.Federator
 
   require Logger

lib/pleroma/web/activity_pub/mrf.ex

@@ -16,7 +16,7 @@ defmodule Pleroma.Web.ActivityPub.MRF do
     end)
   end
 
-  def get_policies() do
+  def get_policies do
     Application.get_env(:pleroma, :instance, [])
     |> Keyword.get(:rewrite_policy, [])
     |> get_policies()

lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex

@@ -23,15 +23,21 @@ defmodule Pleroma.Web.ActivityPub.MRF.AntiFollowbotPolicy do
   defp score_displayname(_), do: 0.0
 
   defp determine_if_followbot(%User{nickname: nickname, name: displayname}) do
+    # nickname will always be a binary string because it's generated by Pleroma.
     nick_score =
       nickname
       |> String.downcase()
       |> score_nickname()
 
+    # displayname will either be a binary string or nil, if a displayname isn't set.
     name_score =
-      displayname
-      |> String.downcase()
-      |> score_displayname()
+      if is_binary(displayname) do
+        displayname
+        |> String.downcase()
+        |> score_displayname()
+      else
+        0.0
+      end
 
     nick_score + name_score
   end

lib/pleroma/web/activity_pub/mrf/keyword_policy.ex

@@ -4,6 +4,10 @@
 
 defmodule Pleroma.Web.ActivityPub.MRF.KeywordPolicy do
   @behaviour Pleroma.Web.ActivityPub.MRF
+  defp string_matches?(string, _) when not is_binary(string) do
+    false
+  end
+
   defp string_matches?(string, pattern) when is_binary(pattern) do
     String.contains?(string, pattern)
   end
@@ -44,14 +48,29 @@ defmodule Pleroma.Web.ActivityPub.MRF.KeywordPolicy do
   end
 
   defp check_replace(%{"object" => %{"content" => content, "summary" => summary}} = message) do
+    content =
+      if is_binary(content) do
+        content
+      else
+        ""
+      end
+
+    summary =
+      if is_binary(summary) do
+        summary
+      else
+        ""
+      end
+
     {content, summary} =
-      Enum.reduce(Pleroma.Config.get([:mrf_keyword, :replace]), {content, summary}, fn {pattern,
-                                                                                        replacement},
-                                                                                       {content_acc,
-                                                                                        summary_acc} ->
-        {String.replace(content_acc, pattern, replacement),
-         String.replace(summary_acc, pattern, replacement)}
-      end)
+      Enum.reduce(
+        Pleroma.Config.get([:mrf_keyword, :replace]),
+        {content, summary},
+        fn {pattern, replacement}, {content_acc, summary_acc} ->
+          {String.replace(content_acc, pattern, replacement),
+           String.replace(summary_acc, pattern, replacement)}
+        end
+      )
 
     {:ok,
      message
@@ -59,11 +78,6 @@ defmodule Pleroma.Web.ActivityPub.MRF.KeywordPolicy do
      |> put_in(["object", "summary"], summary)}
   end
 
-  @impl true
-  def filter(%{"object" => %{"content" => nil}} = message) do
-    {:ok, message}
-  end
-
   @impl true
   def filter(%{"type" => "Create", "object" => %{"content" => _content}} = message) do
     with {:ok, message} <- check_reject(message),

lib/pleroma/web/activity_pub/relay.ex

@@ -3,9 +3,9 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.ActivityPub.Relay do
-  alias Pleroma.User
-  alias Pleroma.Object
   alias Pleroma.Activity
+  alias Pleroma.Object
+  alias Pleroma.User
   alias Pleroma.Web.ActivityPub.ActivityPub
   require Logger
 
@@ -41,7 +41,7 @@ defmodule Pleroma.Web.ActivityPub.Relay do
 
   def publish(%Activity{data: %{"type" => "Create"}} = activity) do
     with %User{} = user <- get_actor(),
-         %Object{} = object <- Object.normalize(activity.data["object"]["id"]) do
+         %Object{} = object <- Object.normalize(activity) do
       ActivityPub.announce(user, object, nil, true, false)
     else
       e -> Logger.error("error: #{inspect(e)}")

lib/pleroma/web/activity_pub/transmogrifier.ex

@@ -7,9 +7,9 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
   A module to handle coding from internal to wire ActivityPub and back.
   """
   alias Pleroma.Activity
-  alias Pleroma.User
   alias Pleroma.Object
   alias Pleroma.Repo
+  alias Pleroma.User
   alias Pleroma.Web.ActivityPub.ActivityPub
   alias Pleroma.Web.ActivityPub.Utils
   alias Pleroma.Web.ActivityPub.Visibility
@@ -86,11 +86,15 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
   end
 
   def fix_addressing_list(map, field) do
-    if is_binary(map[field]) do
-      map
-      |> Map.put(field, [map[field]])
-    else
-      map
+    cond do
+      is_binary(map[field]) ->
+        Map.put(map, field, [map[field]])
+
+      is_nil(map[field]) ->
+        Map.put(map, field, [])
+
+      true ->
+        map
     end
   end
 
@@ -128,13 +132,42 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
     |> fix_explicit_addressing(explicit_mentions)
   end
 
+  # if as:Public is addressed, then make sure the followers collection is also addressed
+  # so that the activities will be delivered to local users.
+  def fix_implicit_addressing(%{"to" => to, "cc" => cc} = object, followers_collection) do
+    recipients = to ++ cc
+
+    if followers_collection not in recipients do
+      cond do
+        "https://www.w3.org/ns/activitystreams#Public" in cc ->
+          to = to ++ [followers_collection]
+          Map.put(object, "to", to)
+
+        "https://www.w3.org/ns/activitystreams#Public" in to ->
+          cc = cc ++ [followers_collection]
+          Map.put(object, "cc", cc)
+
+        true ->
+          object
+      end
+    else
+      object
+    end
+  end
+
+  def fix_implicit_addressing(object, _), do: object
+
   def fix_addressing(object) do
+    %User{} = user = User.get_or_fetch_by_ap_id(object["actor"])
+    followers_collection = User.ap_followers(user)
+
     object
     |> fix_addressing_list("to")
     |> fix_addressing_list("cc")
     |> fix_addressing_list("bto")
     |> fix_addressing_list("bcc")
     |> fix_explicit_addressing
+    |> fix_implicit_addressing(followers_collection)
   end
 
   def fix_actor(%{"attributedTo" => actor} = object) do
@@ -355,6 +388,40 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
     end
   end
 
+  # Flag objects are placed ahead of the ID check because Mastodon 2.8 and earlier send them
+  # with nil ID.
+  def handle_incoming(%{"type" => "Flag", "object" => objects, "actor" => actor} = data) do
+    with context <- data["context"] || Utils.generate_context_id(),
+         content <- data["content"] || "",
+         %User{} = actor <- User.get_cached_by_ap_id(actor),
+
+         # Reduce the object list to find the reported user.
+         %User{} = account <-
+           Enum.reduce_while(objects, nil, fn ap_id, _ ->
+             with %User{} = user <- User.get_cached_by_ap_id(ap_id) do
+               {:halt, user}
+             else
+               _ -> {:cont, nil}
+             end
+           end),
+
+         # Remove the reported user from the object list.
+         statuses <- Enum.filter(objects, fn ap_id -> ap_id != account.ap_id end) do
+      params = %{
+        actor: actor,
+        context: context,
+        account: account,
+        statuses: statuses,
+        content: content,
+        additional: %{
+          "cc" => [account.ap_id]
+        }
+      }
+
+      ActivityPub.flag(params)
+    end
+  end
+
   # disallow objects with bogus IDs
   def handle_incoming(%{"id" => nil}), do: :error
   def handle_incoming(%{"id" => ""}), do: :error
@@ -650,10 +717,10 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
     if object = Object.normalize(id), do: {:ok, object}, else: nil
   end
 
-  def set_reply_to_uri(%{"inReplyTo" => inReplyTo} = object) when is_binary(inReplyTo) do
-    with false <- String.starts_with?(inReplyTo, "http"),
-         {:ok, %{data: replied_to_object}} <- get_obj_helper(inReplyTo) do
-      Map.put(object, "inReplyTo", replied_to_object["external_url"] || inReplyTo)
+  def set_reply_to_uri(%{"inReplyTo" => in_reply_to} = object) when is_binary(in_reply_to) do
+    with false <- String.starts_with?(in_reply_to, "http"),
+         {:ok, %{data: replied_to_object}} <- get_obj_helper(in_reply_to) do
+      Map.put(object, "inReplyTo", replied_to_object["external_url"] || in_reply_to)
     else
       _e -> object
     end
@@ -736,6 +803,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
   def prepare_outgoing(%{"type" => _type} = data) do
     data =
       data
+      |> strip_internal_fields
       |> maybe_fix_object_url
       |> Map.merge(Utils.make_json_ld_header())
 
@@ -829,10 +897,10 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
   end
 
   def add_attributed_to(object) do
-    attributedTo = object["attributedTo"] || object["actor"]
+    attributed_to = object["attributedTo"] || object["actor"]
 
     object
-    |> Map.put("attributedTo", attributedTo)
+    |> Map.put("attributedTo", attributed_to)
   end
 
   def add_likes(%{"id" => id, "like_count" => likes} = object) do
@@ -870,7 +938,8 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
       "announcements",
       "announcement_count",
       "emoji",
-      "context_id"
+      "context_id",
+      "deleted_activity_id"
     ])
   end
 
@@ -886,7 +955,8 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
   defp strip_internal_tags(object), do: object
 
   defp user_upgrade_task(user) do
-    old_follower_address = User.ap_followers(user)
+    # we pass a fake user so that the followers collection is stripped away
+    old_follower_address = User.ap_followers(%User{nickname: user.nickname})
 
     q =
       from(

lib/pleroma/web/activity_pub/utils.ex

@@ -3,16 +3,17 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.ActivityPub.Utils do
-  alias Pleroma.Repo
-  alias Pleroma.Web
-  alias Pleroma.Object
-  alias Pleroma.Activity
-  alias Pleroma.User
-  alias Pleroma.Notification
-  alias Pleroma.Web.Router.Helpers
-  alias Pleroma.Web.Endpoint
   alias Ecto.Changeset
   alias Ecto.UUID
+  alias Pleroma.Activity
+  alias Pleroma.Notification
+  alias Pleroma.Object
+  alias Pleroma.Repo
+  alias Pleroma.User
+  alias Pleroma.Web
+  alias Pleroma.Web.ActivityPub.Visibility
+  alias Pleroma.Web.Endpoint
+  alias Pleroma.Web.Router.Helpers
 
   import Ecto.Query
 
@@ -208,12 +209,12 @@ defmodule Pleroma.Web.ActivityPub.Utils do
   """
   def insert_full_object(%{"object" => %{"type" => type} = object_data})
       when is_map(object_data) and type in @supported_object_types do
-    with {:ok, _} <- Object.create(object_data) do
-      :ok
+    with {:ok, object} <- Object.create(object_data) do
+      {:ok, object}
     end
   end
 
-  def insert_full_object(_), do: :ok
+  def insert_full_object(_), do: {:ok, nil}
 
   def update_object_in_activities(%{data: %{"id" => id}} = object) do
     # TODO
@@ -274,13 +275,31 @@ defmodule Pleroma.Web.ActivityPub.Utils do
     Repo.all(query)
   end
 
-  def make_like_data(%User{ap_id: ap_id} = actor, %{data: %{"id" => id}} = object, activity_id) do
+  def make_like_data(
+        %User{ap_id: ap_id} = actor,
+        %{data: %{"actor" => object_actor_id, "id" => id}} = object,
+        activity_id
+      ) do
+    object_actor = User.get_cached_by_ap_id(object_actor_id)
+
+    to =
+      if Visibility.is_public?(object) do
+        [actor.follower_address, object.data["actor"]]
+      else
+        [object.data["actor"]]
+      end
+
+    cc =
+      (object.data["to"] ++ (object.data["cc"] || []))
+      |> List.delete(actor.ap_id)
+      |> List.delete(object_actor.follower_address)
+
     data = %{
       "type" => "Like",
       "actor" => ap_id,
       "object" => id,
-      "to" => [actor.follower_address, object.data["actor"]],
-      "cc" => ["https://www.w3.org/ns/activitystreams#Public"],
+      "to" => to,
+      "cc" => cc,
       "context" => object.data["context"]
     }
 
@@ -602,7 +621,13 @@ defmodule Pleroma.Web.ActivityPub.Utils do
   #### Flag-related helpers
 
   def make_flag_data(params, additional) do
-    status_ap_ids = Enum.map(params.statuses || [], & &1.data["id"])
+    status_ap_ids =
+      Enum.map(params.statuses || [], fn
+        %Activity{} = act -> act.data["id"]
+        act when is_map(act) -> act["id"]
+        act when is_binary(act) -> act
+      end)
+
     object = [params.account.ap_id] ++ status_ap_ids
 
     %{
@@ -614,4 +639,43 @@ defmodule Pleroma.Web.ActivityPub.Utils do
     }
     |> Map.merge(additional)
   end
+
+  @doc """
+  Fetches the OrderedCollection/OrderedCollectionPage from `from`, limiting the amount of pages fetched after
+  the first one to `pages_left` pages.
+  If the amount of pages is higher than the collection has, it returns whatever was there.
+  """
+  def fetch_ordered_collection(from, pages_left, acc \\ []) do
+    with {:ok, response} <- Tesla.get(from),
+         {:ok, collection} <- Poison.decode(response.body) do
+      case collection["type"] do
+        "OrderedCollection" ->
+          # If we've encountered the OrderedCollection and not the page,
+          # just call the same function on the page address
+          fetch_ordered_collection(collection["first"], pages_left)
+
+        "OrderedCollectionPage" ->
+          if pages_left > 0 do
+            # There are still more pages
+            if Map.has_key?(collection, "next") do
+              # There are still more pages, go deeper saving what we have into the accumulator
+              fetch_ordered_collection(
+                collection["next"],
+                pages_left - 1,
+                acc ++ collection["orderedItems"]
+              )
+            else
+              # No more pages left, just return whatever we already have
+              acc ++ collection["orderedItems"]
+            end
+          else
+            # Got the amount of pages needed, add them all to the accumulator
+            acc ++ collection["orderedItems"]
+          end
+
+        _ ->
+          {:error, "Not an OrderedCollection or OrderedCollectionPage"}
+      end
+    end
+  end
 end

lib/pleroma/web/activity_pub/views/object_view.ex

@@ -17,7 +17,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectView do
 
   def render("object.json", %{object: %Activity{data: %{"type" => "Create"}} = activity}) do
     base = Pleroma.Web.ActivityPub.Utils.make_json_ld_header()
-    object = Object.normalize(activity.data["object"])
+    object = Object.normalize(activity)
 
     additional =
       Transmogrifier.prepare_object(activity.data)
@@ -28,7 +28,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectView do
 
   def render("object.json", %{object: %Activity{} = activity}) do
     base = Pleroma.Web.ActivityPub.Utils.make_json_ld_header()
-    object = Object.normalize(activity.data["object"])
+    object = Object.normalize(activity)
 
     additional =
       Transmogrifier.prepare_object(activity.data)

lib/pleroma/web/activity_pub/views/user_view.ex

@@ -5,15 +5,15 @@
 defmodule Pleroma.Web.ActivityPub.UserView do
   use Pleroma.Web, :view
 
-  alias Pleroma.Web.WebFinger
-  alias Pleroma.Web.Salmon
-  alias Pleroma.User
   alias Pleroma.Repo
+  alias Pleroma.User
   alias Pleroma.Web.ActivityPub.ActivityPub
   alias Pleroma.Web.ActivityPub.Transmogrifier
   alias Pleroma.Web.ActivityPub.Utils
-  alias Pleroma.Web.Router.Helpers
   alias Pleroma.Web.Endpoint
+  alias Pleroma.Web.Router.Helpers
+  alias Pleroma.Web.Salmon
+  alias Pleroma.Web.WebFinger
 
   import Ecto.Query
 
@@ -87,16 +87,10 @@ defmodule Pleroma.Web.ActivityPub.UserView do
         "publicKeyPem" => public_key
       },
       "endpoints" => endpoints,
-      "icon" => %{
-        "type" => "Image",
-        "url" => User.avatar_url(user)
-      },
-      "image" => %{
-        "type" => "Image",
-        "url" => User.banner_url(user)
-      },
       "tag" => user.info.source_data["tag"] || []
     }
+    |> Map.merge(maybe_make_image(&User.avatar_url/2, "icon", user))
+    |> Map.merge(maybe_make_image(&User.banner_url/2, "image", user))
     |> Map.merge(Utils.make_json_ld_header())
   end
 
@@ -294,4 +288,17 @@ defmodule Pleroma.Web.ActivityPub.UserView do
       map
     end
   end
+
+  defp maybe_make_image(func, key, user) do
+    if image = func.(user, no_default: true) do
+      %{
+        key => %{
+          "type" => "Image",
+          "url" => image
+        }
+      }
+    else
+      %{}
+    end
+  end
 end

lib/pleroma/web/admin_api/admin_api_controller.ex

@@ -3,17 +3,18 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.AdminAPI.AdminAPIController do
-  @users_page_size 50
-
   use Pleroma.Web, :controller
   alias Pleroma.User
   alias Pleroma.Web.ActivityPub.Relay
-  alias Pleroma.Web.MastodonAPI.Admin.AccountView
+  alias Pleroma.Web.AdminAPI.AccountView
+  alias Pleroma.Web.AdminAPI.Search
 
   import Pleroma.Web.ControllerHelper, only: [json_response: 3]
 
   require Logger
 
+  @users_page_size 50
+
   action_fallback(:errors)
 
   def user_delete(conn, %{"nickname" => nickname}) do
@@ -44,6 +45,15 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
     |> json(user.nickname)
   end
 
+  def user_show(conn, %{"nickname" => nickname}) do
+    with %User{} = user <- User.get_by_nickname(nickname) do
+      conn
+      |> json(AccountView.render("show.json", %{user: user}))
+    else
+      _ -> {:error, :not_found}
+    end
+  end
+
   def user_toggle_activation(conn, %{"nickname" => nickname}) do
     user = User.get_by_nickname(nickname)
 
@@ -65,8 +75,15 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
 
   def list_users(conn, params) do
     {page, page_size} = page_params(params)
+    filters = maybe_parse_filters(params["filters"])
 
-    with {:ok, users, count} <- User.all_for_admin(page, page_size),
+    search_params = %{
+      query: params["query"],
+      page: page,
+      page_size: page_size
+    }
+
+    with {:ok, users, count} <- Search.user(Map.merge(search_params, filters)),
          do:
            conn
            |> json(
@@ -78,25 +95,17 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
            )
   end
 
-  def search_users(%{assigns: %{user: admin}} = conn, %{"query" => query} = params) do
-    {page, page_size} = page_params(params)
+  @filters ~w(local external active deactivated)
 
-    with {:ok, users, count} <-
-           User.search_for_admin(query, %{
-             admin: admin,
-             local: params["local"] == "true",
-             page: page,
-             page_size: page_size
-           }),
-         do:
-           conn
-           |> json(
-             AccountView.render("index.json",
-               users: users,
-               count: count,
-               page_size: page_size
-             )
-           )
+  defp maybe_parse_filters(filters) when is_nil(filters) or filters == "", do: %{}
+
+  @spec maybe_parse_filters(String.t()) :: %{required(String.t()) => true} | %{}
+  defp maybe_parse_filters(filters) do
+    filters
+    |> String.split(",")
+    |> Enum.filter(&Enum.member?(@filters, &1))
+    |> Enum.map(&String.to_atom(&1))
+    |> Enum.into(%{}, &{&1, true})
   end
 
   def right_add(conn, %{"permission_group" => permission_group, "nickname" => nickname})
@@ -231,6 +240,12 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
     |> json(token.token)
   end
 
+  def errors(conn, {:error, :not_found}) do
+    conn
+    |> put_status(404)
+    |> json("Not found")
+  end
+
   def errors(conn, {:param_cast, _}) do
     conn
     |> put_status(400)

lib/pleroma/web/admin_api/search.ex

@@ -0,0 +1,54 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.AdminAPI.Search do
+  import Ecto.Query
+
+  alias Pleroma.Repo
+  alias Pleroma.User
+
+  @page_size 50
+
+  def user(%{query: term} = params) when is_nil(term) or term == "" do
+    query = maybe_filtered_query(params)
+
+    paginated_query =
+      maybe_filtered_query(params)
+      |> paginate(params[:page] || 1, params[:page_size] || @page_size)
+
+    count = query |> Repo.aggregate(:count, :id)
+
+    results = Repo.all(paginated_query)
+
+    {:ok, results, count}
+  end
+
+  def user(%{query: term} = params) when is_binary(term) do
+    search_query = from(u in maybe_filtered_query(params), where: ilike(u.nickname, ^"%#{term}%"))
+
+    count = search_query |> Repo.aggregate(:count, :id)
+
+    results =
+      search_query
+      |> paginate(params[:page] || 1, params[:page_size] || @page_size)
+      |> Repo.all()
+
+    {:ok, results, count}
+  end
+
+  defp maybe_filtered_query(params) do
+    from(u in User, order_by: u.nickname)
+    |> User.maybe_local_user_query(params[:local])
+    |> User.maybe_external_user_query(params[:external])
+    |> User.maybe_active_user_query(params[:active])
+    |> User.maybe_deactivated_user_query(params[:deactivated])
+  end
+
+  defp paginate(query, page, page_size) do
+    from(u in query,
+      limit: ^page_size,
+      offset: ^((page - 1) * page_size)
+    )
+  end
+end

lib/pleroma/web/admin_api/views/account_view.ex

@@ -2,10 +2,11 @@
 # Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
-defmodule Pleroma.Web.MastodonAPI.Admin.AccountView do
+defmodule Pleroma.Web.AdminAPI.AccountView do
   use Pleroma.Web, :view
 
-  alias Pleroma.Web.MastodonAPI.Admin.AccountView
+  alias Pleroma.User.Info
+  alias Pleroma.Web.AdminAPI.AccountView
 
   def render("index.json", %{users: users, count: count, page_size: page_size}) do
     %{
@@ -19,7 +20,10 @@ defmodule Pleroma.Web.MastodonAPI.Admin.AccountView do
     %{
       "id" => user.id,
       "nickname" => user.nickname,
-      "deactivated" => user.info.deactivated
+      "deactivated" => user.info.deactivated,
+      "local" => user.local,
+      "roles" => Info.roles(user.info),
+      "tags" => user.tags || []
     }
   end
 end

lib/pleroma/web/auth/ldap_authenticator.ex

@@ -0,0 +1,143 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.Auth.LDAPAuthenticator do
+  alias Pleroma.User
+
+  require Logger
+
+  @behaviour Pleroma.Web.Auth.Authenticator
+
+  @connection_timeout 10_000
+  @search_timeout 10_000
+
+  def get_user(%Plug.Conn{} = conn) do
+    if Pleroma.Config.get([:ldap, :enabled]) do
+      {name, password} =
+        case conn.params do
+          %{"authorization" => %{"name" => name, "password" => password}} ->
+            {name, password}
+
+          %{"grant_type" => "password", "username" => name, "password" => password} ->
+            {name, password}
+        end
+
+      case ldap_user(name, password) do
+        %User{} = user ->
+          {:ok, user}
+
+        {:error, {:ldap_connection_error, _}} ->
+          # When LDAP is unavailable, try default authenticator
+          Pleroma.Web.Auth.PleromaAuthenticator.get_user(conn)
+
+        error ->
+          error
+      end
+    else
+      # Fall back to default authenticator
+      Pleroma.Web.Auth.PleromaAuthenticator.get_user(conn)
+    end
+  end
+
+  def handle_error(%Plug.Conn{} = _conn, error) do
+    error
+  end
+
+  def auth_template, do: nil
+
+  defp ldap_user(name, password) do
+    ldap = Pleroma.Config.get(:ldap, [])
+    host = Keyword.get(ldap, :host, "localhost")
+    port = Keyword.get(ldap, :port, 389)
+    ssl = Keyword.get(ldap, :ssl, false)
+    sslopts = Keyword.get(ldap, :sslopts, [])
+
+    options =
+      [{:port, port}, {:ssl, ssl}, {:timeout, @connection_timeout}] ++
+        if sslopts != [], do: [{:sslopts, sslopts}], else: []
+
+    case :eldap.open([to_charlist(host)], options) do
+      {:ok, connection} ->
+        try do
+          if Keyword.get(ldap, :tls, false) do
+            :application.ensure_all_started(:ssl)
+
+            case :eldap.start_tls(
+                   connection,
+                   Keyword.get(ldap, :tlsopts, []),
+                   @connection_timeout
+                 ) do
+              :ok ->
+                :ok
+
+              error ->
+                Logger.error("Could not start TLS: #{inspect(error)}")
+            end
+          end
+
+          bind_user(connection, ldap, name, password)
+        after
+          :eldap.close(connection)
+        end
+
+      {:error, error} ->
+        Logger.error("Could not open LDAP connection: #{inspect(error)}")
+        {:error, {:ldap_connection_error, error}}
+    end
+  end
+
+  defp bind_user(connection, ldap, name, password) do
+    uid = Keyword.get(ldap, :uid, "cn")
+    base = Keyword.get(ldap, :base)
+
+    case :eldap.simple_bind(connection, "#{uid}=#{name},#{base}", password) do
+      :ok ->
+        case User.get_by_nickname_or_email(name) do
+          %User{} = user ->
+            user
+
+          _ ->
+            register_user(connection, base, uid, name, password)
+        end
+
+      error ->
+        error
+    end
+  end
+
+  defp register_user(connection, base, uid, name, password) do
+    case :eldap.search(connection, [
+           {:base, to_charlist(base)},
+           {:filter, :eldap.equalityMatch(to_charlist(uid), to_charlist(name))},
+           {:scope, :eldap.wholeSubtree()},
+           {:attributes, ['mail', 'email']},
+           {:timeout, @search_timeout}
+         ]) do
+      {:ok, {:eldap_search_result, [{:eldap_entry, _, attributes}], _}} ->
+        with {_, [mail]} <- List.keyfind(attributes, 'mail', 0) do
+          params = %{
+            email: :erlang.list_to_binary(mail),
+            name: name,
+            nickname: name,
+            password: password,
+            password_confirmation: password
+          }
+
+          changeset = User.register_changeset(%User{}, params)
+
+          case User.register(changeset) do
+            {:ok, user} -> user
+            error -> error
+          end
+        else
+          _ ->
+            Logger.error("Could not find LDAP attribute mail: #{inspect(attributes)}")
+            {:error, :ldap_registration_missing_attributes}
+        end
+
+      error ->
+        error
+    end
+  end
+end

lib/pleroma/web/auth/pleroma_authenticator.ex

@@ -3,13 +3,20 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.Auth.PleromaAuthenticator do
-  alias Pleroma.User
   alias Comeonin.Pbkdf2
+  alias Pleroma.User
 
   @behaviour Pleroma.Web.Auth.Authenticator
 
   def get_user(%Plug.Conn{} = conn) do
-    %{"authorization" => %{"name" => name, "password" => password}} = conn.params
+    {name, password} =
+      case conn.params do
+        %{"authorization" => %{"name" => name, "password" => password}} ->
+          {name, password}
+
+        %{"grant_type" => "password", "username" => name, "password" => password} ->
+          {name, password}
+      end
 
     with {_, %User{} = user} <- {:user, User.get_by_nickname_or_email(name)},
          {_, true} <- {:checkpw, Pbkdf2.checkpw(password, user.password_hash)} do

lib/pleroma/web/channels/user_socket.ex

@@ -23,7 +23,7 @@ defmodule Pleroma.Web.UserSocket do
   # performing token verification on connect.
   def connect(%{"token" => token}, socket) do
     with true <- Pleroma.Config.get([:chat, :enabled]),
-         {:ok, user_id} <- Phoenix.Token.verify(socket, "user socket", token, max_age: 84600),
+         {:ok, user_id} <- Phoenix.Token.verify(socket, "user socket", token, max_age: 84_600),
          %User{} = user <- Pleroma.Repo.get(User, user_id) do
       {:ok, assign(socket, :user_name, user.nickname)}
     else

lib/pleroma/web/chat_channel.ex

@@ -4,8 +4,8 @@
 
 defmodule Pleroma.Web.ChatChannel do
   use Phoenix.Channel
-  alias Pleroma.Web.ChatChannel.ChatChannelState
   alias Pleroma.User
+  alias Pleroma.Web.ChatChannel.ChatChannelState
 
   def join("chat:public", _message, socket) do
     send(self(), :after_join)
@@ -48,7 +48,7 @@ defmodule Pleroma.Web.ChatChannel.ChatChannelState do
     end)
   end
 
-  def messages() do
+  def messages do
     Agent.get(__MODULE__, fn state -> state[:messages] |> Enum.reverse() end)
   end
 end

lib/pleroma/web/common_api/common_api.ex

@@ -3,14 +3,13 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.CommonAPI do
-  alias Pleroma.User
-  alias Pleroma.Repo
   alias Pleroma.Activity
+  alias Pleroma.Formatter
   alias Pleroma.Object
   alias Pleroma.ThreadMute
+  alias Pleroma.User
   alias Pleroma.Web.ActivityPub.ActivityPub
   alias Pleroma.Web.ActivityPub.Utils
-  alias Pleroma.Formatter
 
   import Pleroma.Web.CommonAPI.Utils
 
@@ -27,10 +26,47 @@ defmodule Pleroma.Web.CommonAPI do
     end
   end
 
+  def unfollow(follower, unfollowed) do
+    with {:ok, follower, _follow_activity} <- User.unfollow(follower, unfollowed),
+         {:ok, _activity} <- ActivityPub.unfollow(follower, unfollowed) do
+      {:ok, follower}
+    end
+  end
+
+  def accept_follow_request(follower, followed) do
+    with {:ok, follower} <- User.maybe_follow(follower, followed),
+         %Activity{} = follow_activity <- Utils.fetch_latest_follow(follower, followed),
+         {:ok, follow_activity} <- Utils.update_follow_state(follow_activity, "accept"),
+         {:ok, _activity} <-
+           ActivityPub.accept(%{
+             to: [follower.ap_id],
+             actor: followed,
+             object: follow_activity.data["id"],
+             type: "Accept"
+           }) do
+      {:ok, follower}
+    end
+  end
+
+  def reject_follow_request(follower, followed) do
+    with %Activity{} = follow_activity <- Utils.fetch_latest_follow(follower, followed),
+         {:ok, follow_activity} <- Utils.update_follow_state(follow_activity, "reject"),
+         {:ok, _activity} <-
+           ActivityPub.reject(%{
+             to: [follower.ap_id],
+             actor: followed,
+             object: follow_activity.data["id"],
+             type: "Reject"
+           }) do
+      {:ok, follower}
+    end
+  end
+
   def delete(activity_id, user) do
-    with %Activity{data: %{"object" => %{"id" => object_id}}} <- Repo.get(Activity, activity_id),
-         %Object{} = object <- Object.normalize(object_id),
-         true <- user.info.is_moderator || user.ap_id == object.data["actor"],
+    with %Activity{data: %{"object" => _}} = activity <-
+           Activity.get_by_id_with_object(activity_id),
+         %Object{} = object <- Object.normalize(activity),
+         true <- User.superuser?(user) || user.ap_id == object.data["actor"],
          {:ok, _} <- unpin(activity_id, user),
          {:ok, delete} <- ActivityPub.delete(object) do
       {:ok, delete}
@@ -39,7 +75,7 @@ defmodule Pleroma.Web.CommonAPI do
 
   def repeat(id_or_ap_id, user) do
     with %Activity{} = activity <- get_by_id_or_ap_id(id_or_ap_id),
-         object <- Object.normalize(activity.data["object"]["id"]),
+         object <- Object.normalize(activity),
          nil <- Utils.get_existing_announce(user.ap_id, object) do
       ActivityPub.announce(user, object)
     else
@@ -50,7 +86,7 @@ defmodule Pleroma.Web.CommonAPI do
 
   def unrepeat(id_or_ap_id, user) do
     with %Activity{} = activity <- get_by_id_or_ap_id(id_or_ap_id),
-         object <- Object.normalize(activity.data["object"]["id"]) do
+         object <- Object.normalize(activity) do
       ActivityPub.unannounce(user, object)
     else
       _ ->
@@ -60,7 +96,7 @@ defmodule Pleroma.Web.CommonAPI do
 
   def favorite(id_or_ap_id, user) do
     with %Activity{} = activity <- get_by_id_or_ap_id(id_or_ap_id),
-         object <- Object.normalize(activity.data["object"]["id"]),
+         object <- Object.normalize(activity),
          nil <- Utils.get_existing_like(user.ap_id, object) do
       ActivityPub.like(user, object)
     else
@@ -71,7 +107,7 @@ defmodule Pleroma.Web.CommonAPI do
 
   def unfavorite(id_or_ap_id, user) do
     with %Activity{} = activity <- get_by_id_or_ap_id(id_or_ap_id),
-         object <- Object.normalize(activity.data["object"]["id"]) do
+         object <- Object.normalize(activity) do
       ActivityPub.unlike(user, object)
     else
       _ ->
@@ -88,8 +124,8 @@ defmodule Pleroma.Web.CommonAPI do
       nil ->
         "public"
 
-      inReplyTo ->
-        Pleroma.Web.MastodonAPI.StatusView.get_visibility(inReplyTo.data["object"])
+      in_reply_to ->
+        Pleroma.Web.MastodonAPI.StatusView.get_visibility(in_reply_to.data["object"])
     end
   end
 
@@ -101,15 +137,16 @@ defmodule Pleroma.Web.CommonAPI do
 
     with status <- String.trim(status),
          attachments <- attachments_from_ids(data),
-         inReplyTo <- get_replied_to_activity(data["in_reply_to_status_id"]),
+         in_reply_to <- get_replied_to_activity(data["in_reply_to_status_id"]),
          {content_html, mentions, tags} <-
            make_content_html(
              status,
              attachments,
-             data
+             data,
+             visibility
            ),
-         {to, cc} <- to_for_user_and_mentions(user, mentions, inReplyTo, visibility),
-         context <- make_context(inReplyTo),
+         {to, cc} <- to_for_user_and_mentions(user, mentions, in_reply_to, visibility),
+         context <- make_context(in_reply_to),
          cw <- data["spoiler_text"],
          full_payload <- String.trim(status <> (data["spoiler_text"] || "")),
          length when length in 1..limit <- String.length(full_payload),
@@ -120,7 +157,7 @@ defmodule Pleroma.Web.CommonAPI do
              context,
              content_html,
              attachments,
-             inReplyTo,
+             in_reply_to,
              tags,
              cw,
              cc
@@ -248,14 +285,9 @@ defmodule Pleroma.Web.CommonAPI do
              actor: user,
              account: account,
              statuses: statuses,
-             content: content_html
+             content: content_html,
+             forward: data["forward"] || false
            }) do
-      Enum.each(User.all_superusers(), fn superuser ->
-        superuser
-        |> Pleroma.AdminEmail.report(user, account, statuses, content_html)
-        |> Pleroma.Mailer.deliver_async()
-      end)
-
       {:ok, activity}
     else
       {:error, err} -> {:error, err}
@@ -263,4 +295,24 @@ defmodule Pleroma.Web.CommonAPI do
       {:account, nil} -> {:error, "Account not found"}
     end
   end
+
+  def hide_reblogs(user, muted) do
+    ap_id = muted.ap_id
+
+    if ap_id not in user.info.muted_reblogs do
+      info_changeset = User.Info.add_reblog_mute(user.info, ap_id)
+      changeset = Ecto.Changeset.change(user) |> Ecto.Changeset.put_embed(:info, info_changeset)
+      User.update_and_set_cache(changeset)
+    end
+  end
+
+  def show_reblogs(user, muted) do
+    ap_id = muted.ap_id
+
+    if ap_id in user.info.muted_reblogs do
+      info_changeset = User.Info.remove_reblog_mute(user.info, ap_id)
+      changeset = Ecto.Changeset.change(user) |> Ecto.Changeset.put_embed(:info, info_changeset)
+      User.update_and_set_cache(changeset)
+    end
+  end
 end

lib/pleroma/web/common_api/utils.ex

@@ -6,24 +6,25 @@ defmodule Pleroma.Web.CommonAPI.Utils do
   alias Calendar.Strftime
   alias Comeonin.Pbkdf2
   alias Pleroma.Activity
+  alias Pleroma.Config
   alias Pleroma.Formatter
   alias Pleroma.Object
   alias Pleroma.Repo
   alias Pleroma.User
-  alias Pleroma.Config
+  alias Pleroma.Web.ActivityPub.Utils
   alias Pleroma.Web.Endpoint
   alias Pleroma.Web.MediaProxy
-  alias Pleroma.Web.ActivityPub.Utils
 
   # This is a hack for twidere.
   def get_by_id_or_ap_id(id) do
-    activity = Repo.get(Activity, id) || Activity.get_create_by_object_ap_id(id)
+    activity =
+      Activity.get_by_id_with_object(id) || Activity.get_create_by_object_ap_id_with_object(id)
 
     activity &&
       if activity.data["type"] == "Create" do
         activity
       else
-        Activity.get_create_by_object_ap_id(activity.data["object"])
+        Activity.get_create_by_object_ap_id_with_object(activity.data["object"])
       end
   end
 
@@ -101,7 +102,8 @@ defmodule Pleroma.Web.CommonAPI.Utils do
   def make_content_html(
         status,
         attachments,
-        data
+        data,
+        visibility
       ) do
     no_attachment_links =
       data
@@ -110,8 +112,15 @@ defmodule Pleroma.Web.CommonAPI.Utils do
 
     content_type = get_content_type(data["content_type"])
 
+    options =
+      if visibility == "direct" && Config.get([:instance, :safe_dm_mentions]) do
+        [safe_mention: true]
+      else
+        []
+      end
+
     status
-    |> format_input(content_type)
+    |> format_input(content_type, options)
     |> maybe_add_attachments(attachments, no_attachment_links)
     |> maybe_add_nsfw_tag(data)
   end
@@ -294,10 +303,10 @@ defmodule Pleroma.Web.CommonAPI.Utils do
 
   def maybe_notify_mentioned_recipients(
         recipients,
-        %Activity{data: %{"to" => _to, "type" => type} = data} = _activity
+        %Activity{data: %{"to" => _to, "type" => type} = data} = activity
       )
       when type == "Create" do
-    object = Object.normalize(data["object"])
+    object = Object.normalize(activity)
 
     object_data =
       cond do
@@ -344,4 +353,33 @@ defmodule Pleroma.Web.CommonAPI.Utils do
   end
 
   def get_report_statuses(_, _), do: {:ok, nil}
+
+  # DEPRECATED mostly, context objects are now created at insertion time.
+  def context_to_conversation_id(context) do
+    with %Object{id: id} <- Object.get_cached_by_ap_id(context) do
+      id
+    else
+      _e ->
+        changeset = Object.context_mapping(context)
+
+        case Repo.insert(changeset) do
+          {:ok, %{id: id}} ->
+            id
+
+          # This should be solved by an upsert, but it seems ecto
+          # has problems accessing the constraint inside the jsonb.
+          {:error, _} ->
+            Object.get_cached_by_ap_id(context).id
+        end
+    end
+  end
+
+  def conversation_id_to_context(id) do
+    with %Object{data: %{"id" => context}} <- Repo.get(Object, id) do
+      context
+    else
+      _e ->
+        {:error, "No such conversation"}
+    end
+  end
 end

lib/pleroma/web/controller_helper.ex

@@ -6,7 +6,8 @@ defmodule Pleroma.Web.ControllerHelper do
   use Pleroma.Web, :controller
 
   def oauth_scopes(params, default) do
-    # Note: `scopes` is used by Mastodon — supporting it but sticking to OAuth's standard `scope` wherever we control it
+    # Note: `scopes` is used by Mastodon — supporting it but sticking to
+    # OAuth's standard `scope` wherever we control it
     Pleroma.Web.OAuth.parse_scopes(params["scope"] || params["scopes"], default)
   end
 

lib/pleroma/web/endpoint.ex

@@ -25,7 +25,8 @@ defmodule Pleroma.Web.Endpoint do
     at: "/",
     from: :pleroma,
     only:
-      ~w(index.html static finmoji emoji packs sounds images instance sw.js sw-pleroma.js favicon.png schemas doc)
+      ~w(index.html robots.txt static finmoji emoji packs sounds images instance sw.js sw-pleroma.js favicon.png schemas doc)
+    # credo:disable-for-previous-line Credo.Check.Readability.MaxLineLength
   )
 
   # Code reloading can be explicitly enabled under the

lib/pleroma/web/federator/federator.ex

@@ -5,65 +5,64 @@
 defmodule Pleroma.Web.Federator do
   alias Pleroma.Activity
   alias Pleroma.User
-  alias Pleroma.Web.WebFinger
-  alias Pleroma.Web.Websub
-  alias Pleroma.Web.Salmon
   alias Pleroma.Web.ActivityPub.ActivityPub
-  alias Pleroma.Web.ActivityPub.Visibility
   alias Pleroma.Web.ActivityPub.Relay
   alias Pleroma.Web.ActivityPub.Transmogrifier
   alias Pleroma.Web.ActivityPub.Utils
+  alias Pleroma.Web.ActivityPub.Visibility
   alias Pleroma.Web.Federator.RetryQueue
   alias Pleroma.Web.OStatus
-  alias Pleroma.Jobs
+  alias Pleroma.Web.Salmon
+  alias Pleroma.Web.WebFinger
+  alias Pleroma.Web.Websub
 
   require Logger
 
   @websub Application.get_env(:pleroma, :websub)
   @ostatus Application.get_env(:pleroma, :ostatus)
 
-  def init() do
+  def init do
     # 1 minute
-    Process.sleep(1000 * 60 * 1)
+    Process.sleep(1000 * 60)
     refresh_subscriptions()
   end
 
   # Client API
 
   def incoming_doc(doc) do
-    Jobs.enqueue(:federator_incoming, __MODULE__, [:incoming_doc, doc])
+    PleromaJobQueue.enqueue(:federator_incoming, __MODULE__, [:incoming_doc, doc])
   end
 
   def incoming_ap_doc(params) do
-    Jobs.enqueue(:federator_incoming, __MODULE__, [:incoming_ap_doc, params])
+    PleromaJobQueue.enqueue(:federator_incoming, __MODULE__, [:incoming_ap_doc, params])
   end
 
   def publish(activity, priority \\ 1) do
-    Jobs.enqueue(:federator_outgoing, __MODULE__, [:publish, activity], priority)
+    PleromaJobQueue.enqueue(:federator_outgoing, __MODULE__, [:publish, activity], priority)
   end
 
   def publish_single_ap(params) do
-    Jobs.enqueue(:federator_outgoing, __MODULE__, [:publish_single_ap, params])
+    PleromaJobQueue.enqueue(:federator_outgoing, __MODULE__, [:publish_single_ap, params])
   end
 
   def publish_single_websub(websub) do
-    Jobs.enqueue(:federator_outgoing, __MODULE__, [:publish_single_websub, websub])
+    PleromaJobQueue.enqueue(:federator_outgoing, __MODULE__, [:publish_single_websub, websub])
   end
 
   def verify_websub(websub) do
-    Jobs.enqueue(:federator_outgoing, __MODULE__, [:verify_websub, websub])
+    PleromaJobQueue.enqueue(:federator_outgoing, __MODULE__, [:verify_websub, websub])
   end
 
   def request_subscription(sub) do
-    Jobs.enqueue(:federator_outgoing, __MODULE__, [:request_subscription, sub])
+    PleromaJobQueue.enqueue(:federator_outgoing, __MODULE__, [:request_subscription, sub])
   end
 
-  def refresh_subscriptions() do
-    Jobs.enqueue(:federator_outgoing, __MODULE__, [:refresh_subscriptions])
+  def refresh_subscriptions do
+    PleromaJobQueue.enqueue(:federator_outgoing, __MODULE__, [:refresh_subscriptions])
   end
 
   def publish_single_salmon(params) do
-    Jobs.enqueue(:federator_outgoing, __MODULE__, [:publish_single_salmon, params])
+    PleromaJobQueue.enqueue(:federator_outgoing, __MODULE__, [:publish_single_salmon, params])
   end
 
   # Job Worker Callbacks

lib/pleroma/web/federator/retry_queue.ex

@@ -13,7 +13,7 @@ defmodule Pleroma.Web.Federator.RetryQueue do
     {:ok, %{args | queue_table: queue_table, running_jobs: :sets.new()}}
   end
 
-  def start_link() do
+  def start_link do
     enabled =
       if Mix.env() == :test, do: true, else: Pleroma.Config.get([__MODULE__, :enabled], false)
 
@@ -39,11 +39,11 @@ defmodule Pleroma.Web.Federator.RetryQueue do
     GenServer.cast(__MODULE__, {:maybe_enqueue, data, transport, retries + 1})
   end
 
-  def get_stats() do
+  def get_stats do
     GenServer.call(__MODULE__, :get_stats)
   end
 
-  def reset_stats() do
+  def reset_stats do
     GenServer.call(__MODULE__, :reset_stats)
   end
 
@@ -55,7 +55,7 @@ defmodule Pleroma.Web.Federator.RetryQueue do
     end
   end
 
-  def get_retry_timer_interval() do
+  def get_retry_timer_interval do
     Pleroma.Config.get([:retry_queue, :interval], 1000)
   end
 
@@ -231,7 +231,7 @@ defmodule Pleroma.Web.Federator.RetryQueue do
     end
   end
 
-  defp maybe_kickoff_timer() do
+  defp maybe_kickoff_timer do
     GenServer.cast(__MODULE__, :kickoff_timer)
   end
 end

lib/pleroma/web/mastodon_api/mastodon_api.ex

@@ -1 +1,51 @@
+defmodule Pleroma.Web.MastodonAPI.MastodonAPI do
+  import Ecto.Query
+  import Ecto.Changeset
 
+  alias Pleroma.Activity
+  alias Pleroma.Notification
+  alias Pleroma.Pagination
+  alias Pleroma.User
+
+  def get_followers(user, params \\ %{}) do
+    user
+    |> User.get_followers_query()
+    |> Pagination.fetch_paginated(params)
+  end
+
+  def get_friends(user, params \\ %{}) do
+    user
+    |> User.get_friends_query()
+    |> Pagination.fetch_paginated(params)
+  end
+
+  def get_notifications(user, params \\ %{}) do
+    options = cast_params(params)
+
+    user
+    |> Notification.for_user_query()
+    |> restrict(:exclude_types, options)
+    |> Pagination.fetch_paginated(params)
+  end
+
+  defp cast_params(params) do
+    param_types = %{
+      exclude_types: {:array, :string}
+    }
+
+    changeset = cast({%{}, param_types}, params, Map.keys(param_types))
+    changeset.changes
+  end
+
+  defp restrict(query, :exclude_types, %{exclude_types: mastodon_types = [_ | _]}) do
+    ap_types =
+      mastodon_types
+      |> Enum.map(&Activity.from_mastodon_notification_type/1)
+      |> Enum.filter(& &1)
+
+    query
+    |> where([q, a], not fragment("? @> ARRAY[?->>'type']::varchar[]", ^ap_types, a.data))
+  end
+
+  defp restrict(query, _, _), do: query
+end

lib/pleroma/web/mastodon_api/mastodon_api_controller.ex

@@ -4,6 +4,7 @@
 
 defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
   use Pleroma.Web, :controller
+
   alias Pleroma.Activity
   alias Pleroma.Config
   alias Pleroma.Filter
@@ -13,21 +14,19 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
   alias Pleroma.Stats
   alias Pleroma.User
   alias Pleroma.Web
+  alias Pleroma.Web.ActivityPub.ActivityPub
+  alias Pleroma.Web.ActivityPub.Visibility
   alias Pleroma.Web.CommonAPI
-  alias Pleroma.Web.MediaProxy
-  alias Pleroma.Web.Push
-  alias Push.Subscription
-
   alias Pleroma.Web.MastodonAPI.AccountView
+  alias Pleroma.Web.MastodonAPI.AppView
   alias Pleroma.Web.MastodonAPI.FilterView
   alias Pleroma.Web.MastodonAPI.ListView
+  alias Pleroma.Web.MastodonAPI.MastodonAPI
   alias Pleroma.Web.MastodonAPI.MastodonView
-  alias Pleroma.Web.MastodonAPI.PushSubscriptionView
-  alias Pleroma.Web.MastodonAPI.StatusView
+  alias Pleroma.Web.MastodonAPI.NotificationView
   alias Pleroma.Web.MastodonAPI.ReportView
-  alias Pleroma.Web.ActivityPub.ActivityPub
-  alias Pleroma.Web.ActivityPub.Utils
-  alias Pleroma.Web.ActivityPub.Visibility
+  alias Pleroma.Web.MastodonAPI.StatusView
+  alias Pleroma.Web.MediaProxy
   alias Pleroma.Web.OAuth.App
   alias Pleroma.Web.OAuth.Authorization
   alias Pleroma.Web.OAuth.Token
@@ -53,16 +52,9 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
     with cs <- App.register_changeset(%App{}, app_attrs),
          false <- cs.changes[:client_name] == @local_mastodon_name,
          {:ok, app} <- Repo.insert(cs) do
-      res = %{
-        id: app.id |> to_string,
-        name: app.client_name,
-        client_id: app.client_id,
-        client_secret: app.client_secret,
-        redirect_uri: app.redirect_uris,
-        website: app.website
-      }
-
-      json(conn, res)
+      conn
+      |> put_view(AppView)
+      |> render("show.json", %{app: app})
     end
   end
 
@@ -134,8 +126,16 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
     json(conn, account)
   end
 
-  def user(%{assigns: %{user: for_user}} = conn, %{"id" => id}) do
-    with %User{} = user <- Repo.get(User, id),
+  def verify_app_credentials(%{assigns: %{user: _user, token: token}} = conn, _) do
+    with %Token{app: %App{} = app} <- Repo.preload(token, :app) do
+      conn
+      |> put_view(AppView)
+      |> render("short.json", %{app: app})
+    end
+  end
+
+  def user(%{assigns: %{user: for_user}} = conn, %{"id" => nickname_or_id}) do
+    with %User{} = user <- User.get_cached_by_nickname_or_id(nickname_or_id),
          true <- User.auth_active?(user) || user.id == for_user.id || User.superuser?(for_user) do
       account = AccountView.render("account.json", %{user: user, for: for_user})
       json(conn, account)
@@ -163,6 +163,9 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
       },
       stats: Stats.get_stats(),
       thumbnail: Web.base_url() <> "/instance/thumbnail.jpeg",
+      languages: ["en"],
+      registrations: Pleroma.Config.get([:instance, :registrations_open]),
+      # Extra (not present in Mastodon):
       max_toot_chars: Keyword.get(instance, :limit)
     }
 
@@ -193,6 +196,11 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
   end
 
   defp add_link_headers(conn, method, activities, param \\ nil, params \\ %{}) do
+    params =
+      conn.params
+      |> Map.drop(["since_id", "max_id"])
+      |> Map.merge(params)
+
     last = List.last(activities)
     first = List.first(activities)
 
@@ -300,7 +308,8 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
       |> Map.put(:visibility, "direct")
 
     activities =
-      ActivityPub.fetch_activities_query([user.ap_id], params)
+      [user.ap_id]
+      |> ActivityPub.fetch_activities_query(params)
       |> Repo.all()
 
     conn
@@ -498,21 +507,19 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
   end
 
   def notifications(%{assigns: %{user: user}} = conn, params) do
-    notifications = Notification.for_user(user, params)
-
-    result =
-      notifications
-      |> Enum.map(fn x -> render_notification(user, x) end)
-      |> Enum.filter(& &1)
+    notifications = MastodonAPI.get_notifications(user, params)
 
     conn
     |> add_link_headers(:notifications, notifications)
-    |> json(result)
+    |> put_view(NotificationView)
+    |> render("index.json", %{notifications: notifications, for: user})
   end
 
   def get_notification(%{assigns: %{user: user}} = conn, %{"id" => id} = _params) do
     with {:ok, notification} <- Notification.get(user, id) do
-      json(conn, render_notification(user, notification))
+      conn
+      |> put_view(NotificationView)
+      |> render("show.json", %{notification: notification, for: user})
     else
       {:error, reason} ->
         conn
@@ -649,9 +656,9 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
     |> render("index.json", %{activities: activities, for: user, as: :activity})
   end
 
-  def followers(%{assigns: %{user: for_user}} = conn, %{"id" => id}) do
+  def followers(%{assigns: %{user: for_user}} = conn, %{"id" => id} = params) do
     with %User{} = user <- Repo.get(User, id),
-         {:ok, followers} <- User.get_followers(user) do
+         followers <- MastodonAPI.get_followers(user, params) do
       followers =
         cond do
           for_user && user.id == for_user.id -> followers
@@ -660,14 +667,15 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
         end
 
       conn
+      |> add_link_headers(:followers, followers, user)
       |> put_view(AccountView)
       |> render("accounts.json", %{users: followers, as: :user})
     end
   end
 
-  def following(%{assigns: %{user: for_user}} = conn, %{"id" => id}) do
+  def following(%{assigns: %{user: for_user}} = conn, %{"id" => id} = params) do
     with %User{} = user <- Repo.get(User, id),
-         {:ok, followers} <- User.get_friends(user) do
+         followers <- MastodonAPI.get_friends(user, params) do
       followers =
         cond do
           for_user && user.id == for_user.id -> followers
@@ -676,6 +684,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
         end
 
       conn
+      |> add_link_headers(:following, followers, user)
       |> put_view(AccountView)
       |> render("accounts.json", %{users: followers, as: :user})
     end
@@ -691,16 +700,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
 
   def authorize_follow_request(%{assigns: %{user: followed}} = conn, %{"id" => id}) do
     with %User{} = follower <- Repo.get(User, id),
-         {:ok, follower} <- User.maybe_follow(follower, followed),
-         %Activity{} = follow_activity <- Utils.fetch_latest_follow(follower, followed),
-         {:ok, follow_activity} <- Utils.update_follow_state(follow_activity, "accept"),
-         {:ok, _activity} <-
-           ActivityPub.accept(%{
-             to: [follower.ap_id],
-             actor: followed,
-             object: follow_activity.data["id"],
-             type: "Accept"
-           }) do
+         {:ok, follower} <- CommonAPI.accept_follow_request(follower, followed) do
       conn
       |> put_view(AccountView)
       |> render("relationship.json", %{user: followed, target: follower})
@@ -714,15 +714,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
 
   def reject_follow_request(%{assigns: %{user: followed}} = conn, %{"id" => id}) do
     with %User{} = follower <- Repo.get(User, id),
-         %Activity{} = follow_activity <- Utils.fetch_latest_follow(follower, followed),
-         {:ok, follow_activity} <- Utils.update_follow_state(follow_activity, "reject"),
-         {:ok, _activity} <-
-           ActivityPub.reject(%{
-             to: [follower.ap_id],
-             actor: followed,
-             object: follow_activity.data["id"],
-             type: "Reject"
-           }) do
+         {:ok, follower} <- CommonAPI.reject_follow_request(follower, followed) do
       conn
       |> put_view(AccountView)
       |> render("relationship.json", %{user: followed, target: follower})
@@ -736,11 +728,25 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
 
   def follow(%{assigns: %{user: follower}} = conn, %{"id" => id}) do
     with %User{} = followed <- Repo.get(User, id),
+         false <- User.following?(follower, followed),
          {:ok, follower, followed, _} <- CommonAPI.follow(follower, followed) do
       conn
       |> put_view(AccountView)
       |> render("relationship.json", %{user: follower, target: followed})
     else
+      true ->
+        followed = User.get_cached_by_id(id)
+
+        {:ok, follower} =
+          case conn.params["reblogs"] do
+            true -> CommonAPI.show_reblogs(follower, followed)
+            false -> CommonAPI.hide_reblogs(follower, followed)
+          end
+
+        conn
+        |> put_view(AccountView)
+        |> render("relationship.json", %{user: follower, target: followed})
+
       {:error, message} ->
         conn
         |> put_resp_content_type("application/json")
@@ -764,8 +770,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
 
   def unfollow(%{assigns: %{user: follower}} = conn, %{"id" => id}) do
     with %User{} = followed <- Repo.get(User, id),
-         {:ok, _activity} <- ActivityPub.unfollow(follower, followed),
-         {:ok, follower, _} <- User.unfollow(follower, followed) do
+         {:ok, follower} <- CommonAPI.unfollow(follower, followed) do
       conn
       |> put_view(AccountView)
       |> render("relationship.json", %{user: follower, target: followed})
@@ -944,12 +949,14 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
   end
 
   def favourites(%{assigns: %{user: user}} = conn, params) do
-    activities =
+    params =
       params
       |> Map.put("type", "Create")
       |> Map.put("favorited_by", user.ap_id)
       |> Map.put("blocking_user", user)
-      |> ActivityPub.fetch_public_activities()
+
+    activities =
+      ActivityPub.fetch_activities([], params)
       |> Enum.reverse()
 
     conn
@@ -1123,7 +1130,8 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
           compose: %{
             me: "#{user.id}",
             default_privacy: user.info.default_scope,
-            default_sensitive: false
+            default_sensitive: false,
+            allow_content_types: Config.get([:instance, :allowed_post_formats])
           },
           media_attachments: %{
             accept_content_types: [
@@ -1268,7 +1276,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
     end
   end
 
-  defp get_or_make_app() do
+  defp get_or_make_app do
     find_attrs = %{client_name: @local_mastodon_name, redirect_uris: "."}
     scopes = ["read", "write", "follow", "push"]
 
@@ -1321,45 +1329,6 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
     json(conn, %{})
   end
 
-  def render_notification(user, %{id: id, activity: activity, inserted_at: created_at} = _params) do
-    actor = User.get_cached_by_ap_id(activity.data["actor"])
-    parent_activity = Activity.get_create_by_object_ap_id(activity.data["object"])
-    mastodon_type = Activity.mastodon_notification_type(activity)
-
-    response = %{
-      id: to_string(id),
-      type: mastodon_type,
-      created_at: CommonAPI.Utils.to_masto_date(created_at),
-      account: AccountView.render("account.json", %{user: actor, for: user})
-    }
-
-    case mastodon_type do
-      "mention" ->
-        response
-        |> Map.merge(%{
-          status: StatusView.render("status.json", %{activity: activity, for: user})
-        })
-
-      "favourite" ->
-        response
-        |> Map.merge(%{
-          status: StatusView.render("status.json", %{activity: parent_activity, for: user})
-        })
-
-      "reblog" ->
-        response
-        |> Map.merge(%{
-          status: StatusView.render("status.json", %{activity: parent_activity, for: user})
-        })
-
-      "follow" ->
-        response
-
-      _ ->
-        nil
-    end
-  end
-
   def get_filters(%{assigns: %{user: user}} = conn, _) do
     filters = Filter.get_filters(user)
     res = FilterView.render("filters.json", filters: filters)
@@ -1419,37 +1388,8 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
     json(conn, %{})
   end
 
-  def create_push_subscription(%{assigns: %{user: user, token: token}} = conn, params) do
-    true = Push.enabled()
-    Subscription.delete_if_exists(user, token)
-    {:ok, subscription} = Subscription.create(user, token, params)
-    view = PushSubscriptionView.render("push_subscription.json", subscription: subscription)
-    json(conn, view)
-  end
-
-  def get_push_subscription(%{assigns: %{user: user, token: token}} = conn, _params) do
-    true = Push.enabled()
-    subscription = Subscription.get(user, token)
-    view = PushSubscriptionView.render("push_subscription.json", subscription: subscription)
-    json(conn, view)
-  end
-
-  def update_push_subscription(
-        %{assigns: %{user: user, token: token}} = conn,
-        params
-      ) do
-    true = Push.enabled()
-    {:ok, subscription} = Subscription.update(user, token, params)
-    view = PushSubscriptionView.render("push_subscription.json", subscription: subscription)
-    json(conn, view)
-  end
-
-  def delete_push_subscription(%{assigns: %{user: user, token: token}} = conn, _params) do
-    true = Push.enabled()
-    {:ok, _response} = Subscription.delete(user, token)
-    json(conn, %{})
-  end
-
+  # fallback action
+  #
   def errors(conn, _) do
     conn
     |> put_status(500)
@@ -1478,7 +1418,6 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
                url,
                [],
                adapter: [
-                 timeout: timeout,
                  recv_timeout: timeout,
                  pool: :default
                ]

lib/pleroma/web/mastodon_api/subscription_controller.ex

@@ -0,0 +1,71 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.MastodonAPI.SubscriptionController do
+  @moduledoc "The module represents functions to manage user subscriptions."
+  use Pleroma.Web, :controller
+
+  alias Pleroma.Web.Push
+  alias Pleroma.Web.Push.Subscription
+  alias Pleroma.Web.MastodonAPI.PushSubscriptionView, as: View
+
+  action_fallback(:errors)
+
+  # Creates PushSubscription
+  # POST /api/v1/push/subscription
+  #
+  def create(%{assigns: %{user: user, token: token}} = conn, params) do
+    with true <- Push.enabled(),
+         {:ok, _} <- Subscription.delete_if_exists(user, token),
+         {:ok, subscription} <- Subscription.create(user, token, params) do
+      view = View.render("push_subscription.json", subscription: subscription)
+      json(conn, view)
+    end
+  end
+
+  # Gets PushSubscription
+  # GET /api/v1/push/subscription
+  #
+  def get(%{assigns: %{user: user, token: token}} = conn, _params) do
+    with true <- Push.enabled(),
+         {:ok, subscription} <- Subscription.get(user, token) do
+      view = View.render("push_subscription.json", subscription: subscription)
+      json(conn, view)
+    end
+  end
+
+  # Updates PushSubscription
+  # PUT /api/v1/push/subscription
+  #
+  def update(%{assigns: %{user: user, token: token}} = conn, params) do
+    with true <- Push.enabled(),
+         {:ok, subscription} <- Subscription.update(user, token, params) do
+      view = View.render("push_subscription.json", subscription: subscription)
+      json(conn, view)
+    end
+  end
+
+  # Deletes PushSubscription
+  # DELETE /api/v1/push/subscription
+  #
+  def delete(%{assigns: %{user: user, token: token}} = conn, _params) do
+    with true <- Push.enabled(),
+         {:ok, _response} <- Subscription.delete(user, token),
+         do: json(conn, %{})
+  end
+
+  # fallback action
+  #
+  def errors(conn, {:error, :not_found}) do
+    conn
+    |> put_status(404)
+    |> json("Not found")
+  end
+
+  def errors(conn, _) do
+    conn
+    |> put_status(500)
+    |> json("Something went wrong")
+  end
+end

lib/pleroma/web/mastodon_api/views/account_view.ex

@@ -55,7 +55,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do
       muting_notifications: false,
       requested: requested,
       domain_blocking: false,
-      showing_reblogs: false,
+      showing_reblogs: User.showing_reblogs?(user, target),
       endorsed: false
     }
   end

lib/pleroma/web/mastodon_api/views/app_view.ex

@@ -0,0 +1,41 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.MastodonAPI.AppView do
+  use Pleroma.Web, :view
+
+  alias Pleroma.Web.OAuth.App
+
+  @vapid_key :web_push_encryption
+             |> Application.get_env(:vapid_details, [])
+             |> Keyword.get(:public_key)
+
+  def render("show.json", %{app: %App{} = app}) do
+    %{
+      id: app.id |> to_string,
+      name: app.client_name,
+      client_id: app.client_id,
+      client_secret: app.client_secret,
+      redirect_uri: app.redirect_uris,
+      website: app.website
+    }
+    |> with_vapid_key()
+  end
+
+  def render("short.json", %{app: %App{website: webiste, client_name: name}}) do
+    %{
+      name: name,
+      website: webiste
+    }
+    |> with_vapid_key()
+  end
+
+  defp with_vapid_key(data) do
+    if @vapid_key do
+      Map.put(data, "vapid_key", @vapid_key)
+    else
+      data
+    end
+  end
+end

lib/pleroma/web/mastodon_api/views/notification_view.ex

@@ -0,0 +1,64 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.MastodonAPI.NotificationView do
+  use Pleroma.Web, :view
+
+  alias Pleroma.Activity
+  alias Pleroma.Notification
+  alias Pleroma.User
+  alias Pleroma.Web.CommonAPI
+  alias Pleroma.Web.MastodonAPI.AccountView
+  alias Pleroma.Web.MastodonAPI.NotificationView
+  alias Pleroma.Web.MastodonAPI.StatusView
+
+  def render("index.json", %{notifications: notifications, for: user}) do
+    render_many(notifications, NotificationView, "show.json", %{for: user})
+  end
+
+  def render("show.json", %{
+        notification: %Notification{activity: activity} = notification,
+        for: user
+      }) do
+    actor = User.get_cached_by_ap_id(activity.data["actor"])
+    parent_activity = Activity.get_create_by_object_ap_id(activity.data["object"])
+    mastodon_type = Activity.mastodon_notification_type(activity)
+
+    response = %{
+      id: to_string(notification.id),
+      type: mastodon_type,
+      created_at: CommonAPI.Utils.to_masto_date(notification.inserted_at),
+      account: AccountView.render("account.json", %{user: actor, for: user}),
+      pleroma: %{
+        is_seen: notification.seen
+      }
+    }
+
+    case mastodon_type do
+      "mention" ->
+        response
+        |> Map.merge(%{
+          status: StatusView.render("status.json", %{activity: activity, for: user})
+        })
+
+      "favourite" ->
+        response
+        |> Map.merge(%{
+          status: StatusView.render("status.json", %{activity: parent_activity, for: user})
+        })
+
+      "reblog" ->
+        response
+        |> Map.merge(%{
+          status: StatusView.render("status.json", %{activity: parent_activity, for: user})
+        })
+
+      "follow" ->
+        response
+
+      _ ->
+        nil
+    end
+  end
+end

lib/pleroma/web/mastodon_api/views/push_subscription_view.ex

@@ -4,6 +4,7 @@
 
 defmodule Pleroma.Web.MastodonAPI.PushSubscriptionView do
   use Pleroma.Web, :view
+  alias Pleroma.Web.Push
 
   def render("push_subscription.json", %{subscription: subscription}) do
     %{
@@ -14,7 +15,5 @@ defmodule Pleroma.Web.MastodonAPI.PushSubscriptionView do
     }
   end
 
-  defp server_key do
-    Keyword.get(Application.get_env(:web_push_encryption, :vapid_details), :public_key)
-  end
+  defp server_key, do: Keyword.get(Push.vapid_config(), :public_key)
 end

lib/pleroma/web/mastodon_api/views/status_view.ex

@@ -46,6 +46,14 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
     end
   end
 
+  defp get_context_id(%{data: %{"context_id" => context_id}}) when not is_nil(context_id),
+    do: context_id
+
+  defp get_context_id(%{data: %{"context" => context}}) when is_binary(context),
+    do: Utils.context_to_conversation_id(context)
+
+  defp get_context_id(_), do: nil
+
   def render("index.json", opts) do
     replied_to_activities = get_replied_to_activities(opts.activities)
 
@@ -102,7 +110,10 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
         website: nil
       },
       language: nil,
-      emojis: []
+      emojis: [],
+      pleroma: %{
+        local: activity.local
+      }
     }
   end
 
@@ -163,7 +174,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
       content: content,
       created_at: created_at,
       reblogs_count: announcement_count,
-      replies_count: 0,
+      replies_count: object["repliesCount"] || 0,
       favourites_count: like_count,
       reblogged: present?(repeated),
       favourited: present?(favorited),
@@ -181,7 +192,11 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
         website: nil
       },
       language: nil,
-      emojis: build_emojis(activity.data["object"]["emoji"])
+      emojis: build_emojis(activity.data["object"]["emoji"]),
+      pleroma: %{
+        local: activity.local,
+        conversation_id: get_context_id(activity)
+      }
     }
   end
 
@@ -251,7 +266,8 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
       preview_url: href,
       text_url: href,
       type: type,
-      description: attachment["name"]
+      description: attachment["name"],
+      pleroma: %{mime_type: media_type}
     }
   end
 

lib/pleroma/web/mastodon_api/websocket_handler.ex

@@ -5,9 +5,9 @@
 defmodule Pleroma.Web.MastodonAPI.WebsocketHandler do
   require Logger
 
-  alias Pleroma.Web.OAuth.Token
   alias Pleroma.Repo
   alias Pleroma.User
+  alias Pleroma.Web.OAuth.Token
 
   @behaviour :cowboy_websocket
 

lib/pleroma/web/media_proxy/media_proxy.ex

@@ -19,7 +19,8 @@ defmodule Pleroma.Web.MediaProxy do
     else
       secret = Application.get_env(:pleroma, Pleroma.Web.Endpoint)[:secret_key_base]
 
-      # Must preserve `%2F` for compatibility with S3 (https://git.pleroma.social/pleroma/pleroma/issues/580)
+      # Must preserve `%2F` for compatibility with S3
+      # https://git.pleroma.social/pleroma/pleroma/issues/580
       replacement = get_replacement(url, ":2F:")
 
       # The URL is url-decoded and encoded again to ensure it is correctly encoded and not twice.

lib/pleroma/web/metadata/opengraph.ex

@@ -88,7 +88,7 @@ defmodule Pleroma.Web.Metadata.Providers.OpenGraph do
 
           # TODO: Add additional properties to objects when we have the data available.
           # Also, Whatsapp only wants JPEG or PNGs. It seems that if we add a second og:image
-          # object when a Video or GIF is attached it will display that in the Whatsapp Rich Preview.
+          # object when a Video or GIF is attached it will display that in Whatsapp Rich Preview.
           case media_type do
             "audio" ->
               [

lib/pleroma/web/metadata/twitter_card.ex

@@ -97,7 +97,8 @@ defmodule Pleroma.Web.Metadata.Providers.TwitterCard do
                 | acc
               ]
 
-            # TODO: Need the true width and height values here or Twitter renders an iFrame with a bad aspect ratio
+            # TODO: Need the true width and height values here or Twitter renders an iFrame with
+            # a bad aspect ratio
             "video" ->
               [
                 {:meta, [property: "twitter:card", content: "player"], []},

lib/pleroma/web/metadata/utils.ex

@@ -1,10 +1,10 @@
 # Pleroma: A lightweight social networking server
-# Copyright \xc2\xa9 2017-2019 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.Metadata.Utils do
-  alias Pleroma.HTML
   alias Pleroma.Formatter
+  alias Pleroma.HTML
   alias Pleroma.Web.MediaProxy
 
   def scrub_html_and_truncate(%{data: %{"content" => content}} = object) do
@@ -17,14 +17,14 @@ defmodule Pleroma.Web.Metadata.Utils do
     |> Formatter.truncate()
   end
 
-  def scrub_html_and_truncate(content) when is_binary(content) do
+  def scrub_html_and_truncate(content, max_length \\ 200) when is_binary(content) do
     content
     # html content comes from DB already encoded, decode first and scrub after
     |> HtmlEntities.decode()
     |> String.replace(~r/<br\s?\/?>/, " ")
     |> HTML.strip_tags()
     |> Formatter.demojify()
-    |> Formatter.truncate()
+    |> Formatter.truncate(max_length)
   end
 
   def attachment_url(url) do

lib/pleroma/web/nodeinfo/nodeinfo.ex

@@ -1 +0,0 @@
-

lib/pleroma/web/nodeinfo/nodeinfo_controller.ex

@@ -6,7 +6,6 @@ defmodule Pleroma.Web.Nodeinfo.NodeinfoController do
   use Pleroma.Web, :controller
 
   alias Pleroma.Config
-  alias Pleroma.Repo
   alias Pleroma.Stats
   alias Pleroma.User
   alias Pleroma.Web
@@ -86,8 +85,7 @@ defmodule Pleroma.Web.Nodeinfo.NodeinfoController do
       end
 
     staff_accounts =
-      User.moderator_user_query()
-      |> Repo.all()
+      User.all_superusers()
       |> Enum.map(fn u -> u.ap_id end)
 
     mrf_user_allowlist =
@@ -126,6 +124,9 @@ defmodule Pleroma.Web.Nodeinfo.NodeinfoController do
         end,
         if Keyword.get(instance, :allow_relay) do
           "relay"
+        end,
+        if Keyword.get(instance, :safe_dm_mentions) do
+          "safe_dm_mentions"
         end
       ]
       |> Enum.filter(& &1)

lib/pleroma/web/oauth/authorization.ex

@@ -5,10 +5,10 @@
 defmodule Pleroma.Web.OAuth.Authorization do
   use Ecto.Schema
 
-  alias Pleroma.User
   alias Pleroma.Repo
-  alias Pleroma.Web.OAuth.Authorization
+  alias Pleroma.User
   alias Pleroma.Web.OAuth.App
+  alias Pleroma.Web.OAuth.Authorization
 
   import Ecto.Changeset
   import Ecto.Query
@@ -16,7 +16,7 @@ defmodule Pleroma.Web.OAuth.Authorization do
   schema "oauth_authorizations" do
     field(:token, :string)
     field(:scopes, {:array, :string}, default: [])
-    field(:valid_until, :naive_datetime)
+    field(:valid_until, :naive_datetime_usec)
     field(:used, :boolean, default: false)
     belongs_to(:user, Pleroma.User, type: Pleroma.FlakeId)
     belongs_to(:app, App)

lib/pleroma/web/oauth/oauth_controller.ex

@@ -5,13 +5,12 @@
 defmodule Pleroma.Web.OAuth.OAuthController do
   use Pleroma.Web, :controller
 
-  alias Pleroma.Web.Auth.Authenticator
-  alias Pleroma.Web.OAuth.Authorization
-  alias Pleroma.Web.OAuth.Token
-  alias Pleroma.Web.OAuth.App
   alias Pleroma.Repo
   alias Pleroma.User
-  alias Comeonin.Pbkdf2
+  alias Pleroma.Web.Auth.Authenticator
+  alias Pleroma.Web.OAuth.App
+  alias Pleroma.Web.OAuth.Authorization
+  alias Pleroma.Web.OAuth.Token
 
   import Pleroma.Web.ControllerHelper, only: [oauth_scopes: 2]
 
@@ -84,14 +83,18 @@ defmodule Pleroma.Web.OAuth.OAuthController do
       end
     else
       {scopes_issue, _} when scopes_issue in [:unsupported_scopes, :missing_scopes] ->
+        # Per https://github.com/tootsuite/mastodon/blob/
+        #   51e154f5e87968d6bb115e053689767ab33e80cd/app/controllers/api/base_controller.rb#L39
         conn
-        |> put_flash(:error, "Permissions not specified.")
+        |> put_flash(:error, "This action is outside the authorized scopes")
         |> put_status(:unauthorized)
         |> authorize(auth_params)
 
       {:auth_active, false} ->
+        # Per https://github.com/tootsuite/mastodon/blob/
+        #   51e154f5e87968d6bb115e053689767ab33e80cd/app/controllers/api/base_controller.rb#L76
         conn
-        |> put_flash(:error, "Account confirmation pending.")
+        |> put_flash(:error, "Your login is missing a confirmed e-mail address")
         |> put_status(:forbidden)
         |> authorize(auth_params)
 
@@ -105,6 +108,7 @@ defmodule Pleroma.Web.OAuth.OAuthController do
          fixed_token = fix_padding(params["code"]),
          %Authorization{} = auth <-
            Repo.get_by(Authorization, token: fixed_token, app_id: app.id),
+         %User{} = user <- Repo.get(User, auth.user_id),
          {:ok, token} <- Token.exchange_token(app, auth),
          {:ok, inserted_at} <- DateTime.from_naive(token.inserted_at, "Etc/UTC") do
       response = %{
@@ -113,7 +117,8 @@ defmodule Pleroma.Web.OAuth.OAuthController do
         refresh_token: token.refresh_token,
         created_at: DateTime.to_unix(inserted_at),
         expires_in: 60 * 10,
-        scope: Enum.join(token.scopes, " ")
+        scope: Enum.join(token.scopes, " "),
+        me: user.ap_id
       }
 
       json(conn, response)
@@ -126,11 +131,10 @@ defmodule Pleroma.Web.OAuth.OAuthController do
 
   def token_exchange(
         conn,
-        %{"grant_type" => "password", "username" => name, "password" => password} = params
+        %{"grant_type" => "password"} = params
       ) do
-    with %App{} = app <- get_app_from_request(conn, params),
-         %User{} = user <- User.get_by_nickname_or_email(name),
-         true <- Pbkdf2.checkpw(password, user.password_hash),
+    with {_, {:ok, %User{} = user}} <- {:get_user, Authenticator.get_user(conn)},
+         %App{} = app <- get_app_from_request(conn, params),
          {:auth_active, true} <- {:auth_active, User.auth_active?(user)},
          scopes <- oauth_scopes(params, app.scopes),
          [] <- scopes -- app.scopes,
@@ -142,15 +146,18 @@ defmodule Pleroma.Web.OAuth.OAuthController do
         access_token: token.token,
         refresh_token: token.refresh_token,
         expires_in: 60 * 10,
-        scope: Enum.join(token.scopes, " ")
+        scope: Enum.join(token.scopes, " "),
+        me: user.ap_id
       }
 
       json(conn, response)
     else
       {:auth_active, false} ->
+        # Per https://github.com/tootsuite/mastodon/blob/
+        #   51e154f5e87968d6bb115e053689767ab33e80cd/app/controllers/api/base_controller.rb#L76
         conn
         |> put_status(:forbidden)
-        |> json(%{error: "Account confirmation pending"})
+        |> json(%{error: "Your login is missing a confirmed e-mail address"})
 
       _error ->
         put_status(conn, 400)

lib/pleroma/web/oauth/token.ex

@@ -7,17 +7,17 @@ defmodule Pleroma.Web.OAuth.Token do
 
   import Ecto.Query
 
-  alias Pleroma.User
   alias Pleroma.Repo
-  alias Pleroma.Web.OAuth.Token
+  alias Pleroma.User
   alias Pleroma.Web.OAuth.App
   alias Pleroma.Web.OAuth.Authorization
+  alias Pleroma.Web.OAuth.Token
 
   schema "oauth_tokens" do
     field(:token, :string)
     field(:refresh_token, :string)
     field(:scopes, {:array, :string}, default: [])
-    field(:valid_until, :naive_datetime)
+    field(:valid_until, :naive_datetime_usec)
     belongs_to(:user, Pleroma.User, type: Pleroma.FlakeId)
     belongs_to(:app, App)
 

lib/pleroma/web/ostatus/activity_representer.ex

@@ -4,8 +4,8 @@
 
 defmodule Pleroma.Web.OStatus.ActivityRepresenter do
   alias Pleroma.Activity
-  alias Pleroma.User
   alias Pleroma.Object
+  alias Pleroma.User
   alias Pleroma.Web.OStatus.UserRepresenter
 
   require Logger

lib/pleroma/web/ostatus/feed_representer.ex

@@ -4,8 +4,8 @@
 
 defmodule Pleroma.Web.OStatus.FeedRepresenter do
   alias Pleroma.User
-  alias Pleroma.Web.OStatus
   alias Pleroma.Web.MediaProxy
+  alias Pleroma.Web.OStatus
   alias Pleroma.Web.OStatus.ActivityRepresenter
   alias Pleroma.Web.OStatus.UserRepresenter
 

lib/pleroma/web/ostatus/handlers/delete_handler.ex

@@ -4,9 +4,9 @@
 
 defmodule Pleroma.Web.OStatus.DeleteHandler do
   require Logger
-  alias Pleroma.Web.XML
   alias Pleroma.Object
   alias Pleroma.Web.ActivityPub.ActivityPub
+  alias Pleroma.Web.XML
 
   def handle_delete(entry, _doc \\ nil) do
     with id <- XML.string_from_xpath("//id", entry),

lib/pleroma/web/ostatus/handlers/follow_handler.ex

@@ -3,10 +3,10 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.OStatus.FollowHandler do
-  alias Pleroma.Web.XML
-  alias Pleroma.Web.OStatus
-  alias Pleroma.Web.ActivityPub.ActivityPub
   alias Pleroma.User
+  alias Pleroma.Web.ActivityPub.ActivityPub
+  alias Pleroma.Web.OStatus
+  alias Pleroma.Web.XML
 
   def handle(entry, doc) do
     with {:ok, actor} <- OStatus.find_make_or_update_user(doc),

lib/pleroma/web/ostatus/handlers/note_handler.ex

@@ -4,13 +4,14 @@
 
 defmodule Pleroma.Web.OStatus.NoteHandler do
   require Logger
-  alias Pleroma.Web.OStatus
-  alias Pleroma.Web.XML
+
   alias Pleroma.Activity
   alias Pleroma.Object
   alias Pleroma.Web.ActivityPub.ActivityPub
   alias Pleroma.Web.ActivityPub.Utils
   alias Pleroma.Web.CommonAPI
+  alias Pleroma.Web.OStatus
+  alias Pleroma.Web.XML
 
   @doc """
   Get the context for this note. Uses this:
@@ -18,13 +19,13 @@ defmodule Pleroma.Web.OStatus.NoteHandler do
   2. The conversation reference in the ostatus xml
   3. A newly generated context id.
   """
-  def get_context(entry, inReplyTo) do
+  def get_context(entry, in_reply_to) do
     context =
       (XML.string_from_xpath("//ostatus:conversation[1]", entry) ||
          XML.string_from_xpath("//ostatus:conversation[1]/@ref", entry) || "")
       |> String.trim()
 
-    with %{data: %{"context" => context}} <- Object.get_cached_by_ap_id(inReplyTo) do
+    with %{data: %{"context" => context}} <- Object.get_cached_by_ap_id(in_reply_to) do
       context
     else
       _e ->
@@ -87,14 +88,14 @@ defmodule Pleroma.Web.OStatus.NoteHandler do
     Map.put(note, "external_url", url)
   end
 
-  def fetch_replied_to_activity(entry, inReplyTo) do
-    with %Activity{} = activity <- Activity.get_create_by_object_ap_id(inReplyTo) do
+  def fetch_replied_to_activity(entry, in_reply_to) do
+    with %Activity{} = activity <- Activity.get_create_by_object_ap_id(in_reply_to) do
       activity
     else
       _e ->
-        with inReplyToHref when not is_nil(inReplyToHref) <-
+        with in_reply_to_href when not is_nil(in_reply_to_href) <-
                XML.string_from_xpath("//thr:in-reply-to[1]/@href", entry),
-             {:ok, [activity | _]} <- OStatus.fetch_activity_from_url(inReplyToHref) do
+             {:ok, [activity | _]} <- OStatus.fetch_activity_from_url(in_reply_to_href) do
           activity
         else
           _e -> nil
@@ -105,16 +106,17 @@ defmodule Pleroma.Web.OStatus.NoteHandler do
   # TODO: Clean this up a bit.
   def handle_note(entry, doc \\ nil) do
     with id <- XML.string_from_xpath("//id", entry),
-         activity when is_nil(activity) <- Activity.get_create_by_object_ap_id(id),
+         activity when is_nil(activity) <- Activity.get_create_by_object_ap_id_with_object(id),
          [author] <- :xmerl_xpath.string('//author[1]', doc),
          {:ok, actor} <- OStatus.find_make_or_update_user(author),
          content_html <- OStatus.get_content(entry),
          cw <- OStatus.get_cw(entry),
-         inReplyTo <- XML.string_from_xpath("//thr:in-reply-to[1]/@ref", entry),
-         inReplyToActivity <- fetch_replied_to_activity(entry, inReplyTo),
-         inReplyTo <- (inReplyToActivity && inReplyToActivity.data["object"]["id"]) || inReplyTo,
+         in_reply_to <- XML.string_from_xpath("//thr:in-reply-to[1]/@ref", entry),
+         in_reply_to_activity <- fetch_replied_to_activity(entry, in_reply_to),
+         in_reply_to <-
+           (in_reply_to_activity && in_reply_to_activity.data["object"]["id"]) || in_reply_to,
          attachments <- OStatus.get_attachments(entry),
-         context <- get_context(entry, inReplyTo),
+         context <- get_context(entry, in_reply_to),
          tags <- OStatus.get_tags(entry),
          mentions <- get_mentions(entry),
          to <- make_to_list(actor, mentions),
@@ -128,7 +130,7 @@ defmodule Pleroma.Web.OStatus.NoteHandler do
              context,
              content_html,
              attachments,
-             inReplyToActivity,
+             in_reply_to_activity,
              [],
              cw
            ),
@@ -140,8 +142,8 @@ defmodule Pleroma.Web.OStatus.NoteHandler do
          # TODO: Handle this case in make_note_data
          note <-
            if(
-             inReplyTo && !inReplyToActivity,
-             do: note |> Map.put("inReplyTo", inReplyTo),
+             in_reply_to && !in_reply_to_activity,
+             do: note |> Map.put("inReplyTo", in_reply_to),
              else: note
            ) do
       ActivityPub.create(%{

lib/pleroma/web/ostatus/handlers/unfollow_handler.ex

@@ -3,10 +3,10 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.OStatus.UnfollowHandler do
-  alias Pleroma.Web.XML
-  alias Pleroma.Web.OStatus
-  alias Pleroma.Web.ActivityPub.ActivityPub
   alias Pleroma.User
+  alias Pleroma.Web.ActivityPub.ActivityPub
+  alias Pleroma.Web.OStatus
+  alias Pleroma.Web.XML
 
   def handle(entry, doc) do
     with {:ok, actor} <- OStatus.find_make_or_update_user(doc),

lib/pleroma/web/ostatus/ostatus.ex

@@ -9,22 +9,22 @@ defmodule Pleroma.Web.OStatus do
   import Pleroma.Web.XML
   require Logger
 
+  alias Pleroma.Activity
+  alias Pleroma.Object
   alias Pleroma.Repo
   alias Pleroma.User
   alias Pleroma.Web
-  alias Pleroma.Object
-  alias Pleroma.Activity
   alias Pleroma.Web.ActivityPub.ActivityPub
   alias Pleroma.Web.ActivityPub.Transmogrifier
+  alias Pleroma.Web.OStatus.DeleteHandler
+  alias Pleroma.Web.OStatus.FollowHandler
+  alias Pleroma.Web.OStatus.NoteHandler
+  alias Pleroma.Web.OStatus.UnfollowHandler
   alias Pleroma.Web.WebFinger
   alias Pleroma.Web.Websub
-  alias Pleroma.Web.OStatus.FollowHandler
-  alias Pleroma.Web.OStatus.UnfollowHandler
-  alias Pleroma.Web.OStatus.NoteHandler
-  alias Pleroma.Web.OStatus.DeleteHandler
 
-  def is_representable?(%Activity{data: data}) do
-    object = Object.normalize(data["object"])
+  def is_representable?(%Activity{} = activity) do
+    object = Object.normalize(activity)
 
     cond do
       is_nil(object) ->
@@ -119,7 +119,7 @@ defmodule Pleroma.Web.OStatus do
 
   def make_share(entry, doc, retweeted_activity) do
     with {:ok, actor} <- find_make_or_update_user(doc),
-         %Object{} = object <- Object.normalize(retweeted_activity.data["object"]),
+         %Object{} = object <- Object.normalize(retweeted_activity),
          id when not is_nil(id) <- string_from_xpath("/entry/id", entry),
          {:ok, activity, _object} = ActivityPub.announce(actor, object, id, false) do
       {:ok, activity}
@@ -137,7 +137,7 @@ defmodule Pleroma.Web.OStatus do
 
   def make_favorite(entry, doc, favorited_activity) do
     with {:ok, actor} <- find_make_or_update_user(doc),
-         %Object{} = object <- Object.normalize(favorited_activity.data["object"]),
+         %Object{} = object <- Object.normalize(favorited_activity),
          id when not is_nil(id) <- string_from_xpath("/entry/id", entry),
          {:ok, activity, _object} = ActivityPub.like(actor, object, id, false) do
       {:ok, activity}
@@ -159,7 +159,7 @@ defmodule Pleroma.Web.OStatus do
     Logger.debug("Trying to get entry from db")
 
     with id when not is_nil(id) <- string_from_xpath("//activity:object[1]/id", entry),
-         %Activity{} = activity <- Activity.get_create_by_object_ap_id(id) do
+         %Activity{} = activity <- Activity.get_create_by_object_ap_id_with_object(id) do
       {:ok, activity}
     else
       _ ->

lib/pleroma/web/ostatus/ostatus_controller.ex

@@ -9,13 +9,13 @@ defmodule Pleroma.Web.OStatus.OStatusController do
   alias Pleroma.Object
   alias Pleroma.User
   alias Pleroma.Web.ActivityPub.ActivityPub
-  alias Pleroma.Web.ActivityPub.Visibility
   alias Pleroma.Web.ActivityPub.ActivityPubController
   alias Pleroma.Web.ActivityPub.ObjectView
-  alias Pleroma.Web.OStatus.ActivityRepresenter
-  alias Pleroma.Web.OStatus.FeedRepresenter
+  alias Pleroma.Web.ActivityPub.Visibility
   alias Pleroma.Web.Federator
   alias Pleroma.Web.OStatus
+  alias Pleroma.Web.OStatus.ActivityRepresenter
+  alias Pleroma.Web.OStatus.FeedRepresenter
   alias Pleroma.Web.XML
 
   plug(Pleroma.Web.FederatingPlug when action in [:salmon_incoming])
@@ -102,7 +102,8 @@ defmodule Pleroma.Web.OStatus.OStatusController do
       ActivityPubController.call(conn, :object)
     else
       with id <- o_status_url(conn, :object, uuid),
-           {_, %Activity{} = activity} <- {:activity, Activity.get_create_by_object_ap_id(id)},
+           {_, %Activity{} = activity} <-
+             {:activity, Activity.get_create_by_object_ap_id_with_object(id)},
            {_, true} <- {:public?, Visibility.is_public?(activity)},
            %User{} = user <- User.get_cached_by_ap_id(activity.data["actor"]) do
         case get_format(conn) do
@@ -148,13 +149,13 @@ defmodule Pleroma.Web.OStatus.OStatusController do
   end
 
   def notice(conn, %{"id" => id}) do
-    with {_, %Activity{} = activity} <- {:activity, Activity.get_by_id(id)},
+    with {_, %Activity{} = activity} <- {:activity, Activity.get_by_id_with_object(id)},
          {_, true} <- {:public?, Visibility.is_public?(activity)},
          %User{} = user <- User.get_cached_by_ap_id(activity.data["actor"]) do
       case format = get_format(conn) do
         "html" ->
           if activity.data["type"] == "Create" do
-            %Object{} = object = Object.normalize(activity.data["object"])
+            %Object{} = object = Object.normalize(activity)
 
             Fallback.RedirectController.redirector_with_meta(conn, %{
               activity_id: activity.id,
@@ -191,9 +192,9 @@ defmodule Pleroma.Web.OStatus.OStatusController do
 
   # Returns an HTML embedded <audio> or <video> player suitable for embed iframes.
   def notice_player(conn, %{"id" => id}) do
-    with %Activity{data: %{"type" => "Create"}} = activity <- Activity.get_by_id(id),
+    with %Activity{data: %{"type" => "Create"}} = activity <- Activity.get_by_id_with_object(id),
          true <- Visibility.is_public?(activity),
-         %Object{} = object <- Object.normalize(activity.data["object"]),
+         %Object{} = object <- Object.normalize(activity),
          %{data: %{"attachment" => [%{"url" => [url | _]} | _]}} <- object,
          true <- String.starts_with?(url["mediaType"], ["audio", "video"]) do
       conn
@@ -219,7 +220,7 @@ defmodule Pleroma.Web.OStatus.OStatusController do
          %Activity{data: %{"type" => "Create"}} = activity,
          _user
        ) do
-    object = Object.normalize(activity.data["object"])
+    object = Object.normalize(activity)
 
     conn
     |> put_resp_header("content-type", "application/activity+json")

lib/pleroma/web/push/impl.ex

@@ -0,0 +1,133 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.Push.Impl do
+  @moduledoc "The module represents implementation push web notification"
+
+  alias Pleroma.Activity
+  alias Pleroma.Notification
+  alias Pleroma.Object
+  alias Pleroma.Repo
+  alias Pleroma.User
+  alias Pleroma.Web.Metadata.Utils
+  alias Pleroma.Web.Push.Subscription
+
+  require Logger
+  import Ecto.Query
+
+  @types ["Create", "Follow", "Announce", "Like"]
+
+  @doc "Performs sending notifications for user subscriptions"
+  @spec perform_send(Notification.t()) :: list(any)
+  def perform_send(
+        %{activity: %{data: %{"type" => activity_type}, id: activity_id}, user_id: user_id} =
+          notif
+      )
+      when activity_type in @types do
+    actor = User.get_cached_by_ap_id(notif.activity.data["actor"])
+
+    type = Activity.mastodon_notification_type(notif.activity)
+    gcm_api_key = Application.get_env(:web_push_encryption, :gcm_api_key)
+    avatar_url = User.avatar_url(actor)
+
+    for subscription <- fetch_subsriptions(user_id),
+        get_in(subscription.data, ["alerts", type]) do
+      %{
+        title: format_title(notif),
+        access_token: subscription.token.token,
+        body: format_body(notif, actor),
+        notification_id: notif.id,
+        notification_type: type,
+        icon: avatar_url,
+        preferred_locale: "en",
+        pleroma: %{
+          activity_id: activity_id
+        }
+      }
+      |> Jason.encode!()
+      |> push_message(build_sub(subscription), gcm_api_key, subscription)
+    end
+  end
+
+  def perform_send(_) do
+    Logger.warn("Unknown notification type")
+    :error
+  end
+
+  @doc "Push message to web"
+  def push_message(body, sub, api_key, subscription) do
+    case WebPushEncryption.send_web_push(body, sub, api_key) do
+      {:ok, %{status_code: code}} when 400 <= code and code < 500 ->
+        Logger.debug("Removing subscription record")
+        Repo.delete!(subscription)
+        :ok
+
+      {:ok, %{status_code: code}} when 200 <= code and code < 300 ->
+        :ok
+
+      {:ok, %{status_code: code}} ->
+        Logger.error("Web Push Notification failed with code: #{code}")
+        :error
+
+      _ ->
+        Logger.error("Web Push Notification failed with unknown error")
+        :error
+    end
+  end
+
+  @doc "Gets user subscriptions"
+  def fetch_subsriptions(user_id) do
+    Subscription
+    |> where(user_id: ^user_id)
+    |> preload(:token)
+    |> Repo.all()
+  end
+
+  def build_sub(subscription) do
+    %{
+      keys: %{
+        p256dh: subscription.key_p256dh,
+        auth: subscription.key_auth
+      },
+      endpoint: subscription.endpoint
+    }
+  end
+
+  def format_body(
+        %{activity: %{data: %{"type" => "Create", "object" => %{"content" => content}}}},
+        actor
+      ) do
+    "@#{actor.nickname}: #{Utils.scrub_html_and_truncate(content, 80)}"
+  end
+
+  def format_body(
+        %{activity: %{data: %{"type" => "Announce", "object" => activity_id}}},
+        actor
+      ) do
+    %Activity{data: %{"object" => %{"id" => object_id}}} = Activity.get_by_ap_id(activity_id)
+    %Object{data: %{"content" => content}} = Object.get_by_ap_id(object_id)
+
+    "@#{actor.nickname} repeated: #{Utils.scrub_html_and_truncate(content, 80)}"
+  end
+
+  def format_body(
+        %{activity: %{data: %{"type" => type}}},
+        actor
+      )
+      when type in ["Follow", "Like"] do
+    case type do
+      "Follow" -> "@#{actor.nickname} has followed you"
+      "Like" -> "@#{actor.nickname} has favorited your post"
+    end
+  end
+
+  def format_title(%{activity: %{data: %{"type" => type}}}) do
+    case type do
+      "Create" -> "New Mention"
+      "Follow" -> "New Follower"
+      "Announce" -> "New Repeat"
+      "Like" -> "New Favorite"
+    end
+  end
+end

lib/pleroma/web/push/push.ex

@@ -5,24 +5,23 @@
 defmodule Pleroma.Web.Push do
   use GenServer
 
-  alias Pleroma.Repo
-  alias Pleroma.User
-  alias Pleroma.Web.Push.Subscription
+  alias Pleroma.Web.Push.Impl
 
   require Logger
-  import Ecto.Query
 
-  @types ["Create", "Follow", "Announce", "Like"]
+  ##############
+  # Client API #
+  ##############
 
-  def start_link() do
+  def start_link do
     GenServer.start_link(__MODULE__, :ok, name: __MODULE__)
   end
 
-  def vapid_config() do
+  def vapid_config do
     Application.get_env(:web_push_encryption, :vapid_details, [])
   end
 
-  def enabled() do
+  def enabled do
     case vapid_config() do
       [] -> false
       list when is_list(list) -> true
@@ -30,14 +29,18 @@ defmodule Pleroma.Web.Push do
     end
   end
 
-  def send(notification) do
-    if enabled() do
-      GenServer.cast(Pleroma.Web.Push, {:send, notification})
-    end
-  end
+  def send(notification),
+    do: GenServer.cast(__MODULE__, {:send, notification})
 
+  ####################
+  # Server Callbacks #
+  ####################
+
+  @impl true
   def init(:ok) do
-    if !enabled() do
+    if enabled() do
+      {:ok, nil}
+    else
       Logger.warn("""
       VAPID key pair is not found. If you wish to enabled web push, please run
 
@@ -47,93 +50,15 @@ defmodule Pleroma.Web.Push do
       """)
 
       :ignore
-    else
-      {:ok, nil}
     end
   end
 
-  def handle_cast(
-        {:send, %{activity: %{data: %{"type" => type}}, user_id: user_id} = notification},
-        state
-      )
-      when type in @types do
-    actor = User.get_cached_by_ap_id(notification.activity.data["actor"])
-
-    type = Pleroma.Activity.mastodon_notification_type(notification.activity)
-
-    Subscription
-    |> where(user_id: ^user_id)
-    |> preload(:token)
-    |> Repo.all()
-    |> Enum.filter(fn subscription ->
-      get_in(subscription.data, ["alerts", type]) || false
-    end)
-    |> Enum.each(fn subscription ->
-      sub = %{
-        keys: %{
-          p256dh: subscription.key_p256dh,
-          auth: subscription.key_auth
-        },
-        endpoint: subscription.endpoint
-      }
-
-      body =
-        Jason.encode!(%{
-          title: format_title(notification),
-          access_token: subscription.token.token,
-          body: format_body(notification, actor),
-          notification_id: notification.id,
-          notification_type: type,
-          icon: User.avatar_url(actor),
-          preferred_locale: "en"
-        })
-
-      case WebPushEncryption.send_web_push(
-             body,
-             sub,
-             Application.get_env(:web_push_encryption, :gcm_api_key)
-           ) do
-        {:ok, %{status_code: code}} when 400 <= code and code < 500 ->
-          Logger.debug("Removing subscription record")
-          Repo.delete!(subscription)
-          :ok
-
-        {:ok, %{status_code: code}} when 200 <= code and code < 300 ->
-          :ok
-
-        {:ok, %{status_code: code}} ->
-          Logger.error("Web Push Notification failed with code: #{code}")
-          :error
-
-        _ ->
-          Logger.error("Web Push Notification failed with unknown error")
-          :error
-      end
-    end)
+  @impl true
+  def handle_cast({:send, notification}, state) do
+    if enabled() do
+      Impl.perform_send(notification)
+    end
 
     {:noreply, state}
   end
-
-  def handle_cast({:send, _}, state) do
-    Logger.warn("Unknown notification type")
-    {:noreply, state}
-  end
-
-  defp format_title(%{activity: %{data: %{"type" => type}}}) do
-    case type do
-      "Create" -> "New Mention"
-      "Follow" -> "New Follower"
-      "Announce" -> "New Repeat"
-      "Like" -> "New Favorite"
-    end
-  end
-
-  defp format_body(%{activity: %{data: %{"type" => type}}}, actor) do
-    case type do
-      "Create" -> "@#{actor.nickname} has mentioned you"
-      "Follow" -> "@#{actor.nickname} has followed you"
-      "Announce" -> "@#{actor.nickname} has repeated your post"
-      "Like" -> "@#{actor.nickname} has favorited your post"
-    end
-  end
 end

lib/pleroma/web/push/subscription.ex

@@ -12,6 +12,8 @@ defmodule Pleroma.Web.Push.Subscription do
   alias Pleroma.Web.OAuth.Token
   alias Pleroma.Web.Push.Subscription
 
+  @type t :: %__MODULE__{}
+
   schema "push_subscriptions" do
     belongs_to(:user, User, type: Pleroma.FlakeId)
     belongs_to(:token, Token)
@@ -50,30 +52,38 @@ defmodule Pleroma.Web.Push.Subscription do
     })
   end
 
+  @doc "Gets subsciption by user & token"
+  @spec get(User.t(), Token.t()) :: {:ok, t()} | {:error, :not_found}
   def get(%User{id: user_id}, %Token{id: token_id}) do
-    Repo.get_by(Subscription, user_id: user_id, token_id: token_id)
+    case Repo.get_by(Subscription, user_id: user_id, token_id: token_id) do
+      nil -> {:error, :not_found}
+      subscription -> {:ok, subscription}
+    end
   end
 
   def update(user, token, params) do
-    get(user, token)
-    |> change(data: alerts(params))
-    |> Repo.update()
+    with {:ok, subscription} <- get(user, token) do
+      subscription
+      |> change(data: alerts(params))
+      |> Repo.update()
+    end
   end
 
   def delete(user, token) do
-    Repo.delete(get(user, token))
+    with {:ok, subscription} <- get(user, token),
+         do: Repo.delete(subscription)
   end
 
   def delete_if_exists(user, token) do
     case get(user, token) do
-      nil -> {:ok, nil}
-      sub -> Repo.delete(sub)
+      {:error, _} -> {:ok, nil}
+      {:ok, sub} -> Repo.delete(sub)
     end
   end
 
   # Some webpush clients (e.g. iOS Toot!) use an non urlsafe base64 as an encoding for the key.
-  # However, the web push rfs specify to use base64 urlsafe, and the `web_push_encryption` library we use
-  # requires the key to be properly encoded. So we just convert base64 to urlsafe base64.
+  # However, the web push rfs specify to use base64 urlsafe, and the `web_push_encryption` library
+  # we use requires the key to be properly encoded. So we just convert base64 to urlsafe base64.
   defp ensure_base64_urlsafe(string) do
     string
     |> String.replace("+", "-")

lib/pleroma/web/rel_me.ex

@@ -5,7 +5,6 @@
 defmodule Pleroma.Web.RelMe do
   @hackney_options [
     pool: :media,
-    timeout: 2_000,
     recv_timeout: 2_000,
     max_body: 2_000_000
   ]
@@ -28,7 +27,8 @@ defmodule Pleroma.Web.RelMe do
     {:ok, %Tesla.Env{body: html}} = Pleroma.HTTP.get(url, [], adapter: @hackney_options)
 
     data =
-      Floki.attribute(html, "link[rel=me]", "href") ++ Floki.attribute(html, "a[rel=me]", "href")
+      Floki.attribute(html, "link[rel~=me]", "href") ++
+        Floki.attribute(html, "a[rel~=me]", "href")
 
     {:ok, data}
   rescue

lib/pleroma/web/rich_media/helpers.ex

@@ -4,18 +4,34 @@
 
 defmodule Pleroma.Web.RichMedia.Helpers do
   alias Pleroma.Activity
-  alias Pleroma.Object
   alias Pleroma.HTML
+  alias Pleroma.Object
   alias Pleroma.Web.RichMedia.Parser
 
-  def fetch_data_for_activity(%Activity{} = activity) do
+  defp validate_page_url(page_url) when is_binary(page_url) do
+    if AutoLinker.Parser.is_url?(page_url, true) do
+      URI.parse(page_url) |> validate_page_url
+    else
+      :error
+    end
+  end
+
+  defp validate_page_url(%URI{authority: nil}), do: :error
+  defp validate_page_url(%URI{scheme: nil}), do: :error
+  defp validate_page_url(%URI{}), do: :ok
+  defp validate_page_url(_), do: :error
+
+  def fetch_data_for_activity(%Activity{data: %{"type" => "Create"}} = activity) do
     with true <- Pleroma.Config.get([:rich_media, :enabled]),
-         %Object{} = object <- Object.normalize(activity.data["object"]),
+         %Object{} = object <- Object.normalize(activity),
          {:ok, page_url} <- HTML.extract_first_external_url(object, object.data["content"]),
+         :ok <- validate_page_url(page_url),
          {:ok, rich_media} <- Parser.parse(page_url) do
       %{page_url: page_url, rich_media: rich_media}
     else
       _ -> %{}
     end
   end
+
+  def fetch_data_for_activity(_), do: %{}
 end

lib/pleroma/web/rich_media/parser.ex

@@ -11,7 +11,6 @@ defmodule Pleroma.Web.RichMedia.Parser do
 
   @hackney_options [
     pool: :media,
-    timeout: 2_000,
     recv_timeout: 2_000,
     max_body: 2_000_000
   ]

lib/pleroma/web/router.ex

@@ -140,7 +140,7 @@ defmodule Pleroma.Web.Router do
     pipe_through([:admin_api, :oauth_write])
 
     get("/users", AdminAPIController, :list_users)
-    get("/users/search", AdminAPIController, :search_users)
+    get("/users/:nickname", AdminAPIController, :user_show)
     delete("/user", AdminAPIController, :user_delete)
     patch("/users/:nickname/toggle_activation", AdminAPIController, :user_toggle_activation)
     post("/user", AdminAPIController, :user_create)
@@ -191,6 +191,12 @@ defmodule Pleroma.Web.Router do
       post("/blocks_import", UtilController, :blocks_import)
       post("/follow_import", UtilController, :follow_import)
     end
+
+    scope [] do
+      pipe_through(:oauth_read)
+
+      post("/notifications/read", UtilController, :notifications_read)
+    end
   end
 
   scope "/oauth", Pleroma.Web.OAuth do
@@ -212,6 +218,7 @@ defmodule Pleroma.Web.Router do
       get("/accounts/search", MastodonAPIController, :account_search)
 
       get("/accounts/:id/lists", MastodonAPIController, :account_lists)
+      get("/accounts/:id/identity_proofs", MastodonAPIController, :empty_array)
 
       get("/follow_requests", MastodonAPIController, :follow_requests)
       get("/blocks", MastodonAPIController, :blocks)
@@ -304,10 +311,10 @@ defmodule Pleroma.Web.Router do
     scope [] do
       pipe_through(:oauth_push)
 
-      post("/push/subscription", MastodonAPIController, :create_push_subscription)
-      get("/push/subscription", MastodonAPIController, :get_push_subscription)
-      put("/push/subscription", MastodonAPIController, :update_push_subscription)
-      delete("/push/subscription", MastodonAPIController, :delete_push_subscription)
+      post("/push/subscription", SubscriptionController, :create)
+      get("/push/subscription", SubscriptionController, :get)
+      put("/push/subscription", SubscriptionController, :update)
+      delete("/push/subscription", SubscriptionController, :delete)
     end
   end
 
@@ -323,6 +330,7 @@ defmodule Pleroma.Web.Router do
     get("/instance", MastodonAPIController, :masto_instance)
     get("/instance/peers", MastodonAPIController, :peers)
     post("/apps", MastodonAPIController, :create_app)
+    get("/apps/verify_credentials", MastodonAPIController, :verify_app_credentials)
     get("/custom_emojis", MastodonAPIController, :custom_emojis)
 
     get("/statuses/:id/card", MastodonAPIController, :status_card)
@@ -632,8 +640,8 @@ end
 
 defmodule Fallback.RedirectController do
   use Pleroma.Web, :controller
-  alias Pleroma.Web.Metadata
   alias Pleroma.User
+  alias Pleroma.Web.Metadata
 
   def redirector(conn, _params, code \\ 200) do
     conn