Merge remote-tracking branch 'remotes/upstream/develop' into 1149-oban-job-queue

lib/mix/tasks/pleroma/digest.ex

@@ -0,0 +1,33 @@
+defmodule Mix.Tasks.Pleroma.Digest do
+  use Mix.Task
+
+  @shortdoc "Manages digest emails"
+  @moduledoc """
+  Manages digest emails
+
+  ## Send digest email since given date (user registration date by default)
+  ignoring user activity status.
+
+  ``mix pleroma.digest test <nickname> <since_date>``
+
+  Example: ``mix pleroma.digest test donaldtheduck 2019-05-20``
+  """
+  def run(["test", nickname | opts]) do
+    Mix.Pleroma.start_pleroma()
+
+    user = Pleroma.User.get_by_nickname(nickname)
+
+    last_digest_emailed_at =
+      with [date] <- opts,
+           {:ok, datetime} <- Timex.parse(date, "{YYYY}-{0M}-{0D}") do
+        datetime
+      else
+        _ -> user.inserted_at
+      end
+
+    patched_user = %{user | last_digest_emailed_at: last_digest_emailed_at}
+
+    _user = Pleroma.DigestEmailWorker.perform(patched_user)
+    Mix.shell().info("Digest email have been sent to #{nickname} (#{user.email})")
+  end
+end

lib/mix/tasks/pleroma/instance.ex

@@ -183,6 +183,7 @@ defmodule Mix.Tasks.Pleroma.Instance do
         )
 
       secret = :crypto.strong_rand_bytes(64) |> Base.encode64() |> binary_part(0, 64)
+      jwt_secret = :crypto.strong_rand_bytes(64) |> Base.encode64() |> binary_part(0, 64)
       signing_salt = :crypto.strong_rand_bytes(8) |> Base.encode64() |> binary_part(0, 8)
       {web_push_public_key, web_push_private_key} = :crypto.generate_key(:ecdh, :prime256v1)
       template_dir = Application.app_dir(:pleroma, "priv") <> "/templates"
@@ -200,6 +201,7 @@ defmodule Mix.Tasks.Pleroma.Instance do
           dbuser: dbuser,
           dbpass: dbpass,
           secret: secret,
+          jwt_secret: jwt_secret,
           signing_salt: signing_salt,
           web_push_public_key: Base.url_encode64(web_push_public_key, padding: false),
           web_push_private_key: Base.url_encode64(web_push_private_key, padding: false),

lib/mix/tasks/pleroma/relay.ex

@@ -5,6 +5,7 @@
 defmodule Mix.Tasks.Pleroma.Relay do
   use Mix.Task
   import Mix.Pleroma
+  alias Pleroma.User
   alias Pleroma.Web.ActivityPub.Relay
 
   @shortdoc "Manages remote relays"
@@ -22,6 +23,10 @@ defmodule Mix.Tasks.Pleroma.Relay do
   ``mix pleroma.relay unfollow <relay_url>``
 
   Example: ``mix pleroma.relay unfollow https://example.org/relay``
+
+  ## List relay subscriptions
+
+  ``mix pleroma.relay list``
   """
   def run(["follow", target]) do
     start_pleroma()
@@ -44,4 +49,19 @@ defmodule Mix.Tasks.Pleroma.Relay do
       {:error, e} -> shell_error("Error while following #{target}: #{inspect(e)}")
     end
   end
+
+  def run(["list"]) do
+    start_pleroma()
+
+    with %User{} = user <- Relay.get_actor() do
+      user.following
+      |> Enum.each(fn entry ->
+        URI.parse(entry)
+        |> Map.get(:host)
+        |> shell_info()
+      end)
+    else
+      e -> shell_error("Error while fetching relay subscription list: #{inspect(e)}")
+    end
+  end
 end

lib/mix/tasks/pleroma/user.ex

@@ -31,8 +31,8 @@ defmodule Mix.Tasks.Pleroma.User do
       mix pleroma.user invite [OPTION...]
 
     Options:
-    - `--expires_at DATE` - last day on which token is active (e.g. "2019-04-05")
-    - `--max_use NUMBER` - maximum numbers of token uses
+    - `--expires-at DATE` - last day on which token is active (e.g. "2019-04-05")
+    - `--max-use NUMBER` - maximum numbers of token uses
 
   ## List generated invites
 

lib/pleroma/application.ex

@@ -162,7 +162,9 @@ defmodule Pleroma.Application do
     # See http://elixir-lang.org/docs/stable/elixir/Supervisor.html
     # for other strategies and supported options
     opts = [strategy: :one_for_one, name: Pleroma.Supervisor]
-    Supervisor.start_link(children, opts)
+    result = Supervisor.start_link(children, opts)
+    :ok = after_supervisor_start()
+    result
   end
 
   defp setup_instrumenters do
@@ -227,4 +229,17 @@ defmodule Pleroma.Application do
       :hackney_pool.child_spec(pool, options)
     end
   end
+
+  defp after_supervisor_start do
+    with digest_config <- Application.get_env(:pleroma, :email_notifications)[:digest],
+         true <- digest_config[:active] do
+      PleromaJobQueue.schedule(
+        digest_config[:schedule],
+        :digest_emails,
+        Pleroma.DigestEmailWorker
+      )
+    end
+
+    :ok
+  end
 end

lib/pleroma/digest_email_worker.ex

@@ -0,0 +1,35 @@
+defmodule Pleroma.DigestEmailWorker do
+  import Ecto.Query
+
+  @queue_name :digest_emails
+
+  def perform do
+    config = Pleroma.Config.get([:email_notifications, :digest])
+    negative_interval = -Map.fetch!(config, :interval)
+    inactivity_threshold = Map.fetch!(config, :inactivity_threshold)
+    inactive_users_query = Pleroma.User.list_inactive_users_query(inactivity_threshold)
+
+    now = NaiveDateTime.truncate(NaiveDateTime.utc_now(), :second)
+
+    from(u in inactive_users_query,
+      where: fragment(~s(? #> '{"email_notifications","digest"}' @> 'true'), u.info),
+      where: u.last_digest_emailed_at < datetime_add(^now, ^negative_interval, "day"),
+      select: u
+    )
+    |> Pleroma.Repo.all()
+    |> Enum.each(&PleromaJobQueue.enqueue(@queue_name, __MODULE__, [&1]))
+  end
+
+  @doc """
+  Send digest email to the given user.
+  Updates `last_digest_emailed_at` field for the user and returns the updated user.
+  """
+  @spec perform(Pleroma.User.t()) :: Pleroma.User.t()
+  def perform(user) do
+    with %Swoosh.Email{} = email <- Pleroma.Emails.UserEmail.digest_email(user) do
+      Pleroma.Emails.Mailer.deliver_async(email)
+    end
+
+    Pleroma.User.touch_last_digest_emailed_at(user)
+  end
+end

lib/pleroma/emails/admin_email.ex

@@ -63,7 +63,6 @@ defmodule Pleroma.Emails.AdminEmail do
     new()
     |> to({to.name, to.email})
     |> from({instance_name(), instance_notify_email()})
-    |> reply_to({reporter.name, reporter.email})
     |> subject("#{instance_name()} Report")
     |> html_body(html_body)
   end

lib/pleroma/emails/user_email.ex

@@ -5,7 +5,7 @@
 defmodule Pleroma.Emails.UserEmail do
   @moduledoc "User emails"
 
-  import Swoosh.Email
+  use Phoenix.Swoosh, view: Pleroma.Web.EmailView, layout: {Pleroma.Web.LayoutView, :email}
 
   alias Pleroma.Web.Endpoint
   alias Pleroma.Web.Router
@@ -87,4 +87,73 @@ defmodule Pleroma.Emails.UserEmail do
     |> subject("#{instance_name()} account confirmation")
     |> html_body(html_body)
   end
+
+  @doc """
+  Email used in digest email notifications
+  Includes Mentions and New Followers data
+  If there are no mentions (even when new followers exist), the function will return nil
+  """
+  @spec digest_email(Pleroma.User.t()) :: Swoosh.Email.t() | nil
+  def digest_email(user) do
+    new_notifications =
+      Pleroma.Notification.for_user_since(user, user.last_digest_emailed_at)
+      |> Enum.reduce(%{followers: [], mentions: []}, fn
+        %{activity: %{data: %{"type" => "Create"}, actor: actor} = activity} = notification,
+        acc ->
+          new_mention = %{
+            data: notification,
+            object: Pleroma.Object.normalize(activity),
+            from: Pleroma.User.get_by_ap_id(actor)
+          }
+
+          %{acc | mentions: [new_mention | acc.mentions]}
+
+        %{activity: %{data: %{"type" => "Follow"}, actor: actor} = activity} = notification,
+        acc ->
+          new_follower = %{
+            data: notification,
+            object: Pleroma.Object.normalize(activity),
+            from: Pleroma.User.get_by_ap_id(actor)
+          }
+
+          %{acc | followers: [new_follower | acc.followers]}
+
+        _, acc ->
+          acc
+      end)
+
+    with [_ | _] = mentions <- new_notifications.mentions do
+      html_data = %{
+        instance: instance_name(),
+        user: user,
+        mentions: mentions,
+        followers: new_notifications.followers,
+        unsubscribe_link: unsubscribe_url(user, "digest")
+      }
+
+      new()
+      |> to(recipient(user))
+      |> from(sender())
+      |> subject("Your digest from #{instance_name()}")
+      |> render_body("digest.html", html_data)
+    else
+      _ ->
+        nil
+    end
+  end
+
+  @doc """
+  Generate unsubscribe link for given user and notifications type.
+  The link contains JWT token with the data, and subscription can be modified without
+  authorization.
+  """
+  @spec unsubscribe_url(Pleroma.User.t(), String.t()) :: String.t()
+  def unsubscribe_url(user, notifications_type) do
+    token =
+      %{"sub" => user.id, "act" => %{"unsubscribe" => notifications_type}, "exp" => false}
+      |> Pleroma.JWT.generate_and_sign!()
+      |> Base.encode64()
+
+    Router.Helpers.subscription_url(Pleroma.Web.Endpoint, :unsubscribe, token)
+  end
 end

lib/pleroma/jwt.ex

@@ -0,0 +1,9 @@
+defmodule Pleroma.JWT do
+  use Joken.Config
+
+  @impl true
+  def token_config do
+    default_claims(skip: [:aud])
+    |> add_claim("aud", &Pleroma.Web.Endpoint.url/0, &(&1 == Pleroma.Web.Endpoint.url()))
+  end
+end

lib/pleroma/notification.ex

@@ -18,6 +18,8 @@ defmodule Pleroma.Notification do
   import Ecto.Query
   import Ecto.Changeset
 
+  @type t :: %__MODULE__{}
+
   schema "notifications" do
     field(:seen, :boolean, default: false)
     belongs_to(:user, User, type: Pleroma.FlakeId)
@@ -31,7 +33,7 @@ defmodule Pleroma.Notification do
     |> cast(attrs, [:seen])
   end
 
-  def for_user_query(user, opts) do
+  def for_user_query(user, opts \\ []) do
     query =
       Notification
       |> where(user_id: ^user.id)
@@ -75,6 +77,25 @@ defmodule Pleroma.Notification do
     |> Pagination.fetch_paginated(opts)
   end
 
+  @doc """
+  Returns notifications for user received since given date.
+
+  ## Examples
+
+      iex> Pleroma.Notification.for_user_since(%Pleroma.User{}, ~N[2019-04-13 11:22:33])
+      [%Pleroma.Notification{}, %Pleroma.Notification{}]
+
+      iex> Pleroma.Notification.for_user_since(%Pleroma.User{}, ~N[2019-04-15 11:22:33])
+      []
+  """
+  @spec for_user_since(Pleroma.User.t(), NaiveDateTime.t()) :: [t()]
+  def for_user_since(user, date) do
+    from(n in for_user_query(user),
+      where: n.updated_at > ^date
+    )
+    |> Repo.all()
+  end
+
   def set_read_up_to(%{id: user_id} = _user, id) do
     query =
       from(
@@ -82,7 +103,10 @@ defmodule Pleroma.Notification do
         where: n.user_id == ^user_id,
         where: n.id <= ^id,
         update: [
-          set: [seen: true]
+          set: [
+            seen: true,
+            updated_at: ^NaiveDateTime.utc_now()
+          ]
         ]
       )
 

lib/pleroma/object/fetcher.ex

@@ -114,7 +114,7 @@ defmodule Pleroma.Object.Fetcher do
     end
   end
 
-  def fetch_and_contain_remote_object_from_id(id) do
+  def fetch_and_contain_remote_object_from_id(id) when is_binary(id) do
     Logger.info("Fetching object #{id} via AP")
 
     date =
@@ -141,4 +141,9 @@ defmodule Pleroma.Object.Fetcher do
         {:error, e}
     end
   end
+
+  def fetch_and_contain_remote_object_from_id(%{"id" => id}),
+    do: fetch_and_contain_remote_object_from_id(id)
+
+  def fetch_and_contain_remote_object_from_id(_id), do: {:error, "id must be a string"}
 end

lib/pleroma/user.ex

@@ -57,6 +57,7 @@ defmodule Pleroma.User do
     field(:search_type, :integer, virtual: true)
     field(:tags, {:array, :string}, default: [])
     field(:last_refreshed_at, :naive_datetime_usec)
+    field(:last_digest_emailed_at, :naive_datetime)
     has_many(:notifications, Notification)
     has_many(:registrations, Registration)
     embeds_one(:info, User.Info)
@@ -114,7 +115,9 @@ defmodule Pleroma.User do
 
   def user_info(%User{} = user, args \\ %{}) do
     following_count =
-      if args[:following_count], do: args[:following_count], else: following_count(user)
+      if args[:following_count],
+        do: args[:following_count],
+        else: user.info.following_count || following_count(user)
 
     follower_count =
       if args[:follower_count], do: args[:follower_count], else: user.info.follower_count
@@ -406,6 +409,8 @@ defmodule Pleroma.User do
 
         {1, [follower]} = Repo.update_all(q, [])
 
+        follower = maybe_update_following_count(follower)
+
         {:ok, _} = update_follower_count(followed)
 
         set_cache(follower)
@@ -425,6 +430,8 @@ defmodule Pleroma.User do
 
       {1, [follower]} = Repo.update_all(q, [])
 
+      follower = maybe_update_following_count(follower)
+
       {:ok, followed} = update_follower_count(followed)
 
       set_cache(follower)
@@ -709,32 +716,73 @@ defmodule Pleroma.User do
     |> update_and_set_cache()
   end
 
-  def update_follower_count(%User{} = user) do
-    follower_count_query =
-      User.Query.build(%{followers: user, deactivated: false})
-      |> select([u], %{count: count(u.id)})
+  def maybe_fetch_follow_information(user) do
+    with {:ok, user} <- fetch_follow_information(user) do
+      user
+    else
+      e ->
+        Logger.error("Follower/Following counter update for #{user.ap_id} failed.\n#{inspect(e)}")
 
-    User
-    |> where(id: ^user.id)
-    |> join(:inner, [u], s in subquery(follower_count_query))
-    |> update([u, s],
-      set: [
-        info:
-          fragment(
-            "jsonb_set(?, '{follower_count}', ?::varchar::jsonb, true)",
-            u.info,
-            s.count
-          )
-      ]
-    )
-    |> select([u], u)
-    |> Repo.update_all([])
-    |> case do
-      {1, [user]} -> set_cache(user)
-      _ -> {:error, user}
+        user
     end
   end
 
+  def fetch_follow_information(user) do
+    with {:ok, info} <- ActivityPub.fetch_follow_information_for_user(user) do
+      info_cng = User.Info.follow_information_update(user.info, info)
+
+      changeset =
+        user
+        |> change()
+        |> put_embed(:info, info_cng)
+
+      update_and_set_cache(changeset)
+    else
+      {:error, _} = e -> e
+      e -> {:error, e}
+    end
+  end
+
+  def update_follower_count(%User{} = user) do
+    if user.local or !Pleroma.Config.get([:instance, :external_user_synchronization]) do
+      follower_count_query =
+        User.Query.build(%{followers: user, deactivated: false})
+        |> select([u], %{count: count(u.id)})
+
+      User
+      |> where(id: ^user.id)
+      |> join(:inner, [u], s in subquery(follower_count_query))
+      |> update([u, s],
+        set: [
+          info:
+            fragment(
+              "jsonb_set(?, '{follower_count}', ?::varchar::jsonb, true)",
+              u.info,
+              s.count
+            )
+        ]
+      )
+      |> select([u], u)
+      |> Repo.update_all([])
+      |> case do
+        {1, [user]} -> set_cache(user)
+        _ -> {:error, user}
+      end
+    else
+      {:ok, maybe_fetch_follow_information(user)}
+    end
+  end
+
+  def maybe_update_following_count(%User{local: false} = user) do
+    if Pleroma.Config.get([:instance, :external_user_synchronization]) do
+      {:ok, maybe_fetch_follow_information(user)}
+    else
+      user
+    end
+  end
+
+  def maybe_update_following_count(user), do: user
+
   def remove_duplicated_following(%User{following: following} = user) do
     uniq_following = Enum.uniq(following)
 
@@ -1372,6 +1420,80 @@ defmodule Pleroma.User do
     target.ap_id not in user.info.muted_reblogs
   end
 
+  @doc """
+  The function returns a query to get users with no activity for given interval of days.
+  Inactive users are those who didn't read any notification, or had any activity where
+  the user is the activity's actor, during `inactivity_threshold` days.
+  Deactivated users will not appear in this list.
+
+  ## Examples
+
+      iex> Pleroma.User.list_inactive_users()
+      %Ecto.Query{}
+  """
+  @spec list_inactive_users_query(integer()) :: Ecto.Query.t()
+  def list_inactive_users_query(inactivity_threshold \\ 7) do
+    negative_inactivity_threshold = -inactivity_threshold
+    now = NaiveDateTime.truncate(NaiveDateTime.utc_now(), :second)
+    # Subqueries are not supported in `where` clauses, join gets too complicated.
+    has_read_notifications =
+      from(n in Pleroma.Notification,
+        where: n.seen == true,
+        group_by: n.id,
+        having: max(n.updated_at) > datetime_add(^now, ^negative_inactivity_threshold, "day"),
+        select: n.user_id
+      )
+      |> Pleroma.Repo.all()
+
+    from(u in Pleroma.User,
+      left_join: a in Pleroma.Activity,
+      on: u.ap_id == a.actor,
+      where: not is_nil(u.nickname),
+      where: fragment("not (?->'deactivated' @> 'true')", u.info),
+      where: u.id not in ^has_read_notifications,
+      group_by: u.id,
+      having:
+        max(a.inserted_at) < datetime_add(^now, ^negative_inactivity_threshold, "day") or
+          is_nil(max(a.inserted_at))
+    )
+  end
+
+  @doc """
+  Enable or disable email notifications for user
+
+  ## Examples
+
+      iex> Pleroma.User.switch_email_notifications(Pleroma.User{info: %{email_notifications: %{"digest" => false}}}, "digest", true)
+      Pleroma.User{info: %{email_notifications: %{"digest" => true}}}
+
+      iex> Pleroma.User.switch_email_notifications(Pleroma.User{info: %{email_notifications: %{"digest" => true}}}, "digest", false)
+      Pleroma.User{info: %{email_notifications: %{"digest" => false}}}
+  """
+  @spec switch_email_notifications(t(), String.t(), boolean()) ::
+          {:ok, t()} | {:error, Ecto.Changeset.t()}
+  def switch_email_notifications(user, type, status) do
+    info = Pleroma.User.Info.update_email_notifications(user.info, %{type => status})
+
+    change(user)
+    |> put_embed(:info, info)
+    |> update_and_set_cache()
+  end
+
+  @doc """
+  Set `last_digest_emailed_at` value for the user to current time
+  """
+  @spec touch_last_digest_emailed_at(t()) :: t()
+  def touch_last_digest_emailed_at(user) do
+    now = NaiveDateTime.truncate(NaiveDateTime.utc_now(), :second)
+
+    {:ok, updated_user} =
+      user
+      |> change(%{last_digest_emailed_at: now})
+      |> update_and_set_cache()
+
+    updated_user
+  end
+
   @spec toggle_confirmation(User.t()) :: {:ok, User.t()} | {:error, Changeset.t()}
   def toggle_confirmation(%User{} = user) do
     need_confirmation? = !user.info.confirmation_pending

lib/pleroma/user/info.ex

@@ -16,6 +16,8 @@ defmodule Pleroma.User.Info do
     field(:source_data, :map, default: %{})
     field(:note_count, :integer, default: 0)
     field(:follower_count, :integer, default: 0)
+    # Should be filled in only for remote users
+    field(:following_count, :integer, default: nil)
     field(:locked, :boolean, default: false)
     field(:confirmation_pending, :boolean, default: false)
     field(:confirmation_token, :string, default: nil)
@@ -43,6 +45,7 @@ defmodule Pleroma.User.Info do
     field(:hide_follows, :boolean, default: false)
     field(:hide_favorites, :boolean, default: true)
     field(:pinned_activities, {:array, :string}, default: [])
+    field(:email_notifications, :map, default: %{"digest" => false})
     field(:mascot, :map, default: nil)
     field(:emoji, {:array, :map}, default: [])
     field(:pleroma_settings_store, :map, default: %{})
@@ -93,6 +96,30 @@ defmodule Pleroma.User.Info do
     |> validate_required([:notification_settings])
   end
 
+  @doc """
+  Update email notifications in the given User.Info struct.
+
+  Examples:
+
+      iex> update_email_notifications(%Pleroma.User.Info{email_notifications: %{"digest" => false}}, %{"digest" => true})
+      %Pleroma.User.Info{email_notifications: %{"digest" => true}}
+
+  """
+  @spec update_email_notifications(t(), map()) :: Ecto.Changeset.t()
+  def update_email_notifications(info, settings) do
+    email_notifications =
+      info.email_notifications
+      |> Map.merge(settings)
+      |> Map.take(["digest"])
+
+    params = %{email_notifications: email_notifications}
+    fields = [:email_notifications]
+
+    info
+    |> cast(params, fields)
+    |> validate_required(fields)
+  end
+
   def add_to_note_count(info, number) do
     set_note_count(info, info.note_count + number)
   end
@@ -223,7 +250,11 @@ defmodule Pleroma.User.Info do
       :uri,
       :hub,
       :topic,
-      :salmon
+      :salmon,
+      :hide_followers,
+      :hide_follows,
+      :follower_count,
+      :following_count
     ])
   end
 
@@ -234,7 +265,11 @@ defmodule Pleroma.User.Info do
       :source_data,
       :banner,
       :locked,
-      :magic_key
+      :magic_key,
+      :follower_count,
+      :following_count,
+      :hide_follows,
+      :hide_followers
     ])
   end
 
@@ -348,4 +383,14 @@ defmodule Pleroma.User.Info do
 
     cast(info, params, [:muted_reblogs])
   end
+
+  def follow_information_update(info, params) do
+    info
+    |> cast(params, [
+      :hide_followers,
+      :hide_follows,
+      :follower_count,
+      :following_count
+    ])
+  end
 end

lib/pleroma/user/search.ex

@@ -44,7 +44,7 @@ defmodule Pleroma.User.Search do
     query_string = String.trim_leading(query_string, "@")
 
     with [name, domain] <- String.split(query_string, "@"),
-         formatted_domain <- String.replace(domain, ~r/[!-\-|@|[-`|{-~|\/|:]+/, "") do
+         formatted_domain <- String.replace(domain, ~r/[!-\-|@|[-`|{-~|\/|:|\s]+/, "") do
       name <> "@" <> to_string(:idna.encode(formatted_domain))
     else
       _ -> query_string

lib/pleroma/web/activity_pub/activity_pub.ex

@@ -267,6 +267,9 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     else
       {:fake, true, activity} ->
         {:ok, activity}
+
+      {:error, message} ->
+        {:error, message}
     end
   end
 
@@ -746,8 +749,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
 
   defp restrict_favorited_by(query, %{"favorited_by" => ap_id}) do
     from(
-      activity in query,
-      where: fragment(~s(? <@ (? #> '{"object","likes"}'\)), ^ap_id, activity.data)
+      [_activity, object] in query,
+      where: fragment("(?)->'likes' \\? (?)", object.data, ^ap_id)
     )
   end
 
@@ -1009,10 +1012,10 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     user_data = %{
       ap_id: data["id"],
       info: %{
-        "ap_enabled" => true,
-        "source_data" => data,
-        "banner" => banner,
-        "locked" => locked
+        ap_enabled: true,
+        source_data: data,
+        banner: banner,
+        locked: locked
       },
       avatar: avatar,
       name: data["name"],
@@ -1036,6 +1039,71 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     {:ok, user_data}
   end
 
+  def fetch_follow_information_for_user(user) do
+    with {:ok, following_data} <-
+           Fetcher.fetch_and_contain_remote_object_from_id(user.following_address),
+         following_count when is_integer(following_count) <- following_data["totalItems"],
+         {:ok, hide_follows} <- collection_private(following_data),
+         {:ok, followers_data} <-
+           Fetcher.fetch_and_contain_remote_object_from_id(user.follower_address),
+         followers_count when is_integer(followers_count) <- followers_data["totalItems"],
+         {:ok, hide_followers} <- collection_private(followers_data) do
+      {:ok,
+       %{
+         hide_follows: hide_follows,
+         follower_count: followers_count,
+         following_count: following_count,
+         hide_followers: hide_followers
+       }}
+    else
+      {:error, _} = e ->
+        e
+
+      e ->
+        {:error, e}
+    end
+  end
+
+  defp maybe_update_follow_information(data) do
+    with {:enabled, true} <-
+           {:enabled, Pleroma.Config.get([:instance, :external_user_synchronization])},
+         {:ok, info} <- fetch_follow_information_for_user(data) do
+      info = Map.merge(data.info, info)
+      Map.put(data, :info, info)
+    else
+      {:enabled, false} ->
+        data
+
+      e ->
+        Logger.error(
+          "Follower/Following counter update for #{data.ap_id} failed.\n" <> inspect(e)
+        )
+
+        data
+    end
+  end
+
+  defp collection_private(data) do
+    if is_map(data["first"]) and
+         data["first"]["type"] in ["CollectionPage", "OrderedCollectionPage"] do
+      {:ok, false}
+    else
+      with {:ok, %{"type" => type}} when type in ["CollectionPage", "OrderedCollectionPage"] <-
+             Fetcher.fetch_and_contain_remote_object_from_id(data["first"]) do
+        {:ok, false}
+      else
+        {:error, {:ok, %{status: code}}} when code in [401, 403] ->
+          {:ok, true}
+
+        {:error, _} = e ->
+          e
+
+        e ->
+          {:error, e}
+      end
+    end
+  end
+
   def user_data_from_user_object(data) do
     with {:ok, data} <- MRF.filter(data),
          {:ok, data} <- object_to_user_data(data) do
@@ -1047,7 +1115,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
 
   def fetch_and_prepare_user_from_ap_id(ap_id) do
     with {:ok, data} <- Fetcher.fetch_and_contain_remote_object_from_id(ap_id),
-         {:ok, data} <- user_data_from_user_object(data) do
+         {:ok, data} <- user_data_from_user_object(data),
+         data <- maybe_update_follow_information(data) do
       {:ok, data}
     else
       e -> Logger.error("Could not decode user at fetch #{ap_id}, #{inspect(e)}")

lib/pleroma/web/activity_pub/transmogrifier.ex

@@ -608,13 +608,13 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
     with %User{ap_id: ^actor_id} = actor <- User.get_cached_by_ap_id(object["id"]) do
       {:ok, new_user_data} = ActivityPub.user_data_from_user_object(object)
 
-      banner = new_user_data[:info]["banner"]
-      locked = new_user_data[:info]["locked"] || false
+      banner = new_user_data[:info][:banner]
+      locked = new_user_data[:info][:locked] || false
 
       update_data =
         new_user_data
         |> Map.take([:name, :bio, :avatar])
-        |> Map.put(:info, %{"banner" => banner, "locked" => locked})
+        |> Map.put(:info, %{banner: banner, locked: locked})
 
       actor
       |> User.upgrade_changeset(update_data)
@@ -1076,10 +1076,6 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
         PleromaJobQueue.enqueue(:transmogrifier, __MODULE__, [:user_upgrade, user])
       end
 
-      if Pleroma.Config.get([:instance, :external_user_synchronization]) do
-        update_following_followers_counters(user)
-      end
-
       {:ok, user}
     else
       %User{} = user -> {:ok, user}
@@ -1112,27 +1108,4 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
     data
     |> maybe_fix_user_url
   end
-
-  def update_following_followers_counters(user) do
-    info = %{}
-
-    following = fetch_counter(user.following_address)
-    info = if following, do: Map.put(info, :following_count, following), else: info
-
-    followers = fetch_counter(user.follower_address)
-    info = if followers, do: Map.put(info, :follower_count, followers), else: info
-
-    User.set_info_cache(user, info)
-  end
-
-  defp fetch_counter(url) do
-    with {:ok, %{body: body, status: code}} when code in 200..299 <-
-           Pleroma.HTTP.get(
-             url,
-             [{:Accept, "application/activity+json"}]
-           ),
-         {:ok, data} <- Jason.decode(body) do
-      data["totalItems"]
-    end
-  end
 end

lib/pleroma/web/activity_pub/utils.ex

@@ -244,20 +244,6 @@ defmodule Pleroma.Web.ActivityPub.Utils do
 
   def insert_full_object(map), do: {:ok, map, nil}
 
-  def update_object_in_activities(%{data: %{"id" => id}} = object) do
-    # TODO
-    # Update activities that already had this. Could be done in a seperate process.
-    # Alternatively, just don't do this and fetch the current object each time. Most
-    # could probably be taken from cache.
-    relevant_activities = Activity.get_all_create_by_object_ap_id(id)
-
-    Enum.map(relevant_activities, fn activity ->
-      new_activity_data = activity.data |> Map.put("object", object.data)
-      changeset = Changeset.change(activity, data: new_activity_data)
-      Repo.update(changeset)
-    end)
-  end
-
   #### Like-related helpers
 
   @doc """
@@ -340,8 +326,7 @@ defmodule Pleroma.Web.ActivityPub.Utils do
            |> Map.put("#{property}_count", length(element))
            |> Map.put("#{property}s", element),
          changeset <- Changeset.change(object, data: new_data),
-         {:ok, object} <- Object.update_and_set_cache(changeset),
-         _ <- update_object_in_activities(object) do
+         {:ok, object} <- Object.update_and_set_cache(changeset) do
       {:ok, object}
     end
   end

lib/pleroma/web/activity_pub/views/object_view.ex

@@ -66,8 +66,10 @@ defmodule Pleroma.Web.ActivityPub.ObjectView do
       "orderedItems" => items
     }
 
-    if offset < total do
+    if offset + length(items) < total do
       Map.put(map, "next", "#{iri}?page=#{page + 1}")
+    else
+      map
     end
   end
 end

lib/pleroma/web/activity_pub/views/user_view.ex

@@ -65,7 +65,7 @@ defmodule Pleroma.Web.ActivityPub.UserView do
     do: render("service.json", %{user: user})
 
   def render("user.json", %{user: %User{nickname: "internal." <> _} = user}),
-    do: render("service.json", %{user: user})
+    do: render("service.json", %{user: user}) |> Map.put("preferredUsername", user.nickname)
 
   def render("user.json", %{user: user}) do
     {:ok, user} = User.ensure_keys_present(user)

lib/pleroma/web/admin_api/admin_api_controller.ex

@@ -402,9 +402,9 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
       if Pleroma.Config.get([:instance, :dynamic_configuration]) do
         updated =
           Enum.map(configs, fn
-            %{"group" => group, "key" => key, "delete" => "true"} ->
-              {:ok, _} = Config.delete(%{group: group, key: key})
-              nil
+            %{"group" => group, "key" => key, "delete" => "true"} = params ->
+              {:ok, config} = Config.delete(%{group: group, key: key, subkeys: params["subkeys"]})
+              config
 
             %{"group" => group, "key" => key, "value" => value} ->
               {:ok, config} = Config.update_or_create(%{group: group, key: key, value: value})

lib/pleroma/web/admin_api/config.ex

@@ -55,8 +55,19 @@ defmodule Pleroma.Web.AdminAPI.Config do
 
   @spec delete(map()) :: {:ok, Config.t()} | {:error, Changeset.t()}
   def delete(params) do
-    with %Config{} = config <- Config.get_by_params(params) do
-      Repo.delete(config)
+    with %Config{} = config <- Config.get_by_params(Map.delete(params, :subkeys)) do
+      if params[:subkeys] do
+        updated_value =
+          Keyword.drop(
+            :erlang.binary_to_term(config.value),
+            Enum.map(params[:subkeys], &do_transform_string(&1))
+          )
+
+        Config.update(config, %{value: updated_value})
+      else
+        Repo.delete(config)
+        {:ok, nil}
+      end
     else
       nil ->
         err =

lib/pleroma/web/common_api/utils.ex

@@ -47,26 +47,43 @@ defmodule Pleroma.Web.CommonAPI.Utils do
 
   def get_replied_to_activity(_), do: nil
 
-  def attachments_from_ids(data) do
-    if Map.has_key?(data, "descriptions") do
-      attachments_from_ids_descs(data["media_ids"], data["descriptions"])
-    else
-      attachments_from_ids_no_descs(data["media_ids"])
-    end
+  def attachments_from_ids(%{"media_ids" => ids, "descriptions" => desc} = _) do
+    attachments_from_ids_descs(ids, desc)
   end
 
-  def attachments_from_ids_no_descs(ids) do
-    Enum.map(ids || [], fn media_id ->
-      Repo.get(Object, media_id).data
-    end)
+  def attachments_from_ids(%{"media_ids" => ids} = _) do
+    attachments_from_ids_no_descs(ids)
   end
 
+  def attachments_from_ids(_), do: []
+
+  def attachments_from_ids_no_descs([]), do: []
+
+  def attachments_from_ids_no_descs(ids) do
+    Enum.map(ids, fn media_id ->
+      case Repo.get(Object, media_id) do
+        %Object{data: data} = _ -> data
+        _ -> nil
+      end
+    end)
+    |> Enum.filter(& &1)
+  end
+
+  def attachments_from_ids_descs([], _), do: []
+
   def attachments_from_ids_descs(ids, descs_str) do
     {_, descs} = Jason.decode(descs_str)
 
-    Enum.map(ids || [], fn media_id ->
-      Map.put(Repo.get(Object, media_id).data, "name", descs[media_id])
+    Enum.map(ids, fn media_id ->
+      case Repo.get(Object, media_id) do
+        %Object{data: data} = _ ->
+          Map.put(data, "name", descs[media_id])
+
+        _ ->
+          nil
+      end
     end)
+    |> Enum.filter(& &1)
   end
 
   @spec get_to_and_cc(User.t(), list(String.t()), Activity.t() | nil, String.t()) ::
@@ -247,20 +264,18 @@ defmodule Pleroma.Web.CommonAPI.Utils do
   end
 
   def add_attachments(text, attachments) do
-    attachment_text =
-      Enum.map(attachments, fn
-        %{"url" => [%{"href" => href} | _]} = attachment ->
-          name = attachment["name"] || URI.decode(Path.basename(href))
-          href = MediaProxy.url(href)
-          "<a href=\"#{href}\" class='attachment'>#{shortname(name)}</a>"
-
-        _ ->
-          ""
-      end)
-
+    attachment_text = Enum.map(attachments, &build_attachment_link/1)
     Enum.join([text | attachment_text], "<br>")
   end
 
+  defp build_attachment_link(%{"url" => [%{"href" => href} | _]} = attachment) do
+    name = attachment["name"] || URI.decode(Path.basename(href))
+    href = MediaProxy.url(href)
+    "<a href=\"#{href}\" class='attachment'>#{shortname(name)}</a>"
+  end
+
+  defp build_attachment_link(_), do: ""
+
   def format_input(text, format, options \\ [])
 
   @doc """
@@ -320,7 +335,7 @@ defmodule Pleroma.Web.CommonAPI.Utils do
         sensitive \\ false,
         merge \\ %{}
       ) do
-    object = %{
+    %{
       "type" => "Note",
       "to" => to,
       "cc" => cc,
@@ -330,18 +345,20 @@ defmodule Pleroma.Web.CommonAPI.Utils do
       "context" => context,
       "attachment" => attachments,
       "actor" => actor,
-      "tag" => tags |> Enum.map(fn {_, tag} -> tag end) |> Enum.uniq()
+      "tag" => Keyword.values(tags) |> Enum.uniq()
     }
+    |> add_in_reply_to(in_reply_to)
+    |> Map.merge(merge)
+  end
 
-    object =
-      with false <- is_nil(in_reply_to),
-           %Object{} = in_reply_to_object <- Object.normalize(in_reply_to) do
-        Map.put(object, "inReplyTo", in_reply_to_object.data["id"])
-      else
-        _ -> object
-      end
+  defp add_in_reply_to(object, nil), do: object
 
-    Map.merge(object, merge)
+  defp add_in_reply_to(object, in_reply_to) do
+    with %Object{} = in_reply_to_object <- Object.normalize(in_reply_to) do
+      Map.put(object, "inReplyTo", in_reply_to_object.data["id"])
+    else
+      _ -> object
+    end
   end
 
   def format_naive_asctime(date) do
@@ -373,17 +390,16 @@ defmodule Pleroma.Web.CommonAPI.Utils do
     |> String.replace(~r/(\.\d+)?$/, ".000Z", global: false)
   end
 
-  def to_masto_date(date) do
-    try do
-      date
-      |> NaiveDateTime.from_iso8601!()
-      |> NaiveDateTime.to_iso8601()
-      |> String.replace(~r/(\.\d+)?$/, ".000Z", global: false)
-    rescue
-      _e -> ""
+  def to_masto_date(date) when is_binary(date) do
+    with {:ok, date} <- NaiveDateTime.from_iso8601(date) do
+      to_masto_date(date)
+    else
+      _ -> ""
     end
   end
 
+  def to_masto_date(_), do: ""
+
   defp shortname(name) do
     if String.length(name) < 30 do
       name
@@ -428,7 +444,7 @@ defmodule Pleroma.Web.CommonAPI.Utils do
 
     object_data =
       cond do
-        !is_nil(object) ->
+        not is_nil(object) ->
           object.data
 
         is_map(data["object"]) ->
@@ -472,9 +488,9 @@ defmodule Pleroma.Web.CommonAPI.Utils do
 
   def maybe_extract_mentions(%{"tag" => tag}) do
     tag
-    |> Enum.filter(fn x -> is_map(x) end)
-    |> Enum.filter(fn x -> x["type"] == "Mention" end)
+    |> Enum.filter(fn x -> is_map(x) && x["type"] == "Mention" end)
     |> Enum.map(fn x -> x["href"] end)
+    |> Enum.uniq()
   end
 
   def maybe_extract_mentions(_), do: []

lib/pleroma/web/mailer/subscription_controller.ex

@@ -0,0 +1,20 @@
+defmodule Pleroma.Web.Mailer.SubscriptionController do
+  use Pleroma.Web, :controller
+
+  alias Pleroma.JWT
+  alias Pleroma.Repo
+  alias Pleroma.User
+
+  def unsubscribe(conn, %{"token" => encoded_token}) do
+    with {:ok, token} <- Base.decode64(encoded_token),
+         {:ok, claims} <- JWT.verify_and_validate(token),
+         %{"act" => %{"unsubscribe" => type}, "sub" => uid} <- claims,
+         %User{} = user <- Repo.get(User, uid),
+         {:ok, _user} <- User.switch_email_notifications(user, type, false) do
+      render(conn, "unsubscribe_success.html", email: user.email)
+    else
+      _err ->
+        render(conn, "unsubscribe_failure.html")
+    end
+  end
+end

lib/pleroma/web/mastodon_api/views/account_view.ex

@@ -28,7 +28,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do
       id: to_string(user.id),
       acct: user.nickname,
       username: username_from_nickname(user.nickname),
-      url: user.ap_id
+      url: User.profile_url(user)
     }
   end
 
@@ -106,7 +106,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do
       following_count: user_info.following_count,
       statuses_count: user_info.note_count,
       note: bio || "",
-      url: user.ap_id,
+      url: User.profile_url(user),
       avatar: image,
       avatar_static: image,
       header: header,

lib/pleroma/web/media_proxy/media_proxy.ex

@@ -4,6 +4,7 @@
 
 defmodule Pleroma.Web.MediaProxy do
   alias Pleroma.Config
+  alias Pleroma.Upload
   alias Pleroma.Web
 
   @base64_opts [padding: false]
@@ -26,7 +27,18 @@ defmodule Pleroma.Web.MediaProxy do
   defp whitelisted?(url) do
     %{host: domain} = URI.parse(url)
 
-    Enum.any?(Config.get([:media_proxy, :whitelist]), fn pattern ->
+    mediaproxy_whitelist = Config.get([:media_proxy, :whitelist])
+
+    upload_base_url_domain =
+      if !is_nil(Config.get([Upload, :base_url])) do
+        [URI.parse(Config.get([Upload, :base_url])).host]
+      else
+        []
+      end
+
+    whitelist = mediaproxy_whitelist ++ upload_base_url_domain
+
+    Enum.any?(whitelist, fn pattern ->
       String.equivalent?(domain, pattern)
     end)
   end

lib/pleroma/web/ostatus/activity_representer.ex

@@ -183,6 +183,7 @@ defmodule Pleroma.Web.OStatus.ActivityRepresenter do
     author = if with_author, do: [{:author, UserRepresenter.to_simple_form(user)}], else: []
 
     retweeted_activity = Activity.get_create_by_object_ap_id(activity.data["object"])
+    retweeted_object = Object.normalize(retweeted_activity)
     retweeted_user = User.get_cached_by_ap_id(retweeted_activity.data["actor"])
 
     retweeted_xml = to_simple_form(retweeted_activity, retweeted_user, true)
@@ -197,7 +198,7 @@ defmodule Pleroma.Web.OStatus.ActivityRepresenter do
       {:"activity:verb", ['http://activitystrea.ms/schema/1.0/share']},
       {:id, h.(activity.data["id"])},
       {:title, ['#{user.nickname} repeated a notice']},
-      {:content, [type: 'html'], ['RT #{retweeted_activity.data["object"]["content"]}']},
+      {:content, [type: 'html'], ['RT #{retweeted_object.data["content"]}']},
       {:published, h.(inserted_at)},
       {:updated, h.(updated_at)},
       {:"ostatus:conversation", [ref: h.(activity.data["context"])],

lib/pleroma/web/rich_media/parsers/twitter_card.ex

@@ -3,13 +3,20 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.RichMedia.Parsers.TwitterCard do
+  alias Pleroma.Web.RichMedia.Parsers.MetaTagsParser
+
+  @spec parse(String.t(), map()) :: {:ok, map()} | {:error, String.t()}
   def parse(html, data) do
-    Pleroma.Web.RichMedia.Parsers.MetaTagsParser.parse(
-      html,
-      data,
-      "twitter",
-      "No twitter card metadata found",
-      "name"
-    )
+    data
+    |> parse_name_attrs(html)
+    |> parse_property_attrs(html)
+  end
+
+  defp parse_name_attrs(data, html) do
+    MetaTagsParser.parse(html, data, "twitter", %{}, "name")
+  end
+
+  defp parse_property_attrs({_, data}, html) do
+    MetaTagsParser.parse(html, data, "twitter", "No twitter card metadata found", "property")
   end
 end

lib/pleroma/web/router.ex

@@ -608,6 +608,8 @@ defmodule Pleroma.Web.Router do
     post("/push/hub/:nickname", Websub.WebsubController, :websub_subscription_request)
     get("/push/subscriptions/:id", Websub.WebsubController, :websub_subscription_confirmation)
     post("/push/subscriptions/:id", Websub.WebsubController, :websub_incoming)
+
+    get("/mailer/unsubscribe/:token", Mailer.SubscriptionController, :unsubscribe)
   end
 
   pipeline :activitypub do

lib/pleroma/web/templates/email/digest.html.eex

@@ -0,0 +1,20 @@
+<h1>Hey <%= @user.nickname %>, here is what you've missed!</h1>
+
+<h2>New Mentions:</h2>
+<ul>
+<%= for %{data: mention, object: object, from: from} <- @mentions do %>
+  <li><%= link from.nickname, to: mention.activity.actor %>: <%= raw object.data["content"] %></li>
+<% end %>
+</ul>
+
+<%= if @followers != [] do %>
+<h2><%= length(@followers) %> New Followers:</h2>
+<ul>
+<%= for %{data: follow, from: from} <- @followers do %>
+  <li><%= link from.nickname, to: follow.activity.actor %></li>
+<% end %>
+</ul>
+<% end %>
+
+<p>You have received this email because you have signed up to receive digest emails from <b><%= @instance %></b> Pleroma instance.</p>
+<p>The email address you are subscribed as is <%= @user.email %>. To unsubscribe, please go <%= link "here", to: @unsubscribe_link %>.</p>

lib/pleroma/web/templates/layout/app.html.eex

@@ -36,6 +36,11 @@
         margin-bottom: 20px;
       }
 
+      a {
+        color: color: #d8a070;
+        text-decoration: none;
+      }
+
       form {
         width: 100%;
       }

lib/pleroma/web/templates/layout/email.html.eex

@@ -0,0 +1,10 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="utf-8">
+    <title><%= @email.subject %></title>
+  </head>
+  <body>
+    <%= render @view_module, @view_template, assigns %>
+  </body>
+</html>

lib/pleroma/web/templates/mailer/subscription/unsubscribe_failure.html.eex

@@ -0,0 +1 @@
+<h1>UNSUBSCRIBE FAILURE</h1>

lib/pleroma/web/templates/mailer/subscription/unsubscribe_success.html.eex

@@ -0,0 +1 @@
+<h1>UNSUBSCRIBE SUCCESSFUL</h1>

lib/pleroma/web/views/email_view.ex

@@ -0,0 +1,5 @@
+defmodule Pleroma.Web.EmailView do
+  use Pleroma.Web, :view
+  import Phoenix.HTML
+  import Phoenix.HTML.Link
+end

lib/pleroma/web/views/mailer/subscription_view.ex

@@ -0,0 +1,3 @@
+defmodule Pleroma.Web.Mailer.SubscriptionView do
+  use Pleroma.Web, :view
+end