Merge remote-tracking branch 'remotes/upstream/develop' into 1149-oban-job-queue

lib/pleroma/web/activity_pub/activity_pub.ex

@@ -267,6 +267,9 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     else
       {:fake, true, activity} ->
         {:ok, activity}
+
+      {:error, message} ->
+        {:error, message}
     end
   end
 
@@ -746,8 +749,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
 
   defp restrict_favorited_by(query, %{"favorited_by" => ap_id}) do
     from(
-      activity in query,
-      where: fragment(~s(? <@ (? #> '{"object","likes"}'\)), ^ap_id, activity.data)
+      [_activity, object] in query,
+      where: fragment("(?)->'likes' \\? (?)", object.data, ^ap_id)
     )
   end
 
@@ -1009,10 +1012,10 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     user_data = %{
       ap_id: data["id"],
       info: %{
-        "ap_enabled" => true,
-        "source_data" => data,
-        "banner" => banner,
-        "locked" => locked
+        ap_enabled: true,
+        source_data: data,
+        banner: banner,
+        locked: locked
       },
       avatar: avatar,
       name: data["name"],
@@ -1036,6 +1039,71 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     {:ok, user_data}
   end
 
+  def fetch_follow_information_for_user(user) do
+    with {:ok, following_data} <-
+           Fetcher.fetch_and_contain_remote_object_from_id(user.following_address),
+         following_count when is_integer(following_count) <- following_data["totalItems"],
+         {:ok, hide_follows} <- collection_private(following_data),
+         {:ok, followers_data} <-
+           Fetcher.fetch_and_contain_remote_object_from_id(user.follower_address),
+         followers_count when is_integer(followers_count) <- followers_data["totalItems"],
+         {:ok, hide_followers} <- collection_private(followers_data) do
+      {:ok,
+       %{
+         hide_follows: hide_follows,
+         follower_count: followers_count,
+         following_count: following_count,
+         hide_followers: hide_followers
+       }}
+    else
+      {:error, _} = e ->
+        e
+
+      e ->
+        {:error, e}
+    end
+  end
+
+  defp maybe_update_follow_information(data) do
+    with {:enabled, true} <-
+           {:enabled, Pleroma.Config.get([:instance, :external_user_synchronization])},
+         {:ok, info} <- fetch_follow_information_for_user(data) do
+      info = Map.merge(data.info, info)
+      Map.put(data, :info, info)
+    else
+      {:enabled, false} ->
+        data
+
+      e ->
+        Logger.error(
+          "Follower/Following counter update for #{data.ap_id} failed.\n" <> inspect(e)
+        )
+
+        data
+    end
+  end
+
+  defp collection_private(data) do
+    if is_map(data["first"]) and
+         data["first"]["type"] in ["CollectionPage", "OrderedCollectionPage"] do
+      {:ok, false}
+    else
+      with {:ok, %{"type" => type}} when type in ["CollectionPage", "OrderedCollectionPage"] <-
+             Fetcher.fetch_and_contain_remote_object_from_id(data["first"]) do
+        {:ok, false}
+      else
+        {:error, {:ok, %{status: code}}} when code in [401, 403] ->
+          {:ok, true}
+
+        {:error, _} = e ->
+          e
+
+        e ->
+          {:error, e}
+      end
+    end
+  end
+
   def user_data_from_user_object(data) do
     with {:ok, data} <- MRF.filter(data),
          {:ok, data} <- object_to_user_data(data) do
@@ -1047,7 +1115,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
 
   def fetch_and_prepare_user_from_ap_id(ap_id) do
     with {:ok, data} <- Fetcher.fetch_and_contain_remote_object_from_id(ap_id),
-         {:ok, data} <- user_data_from_user_object(data) do
+         {:ok, data} <- user_data_from_user_object(data),
+         data <- maybe_update_follow_information(data) do
       {:ok, data}
     else
       e -> Logger.error("Could not decode user at fetch #{ap_id}, #{inspect(e)}")

lib/pleroma/web/activity_pub/transmogrifier.ex

@@ -608,13 +608,13 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
     with %User{ap_id: ^actor_id} = actor <- User.get_cached_by_ap_id(object["id"]) do
       {:ok, new_user_data} = ActivityPub.user_data_from_user_object(object)
 
-      banner = new_user_data[:info]["banner"]
-      locked = new_user_data[:info]["locked"] || false
+      banner = new_user_data[:info][:banner]
+      locked = new_user_data[:info][:locked] || false
 
       update_data =
         new_user_data
         |> Map.take([:name, :bio, :avatar])
-        |> Map.put(:info, %{"banner" => banner, "locked" => locked})
+        |> Map.put(:info, %{banner: banner, locked: locked})
 
       actor
       |> User.upgrade_changeset(update_data)
@@ -1076,10 +1076,6 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
         PleromaJobQueue.enqueue(:transmogrifier, __MODULE__, [:user_upgrade, user])
       end
 
-      if Pleroma.Config.get([:instance, :external_user_synchronization]) do
-        update_following_followers_counters(user)
-      end
-
       {:ok, user}
     else
       %User{} = user -> {:ok, user}
@@ -1112,27 +1108,4 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
     data
     |> maybe_fix_user_url
   end
-
-  def update_following_followers_counters(user) do
-    info = %{}
-
-    following = fetch_counter(user.following_address)
-    info = if following, do: Map.put(info, :following_count, following), else: info
-
-    followers = fetch_counter(user.follower_address)
-    info = if followers, do: Map.put(info, :follower_count, followers), else: info
-
-    User.set_info_cache(user, info)
-  end
-
-  defp fetch_counter(url) do
-    with {:ok, %{body: body, status: code}} when code in 200..299 <-
-           Pleroma.HTTP.get(
-             url,
-             [{:Accept, "application/activity+json"}]
-           ),
-         {:ok, data} <- Jason.decode(body) do
-      data["totalItems"]
-    end
-  end
 end

lib/pleroma/web/activity_pub/utils.ex

@@ -244,20 +244,6 @@ defmodule Pleroma.Web.ActivityPub.Utils do
 
   def insert_full_object(map), do: {:ok, map, nil}
 
-  def update_object_in_activities(%{data: %{"id" => id}} = object) do
-    # TODO
-    # Update activities that already had this. Could be done in a seperate process.
-    # Alternatively, just don't do this and fetch the current object each time. Most
-    # could probably be taken from cache.
-    relevant_activities = Activity.get_all_create_by_object_ap_id(id)
-
-    Enum.map(relevant_activities, fn activity ->
-      new_activity_data = activity.data |> Map.put("object", object.data)
-      changeset = Changeset.change(activity, data: new_activity_data)
-      Repo.update(changeset)
-    end)
-  end
-
   #### Like-related helpers
 
   @doc """
@@ -340,8 +326,7 @@ defmodule Pleroma.Web.ActivityPub.Utils do
            |> Map.put("#{property}_count", length(element))
            |> Map.put("#{property}s", element),
          changeset <- Changeset.change(object, data: new_data),
-         {:ok, object} <- Object.update_and_set_cache(changeset),
-         _ <- update_object_in_activities(object) do
+         {:ok, object} <- Object.update_and_set_cache(changeset) do
       {:ok, object}
     end
   end

lib/pleroma/web/activity_pub/views/object_view.ex

@@ -66,8 +66,10 @@ defmodule Pleroma.Web.ActivityPub.ObjectView do
       "orderedItems" => items
     }
 
-    if offset < total do
+    if offset + length(items) < total do
       Map.put(map, "next", "#{iri}?page=#{page + 1}")
+    else
+      map
     end
   end
 end

lib/pleroma/web/activity_pub/views/user_view.ex

@@ -65,7 +65,7 @@ defmodule Pleroma.Web.ActivityPub.UserView do
     do: render("service.json", %{user: user})
 
   def render("user.json", %{user: %User{nickname: "internal." <> _} = user}),
-    do: render("service.json", %{user: user})
+    do: render("service.json", %{user: user}) |> Map.put("preferredUsername", user.nickname)
 
   def render("user.json", %{user: user}) do
     {:ok, user} = User.ensure_keys_present(user)

lib/pleroma/web/admin_api/admin_api_controller.ex

@@ -402,9 +402,9 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
       if Pleroma.Config.get([:instance, :dynamic_configuration]) do
         updated =
           Enum.map(configs, fn
-            %{"group" => group, "key" => key, "delete" => "true"} ->
-              {:ok, _} = Config.delete(%{group: group, key: key})
-              nil
+            %{"group" => group, "key" => key, "delete" => "true"} = params ->
+              {:ok, config} = Config.delete(%{group: group, key: key, subkeys: params["subkeys"]})
+              config
 
             %{"group" => group, "key" => key, "value" => value} ->
               {:ok, config} = Config.update_or_create(%{group: group, key: key, value: value})

lib/pleroma/web/admin_api/config.ex

@@ -55,8 +55,19 @@ defmodule Pleroma.Web.AdminAPI.Config do
 
   @spec delete(map()) :: {:ok, Config.t()} | {:error, Changeset.t()}
   def delete(params) do
-    with %Config{} = config <- Config.get_by_params(params) do
-      Repo.delete(config)
+    with %Config{} = config <- Config.get_by_params(Map.delete(params, :subkeys)) do
+      if params[:subkeys] do
+        updated_value =
+          Keyword.drop(
+            :erlang.binary_to_term(config.value),
+            Enum.map(params[:subkeys], &do_transform_string(&1))
+          )
+
+        Config.update(config, %{value: updated_value})
+      else
+        Repo.delete(config)
+        {:ok, nil}
+      end
     else
       nil ->
         err =

lib/pleroma/web/common_api/utils.ex

@@ -47,26 +47,43 @@ defmodule Pleroma.Web.CommonAPI.Utils do
 
   def get_replied_to_activity(_), do: nil
 
-  def attachments_from_ids(data) do
-    if Map.has_key?(data, "descriptions") do
-      attachments_from_ids_descs(data["media_ids"], data["descriptions"])
-    else
-      attachments_from_ids_no_descs(data["media_ids"])
-    end
+  def attachments_from_ids(%{"media_ids" => ids, "descriptions" => desc} = _) do
+    attachments_from_ids_descs(ids, desc)
   end
 
-  def attachments_from_ids_no_descs(ids) do
-    Enum.map(ids || [], fn media_id ->
-      Repo.get(Object, media_id).data
-    end)
+  def attachments_from_ids(%{"media_ids" => ids} = _) do
+    attachments_from_ids_no_descs(ids)
   end
 
+  def attachments_from_ids(_), do: []
+
+  def attachments_from_ids_no_descs([]), do: []
+
+  def attachments_from_ids_no_descs(ids) do
+    Enum.map(ids, fn media_id ->
+      case Repo.get(Object, media_id) do
+        %Object{data: data} = _ -> data
+        _ -> nil
+      end
+    end)
+    |> Enum.filter(& &1)
+  end
+
+  def attachments_from_ids_descs([], _), do: []
+
   def attachments_from_ids_descs(ids, descs_str) do
     {_, descs} = Jason.decode(descs_str)
 
-    Enum.map(ids || [], fn media_id ->
-      Map.put(Repo.get(Object, media_id).data, "name", descs[media_id])
+    Enum.map(ids, fn media_id ->
+      case Repo.get(Object, media_id) do
+        %Object{data: data} = _ ->
+          Map.put(data, "name", descs[media_id])
+
+        _ ->
+          nil
+      end
     end)
+    |> Enum.filter(& &1)
   end
 
   @spec get_to_and_cc(User.t(), list(String.t()), Activity.t() | nil, String.t()) ::
@@ -247,20 +264,18 @@ defmodule Pleroma.Web.CommonAPI.Utils do
   end
 
   def add_attachments(text, attachments) do
-    attachment_text =
-      Enum.map(attachments, fn
-        %{"url" => [%{"href" => href} | _]} = attachment ->
-          name = attachment["name"] || URI.decode(Path.basename(href))
-          href = MediaProxy.url(href)
-          "<a href=\"#{href}\" class='attachment'>#{shortname(name)}</a>"
-
-        _ ->
-          ""
-      end)
-
+    attachment_text = Enum.map(attachments, &build_attachment_link/1)
     Enum.join([text | attachment_text], "<br>")
   end
 
+  defp build_attachment_link(%{"url" => [%{"href" => href} | _]} = attachment) do
+    name = attachment["name"] || URI.decode(Path.basename(href))
+    href = MediaProxy.url(href)
+    "<a href=\"#{href}\" class='attachment'>#{shortname(name)}</a>"
+  end
+
+  defp build_attachment_link(_), do: ""
+
   def format_input(text, format, options \\ [])
 
   @doc """
@@ -320,7 +335,7 @@ defmodule Pleroma.Web.CommonAPI.Utils do
         sensitive \\ false,
         merge \\ %{}
       ) do
-    object = %{
+    %{
       "type" => "Note",
       "to" => to,
       "cc" => cc,
@@ -330,18 +345,20 @@ defmodule Pleroma.Web.CommonAPI.Utils do
       "context" => context,
       "attachment" => attachments,
       "actor" => actor,
-      "tag" => tags |> Enum.map(fn {_, tag} -> tag end) |> Enum.uniq()
+      "tag" => Keyword.values(tags) |> Enum.uniq()
     }
+    |> add_in_reply_to(in_reply_to)
+    |> Map.merge(merge)
+  end
 
-    object =
-      with false <- is_nil(in_reply_to),
-           %Object{} = in_reply_to_object <- Object.normalize(in_reply_to) do
-        Map.put(object, "inReplyTo", in_reply_to_object.data["id"])
-      else
-        _ -> object
-      end
+  defp add_in_reply_to(object, nil), do: object
 
-    Map.merge(object, merge)
+  defp add_in_reply_to(object, in_reply_to) do
+    with %Object{} = in_reply_to_object <- Object.normalize(in_reply_to) do
+      Map.put(object, "inReplyTo", in_reply_to_object.data["id"])
+    else
+      _ -> object
+    end
   end
 
   def format_naive_asctime(date) do
@@ -373,17 +390,16 @@ defmodule Pleroma.Web.CommonAPI.Utils do
     |> String.replace(~r/(\.\d+)?$/, ".000Z", global: false)
   end
 
-  def to_masto_date(date) do
-    try do
-      date
-      |> NaiveDateTime.from_iso8601!()
-      |> NaiveDateTime.to_iso8601()
-      |> String.replace(~r/(\.\d+)?$/, ".000Z", global: false)
-    rescue
-      _e -> ""
+  def to_masto_date(date) when is_binary(date) do
+    with {:ok, date} <- NaiveDateTime.from_iso8601(date) do
+      to_masto_date(date)
+    else
+      _ -> ""
     end
   end
 
+  def to_masto_date(_), do: ""
+
   defp shortname(name) do
     if String.length(name) < 30 do
       name
@@ -428,7 +444,7 @@ defmodule Pleroma.Web.CommonAPI.Utils do
 
     object_data =
       cond do
-        !is_nil(object) ->
+        not is_nil(object) ->
           object.data
 
         is_map(data["object"]) ->
@@ -472,9 +488,9 @@ defmodule Pleroma.Web.CommonAPI.Utils do
 
   def maybe_extract_mentions(%{"tag" => tag}) do
     tag
-    |> Enum.filter(fn x -> is_map(x) end)
-    |> Enum.filter(fn x -> x["type"] == "Mention" end)
+    |> Enum.filter(fn x -> is_map(x) && x["type"] == "Mention" end)
     |> Enum.map(fn x -> x["href"] end)
+    |> Enum.uniq()
   end
 
   def maybe_extract_mentions(_), do: []

lib/pleroma/web/mailer/subscription_controller.ex

@@ -0,0 +1,20 @@
+defmodule Pleroma.Web.Mailer.SubscriptionController do
+  use Pleroma.Web, :controller
+
+  alias Pleroma.JWT
+  alias Pleroma.Repo
+  alias Pleroma.User
+
+  def unsubscribe(conn, %{"token" => encoded_token}) do
+    with {:ok, token} <- Base.decode64(encoded_token),
+         {:ok, claims} <- JWT.verify_and_validate(token),
+         %{"act" => %{"unsubscribe" => type}, "sub" => uid} <- claims,
+         %User{} = user <- Repo.get(User, uid),
+         {:ok, _user} <- User.switch_email_notifications(user, type, false) do
+      render(conn, "unsubscribe_success.html", email: user.email)
+    else
+      _err ->
+        render(conn, "unsubscribe_failure.html")
+    end
+  end
+end

lib/pleroma/web/mastodon_api/views/account_view.ex

@@ -28,7 +28,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do
       id: to_string(user.id),
       acct: user.nickname,
       username: username_from_nickname(user.nickname),
-      url: user.ap_id
+      url: User.profile_url(user)
     }
   end
 
@@ -106,7 +106,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do
       following_count: user_info.following_count,
       statuses_count: user_info.note_count,
       note: bio || "",
-      url: user.ap_id,
+      url: User.profile_url(user),
       avatar: image,
       avatar_static: image,
       header: header,

lib/pleroma/web/media_proxy/media_proxy.ex

@@ -4,6 +4,7 @@
 
 defmodule Pleroma.Web.MediaProxy do
   alias Pleroma.Config
+  alias Pleroma.Upload
   alias Pleroma.Web
 
   @base64_opts [padding: false]
@@ -26,7 +27,18 @@ defmodule Pleroma.Web.MediaProxy do
   defp whitelisted?(url) do
     %{host: domain} = URI.parse(url)
 
-    Enum.any?(Config.get([:media_proxy, :whitelist]), fn pattern ->
+    mediaproxy_whitelist = Config.get([:media_proxy, :whitelist])
+
+    upload_base_url_domain =
+      if !is_nil(Config.get([Upload, :base_url])) do
+        [URI.parse(Config.get([Upload, :base_url])).host]
+      else
+        []
+      end
+
+    whitelist = mediaproxy_whitelist ++ upload_base_url_domain
+
+    Enum.any?(whitelist, fn pattern ->
       String.equivalent?(domain, pattern)
     end)
   end

lib/pleroma/web/ostatus/activity_representer.ex

@@ -183,6 +183,7 @@ defmodule Pleroma.Web.OStatus.ActivityRepresenter do
     author = if with_author, do: [{:author, UserRepresenter.to_simple_form(user)}], else: []
 
     retweeted_activity = Activity.get_create_by_object_ap_id(activity.data["object"])
+    retweeted_object = Object.normalize(retweeted_activity)
     retweeted_user = User.get_cached_by_ap_id(retweeted_activity.data["actor"])
 
     retweeted_xml = to_simple_form(retweeted_activity, retweeted_user, true)
@@ -197,7 +198,7 @@ defmodule Pleroma.Web.OStatus.ActivityRepresenter do
       {:"activity:verb", ['http://activitystrea.ms/schema/1.0/share']},
       {:id, h.(activity.data["id"])},
       {:title, ['#{user.nickname} repeated a notice']},
-      {:content, [type: 'html'], ['RT #{retweeted_activity.data["object"]["content"]}']},
+      {:content, [type: 'html'], ['RT #{retweeted_object.data["content"]}']},
       {:published, h.(inserted_at)},
       {:updated, h.(updated_at)},
       {:"ostatus:conversation", [ref: h.(activity.data["context"])],

lib/pleroma/web/rich_media/parsers/twitter_card.ex

@@ -3,13 +3,20 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.RichMedia.Parsers.TwitterCard do
+  alias Pleroma.Web.RichMedia.Parsers.MetaTagsParser
+
+  @spec parse(String.t(), map()) :: {:ok, map()} | {:error, String.t()}
   def parse(html, data) do
-    Pleroma.Web.RichMedia.Parsers.MetaTagsParser.parse(
-      html,
-      data,
-      "twitter",
-      "No twitter card metadata found",
-      "name"
-    )
+    data
+    |> parse_name_attrs(html)
+    |> parse_property_attrs(html)
+  end
+
+  defp parse_name_attrs(data, html) do
+    MetaTagsParser.parse(html, data, "twitter", %{}, "name")
+  end
+
+  defp parse_property_attrs({_, data}, html) do
+    MetaTagsParser.parse(html, data, "twitter", "No twitter card metadata found", "property")
   end
 end

lib/pleroma/web/router.ex

@@ -608,6 +608,8 @@ defmodule Pleroma.Web.Router do
     post("/push/hub/:nickname", Websub.WebsubController, :websub_subscription_request)
     get("/push/subscriptions/:id", Websub.WebsubController, :websub_subscription_confirmation)
     post("/push/subscriptions/:id", Websub.WebsubController, :websub_incoming)
+
+    get("/mailer/unsubscribe/:token", Mailer.SubscriptionController, :unsubscribe)
   end
 
   pipeline :activitypub do

lib/pleroma/web/templates/email/digest.html.eex

@@ -0,0 +1,20 @@
+<h1>Hey <%= @user.nickname %>, here is what you've missed!</h1>
+
+<h2>New Mentions:</h2>
+<ul>
+<%= for %{data: mention, object: object, from: from} <- @mentions do %>
+  <li><%= link from.nickname, to: mention.activity.actor %>: <%= raw object.data["content"] %></li>
+<% end %>
+</ul>
+
+<%= if @followers != [] do %>
+<h2><%= length(@followers) %> New Followers:</h2>
+<ul>
+<%= for %{data: follow, from: from} <- @followers do %>
+  <li><%= link from.nickname, to: follow.activity.actor %></li>
+<% end %>
+</ul>
+<% end %>
+
+<p>You have received this email because you have signed up to receive digest emails from <b><%= @instance %></b> Pleroma instance.</p>
+<p>The email address you are subscribed as is <%= @user.email %>. To unsubscribe, please go <%= link "here", to: @unsubscribe_link %>.</p>

lib/pleroma/web/templates/layout/app.html.eex

@@ -36,6 +36,11 @@
         margin-bottom: 20px;
       }
 
+      a {
+        color: color: #d8a070;
+        text-decoration: none;
+      }
+
       form {
         width: 100%;
       }

lib/pleroma/web/templates/layout/email.html.eex

@@ -0,0 +1,10 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="utf-8">
+    <title><%= @email.subject %></title>
+  </head>
+  <body>
+    <%= render @view_module, @view_template, assigns %>
+  </body>
+</html>

lib/pleroma/web/templates/mailer/subscription/unsubscribe_failure.html.eex

@@ -0,0 +1 @@
+<h1>UNSUBSCRIBE FAILURE</h1>

lib/pleroma/web/templates/mailer/subscription/unsubscribe_success.html.eex

@@ -0,0 +1 @@
+<h1>UNSUBSCRIBE SUCCESSFUL</h1>

lib/pleroma/web/views/email_view.ex

@@ -0,0 +1,5 @@
+defmodule Pleroma.Web.EmailView do
+  use Pleroma.Web, :view
+  import Phoenix.HTML
+  import Phoenix.HTML.Link
+end

lib/pleroma/web/views/mailer/subscription_view.ex

@@ -0,0 +1,3 @@
+defmodule Pleroma.Web.Mailer.SubscriptionView do
+  use Pleroma.Web, :view
+end