From 40c58d3383001a63a787f1e0956aeaad902afb64 Mon Sep 17 00:00:00 2001 From: link0ff Date: Wed, 20 Mar 2019 13:28:06 +0200 Subject: [PATCH 01/55] Ignore compilation warnings for undefined module :eldap --- mix.exs | 1 + 1 file changed, 1 insertion(+) diff --git a/mix.exs b/mix.exs index efdf15d3a..352620e63 100644 --- a/mix.exs +++ b/mix.exs @@ -9,6 +9,7 @@ defmodule Pleroma.Mixfile do elixirc_paths: elixirc_paths(Mix.env()), compilers: [:phoenix, :gettext] ++ Mix.compilers(), elixirc_options: [warnings_as_errors: true], + xref: [exclude: [:eldap]], start_permanent: Mix.env() == :prod, aliases: aliases(), deps: deps(), From 8468f3f6d48693d2a27a257e5555aa71decff3df Mon Sep 17 00:00:00 2001 From: lain Date: Wed, 20 Mar 2019 21:09:36 +0100 Subject: [PATCH 02/55] Add safe dm mode option. --- config/config.exs | 3 ++- docs/config.md | 3 ++- lib/pleroma/formatter.ex | 20 ++++++++++++++++--- lib/pleroma/web/common_api/common_api.ex | 3 ++- lib/pleroma/web/common_api/utils.ex | 12 ++++++++++-- test/formatter_test.exs | 25 ++++++++++++++++++++++++ test/web/common_api/common_api_test.exs | 18 +++++++++++++++++ 7 files changed, 76 insertions(+), 8 deletions(-) diff --git a/config/config.exs b/config/config.exs index ccdd35777..b01c097c5 100644 --- a/config/config.exs +++ b/config/config.exs @@ -174,7 +174,8 @@ config :pleroma, :instance, no_attachment_links: false, welcome_user_nickname: nil, welcome_message: nil, - max_report_comment_size: 1000 + max_report_comment_size: 1000, + safe_dm_mentions: false config :pleroma, :markup, # XXX - unfortunately, inline images must be enabled by default right now, because diff --git a/docs/config.md b/docs/config.md index 201180373..78967204b 100644 --- a/docs/config.md +++ b/docs/config.md @@ -101,7 +101,8 @@ config :pleroma, Pleroma.Mailer, * `no_attachment_links`: Set to true to disable automatically adding attachment link text to statuses * `welcome_message`: A message that will be send to a newly registered users as a direct message. * `welcome_user_nickname`: The nickname of the local user that sends the welcome message. -* `max_report_size`: The maximum size of the report comment (Default: `1000`) +* `max_report_comment_size`: The maximum size of the report comment (Default: `1000`) +* `safe_dm_mentions`: If set to true, only mentions at the beginning of a post will be used to address people in direct messages. This is to prevent accidental mentioning of people when talking about them (e.g. "@friend hey i really don't like @enemy"). (Default: `false`) ## :logger * `backends`: `:console` is used to send logs to stdout, `{ExSyslogger, :ex_syslogger}` to log to syslog diff --git a/lib/pleroma/formatter.ex b/lib/pleroma/formatter.ex index 1e4ede3f2..e3625383b 100644 --- a/lib/pleroma/formatter.ex +++ b/lib/pleroma/formatter.ex @@ -8,6 +8,7 @@ defmodule Pleroma.Formatter do alias Pleroma.User alias Pleroma.Web.MediaProxy + @safe_mention_regex ~r/^(\s*(?@.+?\s+)+)(?.*)/ @markdown_characters_regex ~r/(`|\*|_|{|}|[|]|\(|\)|#|\+|-|\.|!)/ @link_regex ~r{((?:http(s)?:\/\/)?[\w.-]+(?:\.[\w\.-]+)+[\w\-\._~%:/?#[\]@!\$&'\(\)\*\+,;=.]+)|[0-9a-z+\-\.]+:[0-9a-z$-_.+!*'(),]+}ui # credo:disable-for-previous-line Credo.Check.Readability.MaxLineLength @@ -45,15 +46,28 @@ defmodule Pleroma.Formatter do @doc """ Parses a text and replace plain text links with HTML. Returns a tuple with a result text, mentions, and hashtags. + + If the 'safe_mention' option is given, only consecutive mentions at the start the post are actually mentioned. """ @spec linkify(String.t(), keyword()) :: {String.t(), [{String.t(), User.t()}], [{String.t(), String.t()}]} def linkify(text, options \\ []) do options = options ++ @auto_linker_config - acc = %{mentions: MapSet.new(), tags: MapSet.new()} - {text, %{mentions: mentions, tags: tags}} = AutoLinker.link_map(text, acc, options) - {text, MapSet.to_list(mentions), MapSet.to_list(tags)} + if options[:safe_mention] && Regex.named_captures(@safe_mention_regex, text) do + %{"mentions" => mentions, "rest" => rest} = Regex.named_captures(@safe_mention_regex, text) + acc = %{mentions: MapSet.new(), tags: MapSet.new()} + + {text_mentions, %{mentions: mentions}} = AutoLinker.link_map(mentions, acc, options) + {text_rest, %{tags: tags}} = AutoLinker.link_map(rest, acc, options) + + {text_mentions <> text_rest, MapSet.to_list(mentions), MapSet.to_list(tags)} + else + acc = %{mentions: MapSet.new(), tags: MapSet.new()} + {text, %{mentions: mentions, tags: tags}} = AutoLinker.link_map(text, acc, options) + + {text, MapSet.to_list(mentions), MapSet.to_list(tags)} + end end def emojify(text) do diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex index b5f79c3bf..50d60aade 100644 --- a/lib/pleroma/web/common_api/common_api.ex +++ b/lib/pleroma/web/common_api/common_api.ex @@ -142,7 +142,8 @@ defmodule Pleroma.Web.CommonAPI do make_content_html( status, attachments, - data + data, + visibility ), {to, cc} <- to_for_user_and_mentions(user, mentions, in_reply_to, visibility), context <- make_context(in_reply_to), diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex index b7513ef28..368945418 100644 --- a/lib/pleroma/web/common_api/utils.ex +++ b/lib/pleroma/web/common_api/utils.ex @@ -101,7 +101,8 @@ defmodule Pleroma.Web.CommonAPI.Utils do def make_content_html( status, attachments, - data + data, + visibility ) do no_attachment_links = data @@ -110,8 +111,15 @@ defmodule Pleroma.Web.CommonAPI.Utils do content_type = get_content_type(data["content_type"]) + options = + if visibility == "direct" && Config.get([:instance, :safe_dm_mentions]) do + [safe_mention: true] + else + [] + end + status - |> format_input(content_type) + |> format_input(content_type, options) |> maybe_add_attachments(attachments, no_attachment_links) |> maybe_add_nsfw_tag(data) end diff --git a/test/formatter_test.exs b/test/formatter_test.exs index 7d8864bf4..fcdf931b7 100644 --- a/test/formatter_test.exs +++ b/test/formatter_test.exs @@ -181,6 +181,31 @@ defmodule Pleroma.FormatterTest do expected_text = "@a hi" assert {^expected_text, [] = _mentions, [] = _tags} = Formatter.linkify(text) end + + test "given the 'safe_mention' option, it will only mention people in the beginning" do + user = insert(:user) + _other_user = insert(:user) + third_user = insert(:user) + text = " @#{user.nickname} hey dude i hate @#{third_user.nickname}" + {expected_text, mentions, [] = _tags} = Formatter.linkify(text, safe_mention: true) + + assert mentions == [{"@#{user.nickname}", user}] + + assert expected_text == + "@#{user.nickname} hey dude i hate @#{third_user.nickname}" + end + + test "given the 'safe_mention' option, it will still work without any mention" do + text = "A post without any mention" + {expected_text, mentions, [] = _tags} = Formatter.linkify(text, safe_mention: true) + + assert mentions == [] + assert expected_text == text + end end describe ".parse_tags" do diff --git a/test/web/common_api/common_api_test.exs b/test/web/common_api/common_api_test.exs index f83f80b40..34aa5bf18 100644 --- a/test/web/common_api/common_api_test.exs +++ b/test/web/common_api/common_api_test.exs @@ -10,6 +10,24 @@ defmodule Pleroma.Web.CommonAPITest do import Pleroma.Factory + test "with the safe_dm_mention option set, it does not mention people beyond the initial tags" do + har = insert(:user) + jafnhar = insert(:user) + tridi = insert(:user) + option = Pleroma.Config.get([:instance, :safe_dm_mentions]) + Pleroma.Config.put([:instance, :safe_dm_mentions], true) + + {:ok, activity} = + CommonAPI.post(har, %{ + "status" => "@#{jafnhar.nickname} hey, i never want to see @#{tridi.nickname} again", + "visibility" => "direct" + }) + + refute tridi.ap_id in activity.recipients + assert jafnhar.ap_id in activity.recipients + Pleroma.Config.put([:instance, :safe_dm_mentions], option) + end + test "it de-duplicates tags" do user = insert(:user) {:ok, activity} = CommonAPI.post(user, %{"status" => "#2hu #2HU"}) From f858df819bf619bada7468f48b945cfab7f52ad0 Mon Sep 17 00:00:00 2001 From: link0ff Date: Thu, 21 Mar 2019 12:31:16 +0200 Subject: [PATCH 03/55] Skip tests in LDAPAuthorizationTest if :eldap module is unavailable --- test/web/oauth/ldap_authorization_test.exs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/web/oauth/ldap_authorization_test.exs b/test/web/oauth/ldap_authorization_test.exs index 5bf7eb93c..0eb191c76 100644 --- a/test/web/oauth/ldap_authorization_test.exs +++ b/test/web/oauth/ldap_authorization_test.exs @@ -10,6 +10,8 @@ defmodule Pleroma.Web.OAuth.LDAPAuthorizationTest do import ExUnit.CaptureLog import Mock + @skip if !Code.ensure_loaded?(:eldap), do: :skip + setup_all do ldap_authenticator = Pleroma.Config.get(Pleroma.Web.Auth.Authenticator, Pleroma.Web.Auth.PleromaAuthenticator) @@ -27,6 +29,7 @@ defmodule Pleroma.Web.OAuth.LDAPAuthorizationTest do :ok end + @tag @skip test "authorizes the existing user using LDAP credentials" do password = "testpassword" user = insert(:user, password_hash: Comeonin.Pbkdf2.hashpwsalt(password)) @@ -65,6 +68,7 @@ defmodule Pleroma.Web.OAuth.LDAPAuthorizationTest do end end + @tag @skip test "creates a new user after successful LDAP authorization" do password = "testpassword" user = build(:user) @@ -110,6 +114,7 @@ defmodule Pleroma.Web.OAuth.LDAPAuthorizationTest do end end + @tag @skip test "falls back to the default authorization when LDAP is unavailable" do password = "testpassword" user = insert(:user, password_hash: Comeonin.Pbkdf2.hashpwsalt(password)) @@ -153,6 +158,7 @@ defmodule Pleroma.Web.OAuth.LDAPAuthorizationTest do end end + @tag @skip test "disallow authorization for wrong LDAP credentials" do password = "testpassword" user = insert(:user, password_hash: Comeonin.Pbkdf2.hashpwsalt(password)) From bf27190f7f0942a05de518f2085a299eb011614c Mon Sep 17 00:00:00 2001 From: lain Date: Thu, 21 Mar 2019 16:16:26 +0100 Subject: [PATCH 04/55] UtilController: Return state of safe dm mentions. --- .../controllers/util_controller.ex | 4 +++- test/web/twitter_api/util_controller_test.exs | 23 +++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/twitter_api/controllers/util_controller.ex b/lib/pleroma/web/twitter_api/controllers/util_controller.ex index 320ec778c..faa733fec 100644 --- a/lib/pleroma/web/twitter_api/controllers/util_controller.ex +++ b/lib/pleroma/web/twitter_api/controllers/util_controller.ex @@ -197,7 +197,9 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do vapidPublicKey: vapid_public_key, accountActivationRequired: if(Keyword.get(instance, :account_activation_required, false), do: "1", else: "0"), - invitesEnabled: if(Keyword.get(instance, :invites_enabled, false), do: "1", else: "0") + invitesEnabled: if(Keyword.get(instance, :invites_enabled, false), do: "1", else: "0"), + safeDMMentionsEnabled: + if(Pleroma.Config.get([:instance, :safe_dm_mentions]), do: "1", else: "0") } pleroma_fe = diff --git a/test/web/twitter_api/util_controller_test.exs b/test/web/twitter_api/util_controller_test.exs index 6e8a25056..832fdc096 100644 --- a/test/web/twitter_api/util_controller_test.exs +++ b/test/web/twitter_api/util_controller_test.exs @@ -75,6 +75,29 @@ defmodule Pleroma.Web.TwitterAPI.UtilControllerTest do end describe "GET /api/statusnet/config.json" do + test "returns the state of safe_dm_mentions flag", %{conn: conn} do + option = Pleroma.Config.get([:instance, :safe_dm_mentions]) + Pleroma.Config.put([:instance, :safe_dm_mentions], true) + + response = + conn + |> get("/api/statusnet/config.json") + |> json_response(:ok) + + assert response["site"]["safeDMMentionsEnabled"] == "1" + + Pleroma.Config.put([:instance, :safe_dm_mentions], false) + + response = + conn + |> get("/api/statusnet/config.json") + |> json_response(:ok) + + assert response["site"]["safeDMMentionsEnabled"] == "0" + + Pleroma.Config.put([:instance, :safe_dm_mentions], option) + end + test "it returns the managed config", %{conn: conn} do Pleroma.Config.put([:instance, :managed_config], false) Pleroma.Config.put([:fe], theme: "rei-ayanami-towel") From 88096c65a525a44c845e651fc082f6374fad6042 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Thu, 21 Mar 2019 23:16:32 +0300 Subject: [PATCH 05/55] Move gluing search results from application to database and get mutuals a higher score multiplier --- lib/pleroma/user.ex | 83 +++++++++++++++++++++------------------------ 1 file changed, 39 insertions(+), 44 deletions(-) diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index eb0933c85..e16141edc 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -823,31 +823,53 @@ defmodule Pleroma.User do if resolve, do: get_or_fetch(query) - fts_results = do_search(fts_search_subquery(query), for_user) - - {:ok, trigram_results} = + {:ok, results} = Repo.transaction(fn -> Ecto.Adapters.SQL.query(Repo, "select set_limit(0.25)", []) - do_search(trigram_search_subquery(query), for_user) + Repo.all(search_query(query, for_user)) end) - Enum.uniq_by(fts_results ++ trigram_results, & &1.id) + results end - defp do_search(subquery, for_user, options \\ []) do - q = - from( - s in subquery(subquery), - order_by: [desc: s.search_rank], - limit: ^(options[:limit] || 20) - ) + def search_query(query, for_user) do + fts_subquery = fts_search_subquery(query) + trigram_subquery = trigram_search_subquery(query) + union_query = from(s in trigram_subquery, union: ^fts_subquery) + distinct_query = from(s in subquery(union_query), distinct: s.id) - results = - q - |> Repo.all() - |> Enum.filter(&(&1.search_rank > 0)) + from(s in subquery(boost_search_rank_query(distinct_query, for_user)), + order_by: [desc: s.search_rank], + limit: 20 + ) + end - boost_search_results(results, for_user) + defp boost_search_rank_query(query, nil), do: query + + defp boost_search_rank_query(query, for_user) do + friends_ids = get_friends_ids(for_user) + followers_ids = get_followers_ids(for_user) + + from(u in subquery(query), + select_merge: %{ + search_rank: + fragment( + """ + CASE WHEN (?) THEN (?) * 1.3 + WHEN (?) THEN (?) * 1.2 + WHEN (?) THEN (?) * 1.1 + ELSE (?) END + """, + u.id in ^friends_ids and u.id in ^followers_ids, + u.search_rank, + u.id in ^friends_ids, + u.search_rank, + u.id in ^followers_ids, + u.search_rank, + u.search_rank + ) + } + ) end defp fts_search_subquery(term, query \\ User) do @@ -906,33 +928,6 @@ defmodule Pleroma.User do ) end - defp boost_search_results(results, nil), do: results - - defp boost_search_results(results, for_user) do - friends_ids = get_friends_ids(for_user) - followers_ids = get_followers_ids(for_user) - - Enum.map( - results, - fn u -> - search_rank_coef = - cond do - u.id in friends_ids -> - 1.2 - - u.id in followers_ids -> - 1.1 - - true -> - 1 - end - - Map.put(u, :search_rank, u.search_rank * search_rank_coef) - end - ) - |> Enum.sort_by(&(-&1.search_rank)) - end - def blocks_import(%User{} = blocker, blocked_identifiers) when is_list(blocked_identifiers) do Enum.map( blocked_identifiers, From fea36967999fed5399ab3533e806e4cbc990ad05 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Thu, 21 Mar 2019 23:17:53 +0000 Subject: [PATCH 06/55] common api: move context functions from twitterapi --- lib/pleroma/web/common_api/utils.ex | 29 ++++++++++++++++++ lib/pleroma/web/twitter_api/twitter_api.ex | 30 ------------------- .../web/twitter_api/twitter_api_controller.ex | 3 +- .../web/twitter_api/views/activity_view.ex | 3 +- test/web/common_api/common_api_utils_test.exs | 16 ++++++++++ test/web/twitter_api/twitter_api_test.exs | 16 ---------- .../twitter_api/views/activity_view_test.exs | 9 +++--- 7 files changed, 52 insertions(+), 54 deletions(-) diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex index b7513ef28..fcdfea8e1 100644 --- a/lib/pleroma/web/common_api/utils.ex +++ b/lib/pleroma/web/common_api/utils.ex @@ -344,4 +344,33 @@ defmodule Pleroma.Web.CommonAPI.Utils do end def get_report_statuses(_, _), do: {:ok, nil} + + # DEPRECATED mostly, context objects are now created at insertion time. + def context_to_conversation_id(context) do + with %Object{id: id} <- Object.get_cached_by_ap_id(context) do + id + else + _e -> + changeset = Object.context_mapping(context) + + case Repo.insert(changeset) do + {:ok, %{id: id}} -> + id + + # This should be solved by an upsert, but it seems ecto + # has problems accessing the constraint inside the jsonb. + {:error, _} -> + Object.get_cached_by_ap_id(context).id + end + end + end + + def conversation_id_to_context(id) do + with %Object{data: %{"id" => context}} <- Repo.get(Object, id) do + context + else + _e -> + {:error, "No such conversation"} + end + end end diff --git a/lib/pleroma/web/twitter_api/twitter_api.ex b/lib/pleroma/web/twitter_api/twitter_api.ex index d57100491..9978c7f64 100644 --- a/lib/pleroma/web/twitter_api/twitter_api.ex +++ b/lib/pleroma/web/twitter_api/twitter_api.ex @@ -5,7 +5,6 @@ defmodule Pleroma.Web.TwitterAPI.TwitterAPI do alias Pleroma.Activity alias Pleroma.Mailer - alias Pleroma.Object alias Pleroma.Repo alias Pleroma.User alias Pleroma.UserEmail @@ -282,35 +281,6 @@ defmodule Pleroma.Web.TwitterAPI.TwitterAPI do _activities = Repo.all(q) end - # DEPRECATED mostly, context objects are now created at insertion time. - def context_to_conversation_id(context) do - with %Object{id: id} <- Object.get_cached_by_ap_id(context) do - id - else - _e -> - changeset = Object.context_mapping(context) - - case Repo.insert(changeset) do - {:ok, %{id: id}} -> - id - - # This should be solved by an upsert, but it seems ecto - # has problems accessing the constraint inside the jsonb. - {:error, _} -> - Object.get_cached_by_ap_id(context).id - end - end - end - - def conversation_id_to_context(id) do - with %Object{data: %{"id" => context}} <- Repo.get(Object, id) do - context - else - _e -> - {:error, "No such conversation"} - end - end - def get_external_profile(for_user, uri) do with %User{} = user <- User.get_or_fetch(uri) do {:ok, UserView.render("show.json", %{user: user, for: for_user})} diff --git a/lib/pleroma/web/twitter_api/twitter_api_controller.ex b/lib/pleroma/web/twitter_api/twitter_api_controller.ex index 6ea0b110b..62cce18dc 100644 --- a/lib/pleroma/web/twitter_api/twitter_api_controller.ex +++ b/lib/pleroma/web/twitter_api/twitter_api_controller.ex @@ -16,6 +16,7 @@ defmodule Pleroma.Web.TwitterAPI.Controller do alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.ActivityPub.Visibility alias Pleroma.Web.CommonAPI + alias Pleroma.Web.CommonAPI.Utils alias Pleroma.Web.OAuth.Token alias Pleroma.Web.TwitterAPI.ActivityView alias Pleroma.Web.TwitterAPI.NotificationView @@ -278,7 +279,7 @@ defmodule Pleroma.Web.TwitterAPI.Controller do end def fetch_conversation(%{assigns: %{user: user}} = conn, %{"id" => id}) do - with context when is_binary(context) <- TwitterAPI.conversation_id_to_context(id), + with context when is_binary(context) <- Utils.conversation_id_to_context(id), activities <- ActivityPub.fetch_activities_for_context(context, %{ "blocking_user" => user, diff --git a/lib/pleroma/web/twitter_api/views/activity_view.ex b/lib/pleroma/web/twitter_api/views/activity_view.ex index 4926f007e..fe7d49975 100644 --- a/lib/pleroma/web/twitter_api/views/activity_view.ex +++ b/lib/pleroma/web/twitter_api/views/activity_view.ex @@ -15,7 +15,6 @@ defmodule Pleroma.Web.TwitterAPI.ActivityView do alias Pleroma.Web.MastodonAPI.StatusView alias Pleroma.Web.TwitterAPI.ActivityView alias Pleroma.Web.TwitterAPI.Representers.ObjectRepresenter - alias Pleroma.Web.TwitterAPI.TwitterAPI alias Pleroma.Web.TwitterAPI.UserView import Ecto.Query @@ -78,7 +77,7 @@ defmodule Pleroma.Web.TwitterAPI.ActivityView do defp get_context_id(%{data: %{"context" => context}}, options) do cond do id = options[:context_ids][context] -> id - true -> TwitterAPI.context_to_conversation_id(context) + true -> Utils.context_to_conversation_id(context) end end diff --git a/test/web/common_api/common_api_utils_test.exs b/test/web/common_api/common_api_utils_test.exs index 4c97b0d62..d095762ab 100644 --- a/test/web/common_api/common_api_utils_test.exs +++ b/test/web/common_api/common_api_utils_test.exs @@ -136,4 +136,20 @@ defmodule Pleroma.Web.CommonAPI.UtilsTest do assert output == expected end end + + describe "context_to_conversation_id" do + test "creates a mapping object" do + conversation_id = Utils.context_to_conversation_id("random context") + object = Object.get_by_ap_id("random context") + + assert conversation_id == object.id + end + + test "returns an existing mapping for an existing object" do + {:ok, object} = Object.context_mapping("random context") |> Repo.insert() + conversation_id = Utils.context_to_conversation_id("random context") + + assert conversation_id == object.id + end + end end diff --git a/test/web/twitter_api/twitter_api_test.exs b/test/web/twitter_api/twitter_api_test.exs index c8dd3fd7a..b823bfd68 100644 --- a/test/web/twitter_api/twitter_api_test.exs +++ b/test/web/twitter_api/twitter_api_test.exs @@ -445,22 +445,6 @@ defmodule Pleroma.Web.TwitterAPI.TwitterAPITest do :ok end - describe "context_to_conversation_id" do - test "creates a mapping object" do - conversation_id = TwitterAPI.context_to_conversation_id("random context") - object = Object.get_by_ap_id("random context") - - assert conversation_id == object.id - end - - test "returns an existing mapping for an existing object" do - {:ok, object} = Object.context_mapping("random context") |> Repo.insert() - conversation_id = TwitterAPI.context_to_conversation_id("random context") - - assert conversation_id == object.id - end - end - describe "fetching a user by uri" do test "fetches a user by uri" do id = "https://mastodon.social/users/lambadalambda" diff --git a/test/web/twitter_api/views/activity_view_test.exs b/test/web/twitter_api/views/activity_view_test.exs index d9df01c6e..ed18a60a3 100644 --- a/test/web/twitter_api/views/activity_view_test.exs +++ b/test/web/twitter_api/views/activity_view_test.exs @@ -12,7 +12,6 @@ defmodule Pleroma.Web.TwitterAPI.ActivityViewTest do alias Pleroma.Web.CommonAPI alias Pleroma.Web.CommonAPI.Utils alias Pleroma.Web.TwitterAPI.ActivityView - alias Pleroma.Web.TwitterAPI.TwitterAPI alias Pleroma.Web.TwitterAPI.UserView import Pleroma.Factory @@ -129,7 +128,7 @@ defmodule Pleroma.Web.TwitterAPI.ActivityViewTest do result = ActivityView.render("activity.json", activity: activity) - convo_id = TwitterAPI.context_to_conversation_id(activity.data["object"]["context"]) + convo_id = Utils.context_to_conversation_id(activity.data["object"]["context"]) expected = %{ "activity_type" => "post", @@ -177,7 +176,7 @@ defmodule Pleroma.Web.TwitterAPI.ActivityViewTest do other_user = insert(:user, %{nickname: "shp"}) {:ok, activity} = CommonAPI.post(user, %{"status" => "Hey @shp!"}) - convo_id = TwitterAPI.context_to_conversation_id(activity.data["object"]["context"]) + convo_id = Utils.context_to_conversation_id(activity.data["object"]["context"]) mocks = [ { @@ -197,7 +196,7 @@ defmodule Pleroma.Web.TwitterAPI.ActivityViewTest do assert result["statusnet_conversation_id"] == convo_id assert result["user"] - refute called(TwitterAPI.context_to_conversation_id(:_)) + refute called(Utils.context_to_conversation_id(:_)) refute called(User.get_cached_by_ap_id(user.ap_id)) refute called(User.get_cached_by_ap_id(other_user.ap_id)) end @@ -280,7 +279,7 @@ defmodule Pleroma.Web.TwitterAPI.ActivityViewTest do {:ok, activity} = CommonAPI.post(user, %{"status" => "Hey @shp!"}) {:ok, announce, _object} = CommonAPI.repeat(activity.id, other_user) - convo_id = TwitterAPI.context_to_conversation_id(activity.data["object"]["context"]) + convo_id = Utils.context_to_conversation_id(activity.data["object"]["context"]) activity = Repo.get(Activity, activity.id) From 3cc2554fa3d63ba22dc5f598229a02b928b9fd14 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Thu, 21 Mar 2019 23:25:41 +0000 Subject: [PATCH 07/55] mastodon api: add conversation_id extension (ref #674) --- lib/pleroma/web/mastodon_api/views/status_view.ex | 11 ++++++++++- test/web/mastodon_api/status_view_test.exs | 6 +++++- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index 209119dd5..1ca8338cc 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -46,6 +46,14 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do end end + defp get_context_id(%{data: %{"context_id" => context_id}}) when not is_nil(context_id), + do: context_id + + defp get_context_id(%{data: %{"context" => context}}) when is_binary(context), + do: Utils.context_to_conversation_id(context) + + defp get_context_id(_), do: nil + def render("index.json", opts) do replied_to_activities = get_replied_to_activities(opts.activities) @@ -186,7 +194,8 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do language: nil, emojis: build_emojis(activity.data["object"]["emoji"]), pleroma: %{ - local: activity.local + local: activity.local, + conversation_id: get_context_id(activity) } } end diff --git a/test/web/mastodon_api/status_view_test.exs b/test/web/mastodon_api/status_view_test.exs index ade0ca9f9..e1c9b2c8f 100644 --- a/test/web/mastodon_api/status_view_test.exs +++ b/test/web/mastodon_api/status_view_test.exs @@ -9,6 +9,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusViewTest do alias Pleroma.User alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.CommonAPI + alias Pleroma.Web.CommonAPI.Utils alias Pleroma.Web.MastodonAPI.AccountView alias Pleroma.Web.MastodonAPI.StatusView alias Pleroma.Web.OStatus @@ -72,6 +73,8 @@ defmodule Pleroma.Web.MastodonAPI.StatusViewTest do note = insert(:note_activity) user = User.get_cached_by_ap_id(note.data["actor"]) + convo_id = Utils.context_to_conversation_id(note.data["object"]["context"]) + status = StatusView.render("status.json", %{activity: note}) created_at = @@ -122,7 +125,8 @@ defmodule Pleroma.Web.MastodonAPI.StatusViewTest do } ], pleroma: %{ - local: true + local: true, + conversation_id: convo_id } } From ae8fa5d0aab3561bf66506766e2beb03a251e499 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Thu, 21 Mar 2019 23:27:42 +0000 Subject: [PATCH 08/55] docs: document `conversation_id` extension --- docs/Differences-in-MastodonAPI-Responses.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/Differences-in-MastodonAPI-Responses.md b/docs/Differences-in-MastodonAPI-Responses.md index 14b67ca7d..d993d1383 100644 --- a/docs/Differences-in-MastodonAPI-Responses.md +++ b/docs/Differences-in-MastodonAPI-Responses.md @@ -19,6 +19,7 @@ Adding the parameter `with_muted=true` to the timeline queries will also return Has these additional fields under the `pleroma` object: - `local`: true if the post was made on the local instance. +- `conversation_id`: the ID of the conversation the status is associated with (if any) ## Attachments From a223e65f35da158ef79f05f65316920dcecaa54b Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Thu, 21 Mar 2019 23:37:00 +0000 Subject: [PATCH 09/55] tests: fixup --- test/web/common_api/common_api_utils_test.exs | 1 + test/web/twitter_api/views/activity_view_test.exs | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/test/web/common_api/common_api_utils_test.exs b/test/web/common_api/common_api_utils_test.exs index d095762ab..e04b9f9b5 100644 --- a/test/web/common_api/common_api_utils_test.exs +++ b/test/web/common_api/common_api_utils_test.exs @@ -4,6 +4,7 @@ defmodule Pleroma.Web.CommonAPI.UtilsTest do alias Pleroma.Builders.UserBuilder + alias Pleroma.Object alias Pleroma.Web.CommonAPI.Utils alias Pleroma.Web.Endpoint use Pleroma.DataCase diff --git a/test/web/twitter_api/views/activity_view_test.exs b/test/web/twitter_api/views/activity_view_test.exs index ed18a60a3..a1776b3e6 100644 --- a/test/web/twitter_api/views/activity_view_test.exs +++ b/test/web/twitter_api/views/activity_view_test.exs @@ -180,8 +180,8 @@ defmodule Pleroma.Web.TwitterAPI.ActivityViewTest do mocks = [ { - TwitterAPI, - [], + Utils, + [:passthrough], [context_to_conversation_id: fn _ -> false end] }, { From 27e03a21773445375da7b0888ef517f14e6ad219 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Fri, 22 Mar 2019 01:17:14 +0000 Subject: [PATCH 10/55] reports: fix up email generation for remote reports --- lib/pleroma/emails/admin_email.ex | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/emails/admin_email.ex b/lib/pleroma/emails/admin_email.ex index 9b20c7e08..afefccec5 100644 --- a/lib/pleroma/emails/admin_email.ex +++ b/lib/pleroma/emails/admin_email.ex @@ -29,9 +29,13 @@ defmodule Pleroma.AdminEmail do if length(statuses) > 0 do statuses_list_html = statuses - |> Enum.map(fn %{id: id} -> - status_url = Helpers.o_status_url(Pleroma.Web.Endpoint, :notice, id) - "
  • #{status_url}
    • " + |> Enum.map(fn + %{id: id} -> + status_url = Helpers.o_status_url(Pleroma.Web.Endpoint, :notice, id) + "
  • #{status_url}
    • " + + id when is_binary(id) -> + "
  • #{id}
    • " end) |> Enum.join("\n") From 3229c7a1d689802f21a6efe648fcc56b8def3aa0 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Fri, 22 Mar 2019 08:39:49 +0300 Subject: [PATCH 11/55] Ensure fts is prefered over trigram and use union_all instead of union in user search query --- lib/pleroma/user.ex | 8 ++++++-- test/user_test.exs | 15 ++++++++++++--- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index e16141edc..f2ef0a838 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -50,6 +50,7 @@ defmodule Pleroma.User do field(:local, :boolean, default: true) field(:follower_address, :string) field(:search_rank, :float, virtual: true) + field(:search_type, :integer, virtual: true) field(:tags, {:array, :string}, default: []) field(:bookmarks, {:array, :string}, default: []) field(:last_refreshed_at, :naive_datetime_usec) @@ -835,8 +836,8 @@ defmodule Pleroma.User do def search_query(query, for_user) do fts_subquery = fts_search_subquery(query) trigram_subquery = trigram_search_subquery(query) - union_query = from(s in trigram_subquery, union: ^fts_subquery) - distinct_query = from(s in subquery(union_query), distinct: s.id) + union_query = from(s in trigram_subquery, union_all: ^fts_subquery) + distinct_query = from(s in subquery(union_query), order_by: s.search_type, distinct: s.id) from(s in subquery(boost_search_rank_query(distinct_query, for_user)), order_by: [desc: s.search_rank], @@ -884,6 +885,7 @@ defmodule Pleroma.User do from( u in query, select_merge: %{ + search_type: ^0, search_rank: fragment( """ @@ -916,6 +918,8 @@ defmodule Pleroma.User do from( u in User, select_merge: %{ + # ^1 gives 'Postgrex expected a binary, got 1' for some weird reason + search_type: fragment("?", 1), search_rank: fragment( "similarity(?, trim(? || ' ' || coalesce(?, '')))", diff --git a/test/user_test.exs b/test/user_test.exs index 1f54f3e30..442599910 100644 --- a/test/user_test.exs +++ b/test/user_test.exs @@ -879,7 +879,11 @@ defmodule Pleroma.UserTest do user = insert(:user, %{nickname: "john"}) Enum.each(["john", "jo", "j"], fn query -> - assert user == User.search(query) |> List.first() |> Map.put(:search_rank, nil) + assert user == + User.search(query) + |> List.first() + |> Map.put(:search_rank, nil) + |> Map.put(:search_type, nil) end) end @@ -887,7 +891,11 @@ defmodule Pleroma.UserTest do user = insert(:user, %{name: "John Doe"}) Enum.each(["John Doe", "JOHN", "doe", "j d", "j", "d"], fn query -> - assert user == User.search(query) |> List.first() |> Map.put(:search_rank, nil) + assert user == + User.search(query) + |> List.first() + |> Map.put(:search_rank, nil) + |> Map.put(:search_type, nil) end) end @@ -941,6 +949,7 @@ defmodule Pleroma.UserTest do User.search("lain@pleroma.soykaf.com") |> List.first() |> Map.put(:search_rank, nil) + |> Map.put(:search_type, nil) end test "does not yield false-positive matches" do @@ -958,7 +967,7 @@ defmodule Pleroma.UserTest do user = User.get_by_ap_id("http://mastodon.example.org/users/admin") assert length(results) == 1 - assert user == result |> Map.put(:search_rank, nil) + assert user == result |> Map.put(:search_rank, nil) |> Map.put(:search_type, nil) end end From e2afce34b68d88ec8e1ff624fe0a245fd0726fee Mon Sep 17 00:00:00 2001 From: lain Date: Fri, 22 Mar 2019 11:57:20 +0100 Subject: [PATCH 12/55] NodeInfo: Return safe_dm_mentions feature flag. --- .../web/nodeinfo/nodeinfo_controller.ex | 3 +++ test/web/node_info_test.exs | 23 +++++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/lib/pleroma/web/nodeinfo/nodeinfo_controller.ex b/lib/pleroma/web/nodeinfo/nodeinfo_controller.ex index 8c775ce24..216a962bd 100644 --- a/lib/pleroma/web/nodeinfo/nodeinfo_controller.ex +++ b/lib/pleroma/web/nodeinfo/nodeinfo_controller.ex @@ -124,6 +124,9 @@ defmodule Pleroma.Web.Nodeinfo.NodeinfoController do end, if Keyword.get(instance, :allow_relay) do "relay" + end, + if Keyword.get(instance, :safe_dm_mentions) do + "safe_dm_mentions" end ] |> Enum.filter(& &1) diff --git a/test/web/node_info_test.exs b/test/web/node_info_test.exs index 038feecc1..2fc42b7cc 100644 --- a/test/web/node_info_test.exs +++ b/test/web/node_info_test.exs @@ -108,4 +108,27 @@ defmodule Pleroma.Web.NodeInfoTest do assert result = json_response(conn, 200) assert Pleroma.Application.repository() == result["software"]["repository"] end + + test "it returns the safe_dm_mentions feature if enabled", %{conn: conn} do + option = Pleroma.Config.get([:instance, :safe_dm_mentions]) + Pleroma.Config.put([:instance, :safe_dm_mentions], true) + + response = + conn + |> get("/nodeinfo/2.1.json") + |> json_response(:ok) + + assert "safe_dm_mentions" in response["metadata"]["features"] + + Pleroma.Config.put([:instance, :safe_dm_mentions], false) + + response = + conn + |> get("/nodeinfo/2.1.json") + |> json_response(:ok) + + refute "safe_dm_mentions" in response["metadata"]["features"] + + Pleroma.Config.put([:instance, :safe_dm_mentions], option) + end end From 1b33986bfa3e924d49e0b37228979ed5f8f82d8c Mon Sep 17 00:00:00 2001 From: rinpatch Date: Sat, 23 Mar 2019 00:10:50 +0300 Subject: [PATCH 13/55] Fix text being nullable in TwitterAPI --- lib/pleroma/web/twitter_api/views/activity_view.ex | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/pleroma/web/twitter_api/views/activity_view.ex b/lib/pleroma/web/twitter_api/views/activity_view.ex index fe7d49975..aa1d41fa2 100644 --- a/lib/pleroma/web/twitter_api/views/activity_view.ex +++ b/lib/pleroma/web/twitter_api/views/activity_view.ex @@ -266,6 +266,8 @@ defmodule Pleroma.Web.TwitterAPI.ActivityView do content |> String.replace(~r//, "\n") |> HTML.get_cached_stripped_html_for_object(activity, __MODULE__) + else + "" end reply_parent = Activity.get_in_reply_to_activity(activity) From 62bccddde02edbf825c1806805cc9d7d7c9a0f13 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Fri, 22 Mar 2019 23:34:47 +0000 Subject: [PATCH 14/55] object: add support for preloading objects when walking an activity graph in normal form --- lib/pleroma/activity.ex | 62 ++++++++++++++++++++++++++++++++++++++++- lib/pleroma/object.ex | 25 +++++++++++++++++ 2 files changed, 86 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex index 04c3bb673..2f91b3c77 100644 --- a/lib/pleroma/activity.ex +++ b/lib/pleroma/activity.ex @@ -7,6 +7,7 @@ defmodule Pleroma.Activity do alias Pleroma.Activity alias Pleroma.Notification + alias Pleroma.Object alias Pleroma.Repo import Ecto.Query @@ -33,6 +34,18 @@ defmodule Pleroma.Activity do field(:recipients, {:array, :string}) has_many(:notifications, Notification, on_delete: :delete_all) + # Attention: this is a fake relation, don't try to preload it blindly and expect it to work! + # The foreign key is embedded in a jsonb field. + # + # To use it, you probably want to do an inner join and a preload: + # + # ``` + # |> join(:inner, [activity], o in Object, + # fragment("(?->>'id') = COALESCE((? -> 'object'::text) ->> 'id'::text)", o.data, activity.data)) + # |> preload([activity, object], [object: object]) + # ``` + has_one(:object, Object, on_delete: :nothing, foreign_key: :id) + timestamps() end @@ -49,6 +62,21 @@ defmodule Pleroma.Activity do Repo.get(Activity, id) end + def get_by_id_with_object(id) do + from(activity in Activity, + where: activity.id == ^id, + inner_join: o in Object, + on: + fragment( + "(?->>'id') = COALESCE((? -> 'object'::text) ->> 'id'::text)", + o.data, + activity.data + ), + preload: [object: o] + ) + |> Repo.one() + end + def by_object_ap_id(ap_id) do from( activity in Activity, @@ -76,7 +104,7 @@ defmodule Pleroma.Activity do ) end - def create_by_object_ap_id(ap_id) do + def create_by_object_ap_id(ap_id) when is_binary(ap_id) do from( activity in Activity, where: @@ -90,6 +118,8 @@ defmodule Pleroma.Activity do ) end + def create_by_object_ap_id(_), do: nil + def get_all_create_by_object_ap_id(ap_id) do Repo.all(create_by_object_ap_id(ap_id)) end @@ -101,6 +131,36 @@ defmodule Pleroma.Activity do def get_create_by_object_ap_id(_), do: nil + def create_by_object_ap_id_with_object(ap_id) when is_binary(ap_id) do + from( + activity in Activity, + where: + fragment( + "coalesce((?)->'object'->>'id', (?)->>'object') = ?", + activity.data, + activity.data, + ^to_string(ap_id) + ), + where: fragment("(?)->>'type' = 'Create'", activity.data), + inner_join: o in Object, + on: + fragment( + "(?->>'id') = COALESCE((? -> 'object'::text) ->> 'id'::text)", + o.data, + activity.data + ), + preload: [object: o] + ) + end + + def create_by_object_ap_id_with_object(_), do: nil + + def get_create_by_object_ap_id_with_object(ap_id) do + ap_id + |> create_by_object_ap_id_with_object() + |> Repo.one() + end + def normalize(obj) when is_map(obj), do: Activity.get_by_ap_id(obj["id"]) def normalize(ap_id) when is_binary(ap_id), do: Activity.get_by_ap_id(ap_id) def normalize(_), do: nil diff --git a/lib/pleroma/object.ex b/lib/pleroma/object.ex index 58e46ef1d..0f5c532ec 100644 --- a/lib/pleroma/object.ex +++ b/lib/pleroma/object.ex @@ -14,6 +14,8 @@ defmodule Pleroma.Object do import Ecto.Query import Ecto.Changeset + require Logger + schema "objects" do field(:data, :map) @@ -38,6 +40,29 @@ defmodule Pleroma.Object do Repo.one(from(object in Object, where: fragment("(?)->>'id' = ?", object.data, ^ap_id))) end + # If we pass an Activity to Object.normalize(), we can try to use the preloaded object. + # Use this whenever possible, especially when walking graphs in an O(N) loop! + def normalize(%Activity{object: %Object{} = object}), do: object + + # Catch and log Object.normalize() calls where the Activity's child object is not + # preloaded. + def normalize(%Activity{data: %{"object" => %{"id" => ap_id}}}) do + Logger.info( + "Object.normalize() called without preloaded object (#{ap_id}). Consider preloading the object!" + ) + + normalize(ap_id) + end + + def normalize(%Activity{data: %{"object" => ap_id}}) do + Logger.info( + "Object.normalize() called without preloaded object (#{ap_id}). Consider preloading the object!" + ) + + normalize(ap_id) + end + + # Old way, try fetching the object through cache. def normalize(%{"id" => ap_id}), do: normalize(ap_id) def normalize(ap_id) when is_binary(ap_id), do: get_cached_by_ap_id(ap_id) def normalize(_), do: nil From 092cedede507b3e89134deb6f5d09c5db339dfae Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sat, 23 Mar 2019 00:09:56 +0000 Subject: [PATCH 15/55] activity: add with_preloaded_object() convenience --- lib/pleroma/activity.ex | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex index 2f91b3c77..370721070 100644 --- a/lib/pleroma/activity.ex +++ b/lib/pleroma/activity.ex @@ -44,11 +44,29 @@ defmodule Pleroma.Activity do # fragment("(?->>'id') = COALESCE((? -> 'object'::text) ->> 'id'::text)", o.data, activity.data)) # |> preload([activity, object], [object: object]) # ``` + # + # As a convenience, Activity.with_preloaded_object() sets up an inner join and preload for the + # typical case. has_one(:object, Object, on_delete: :nothing, foreign_key: :id) timestamps() end + def with_preloaded_object(query) do + query + |> join( + :inner, + [activity], + o in Object, + fragment( + "(?->>'id') = COALESCE((? -> 'object'::text) ->> 'id'::text)", + o.data, + activity.data + ) + ) + |> preload([activity, object], object: object) + end + def get_by_ap_id(ap_id) do Repo.one( from( From 9aea7cc224c09e37a9a46277a9fbfb0af383fc0e Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sat, 23 Mar 2019 00:10:17 +0000 Subject: [PATCH 16/55] activitypub: preload child objects when fetching timelines --- lib/pleroma/web/activity_pub/activity_pub.ex | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 2470b4a71..7d38a46e5 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -716,6 +716,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do limit: 20, order_by: [fragment("? desc nulls last", activity.id)] ) + |> Activity.with_preloaded_object() base_query |> restrict_recipients(recipients, opts["user"]) From e75e43b949ab6f6fb5751eeff10644d04259c149 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sat, 23 Mar 2019 00:22:14 +0000 Subject: [PATCH 17/55] common api: use the optimized Object.normalize whenever possible --- lib/pleroma/web/common_api/common_api.ex | 13 ++++++------- lib/pleroma/web/common_api/utils.ex | 8 ++++---- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex index 50d60aade..8625f6621 100644 --- a/lib/pleroma/web/common_api/common_api.ex +++ b/lib/pleroma/web/common_api/common_api.ex @@ -6,7 +6,6 @@ defmodule Pleroma.Web.CommonAPI do alias Pleroma.Activity alias Pleroma.Formatter alias Pleroma.Object - alias Pleroma.Repo alias Pleroma.ThreadMute alias Pleroma.User alias Pleroma.Web.ActivityPub.ActivityPub @@ -64,8 +63,8 @@ defmodule Pleroma.Web.CommonAPI do end def delete(activity_id, user) do - with %Activity{data: %{"object" => %{"id" => object_id}}} <- Repo.get(Activity, activity_id), - %Object{} = object <- Object.normalize(object_id), + with %Activity{data: %{"object" => _}} = activity <- Activity.get_by_id_with_object(activity_id), + %Object{} = object <- Object.normalize(activity), true <- User.superuser?(user) || user.ap_id == object.data["actor"], {:ok, _} <- unpin(activity_id, user), {:ok, delete} <- ActivityPub.delete(object) do @@ -75,7 +74,7 @@ defmodule Pleroma.Web.CommonAPI do def repeat(id_or_ap_id, user) do with %Activity{} = activity <- get_by_id_or_ap_id(id_or_ap_id), - object <- Object.normalize(activity.data["object"]["id"]), + object <- Object.normalize(activity), nil <- Utils.get_existing_announce(user.ap_id, object) do ActivityPub.announce(user, object) else @@ -86,7 +85,7 @@ defmodule Pleroma.Web.CommonAPI do def unrepeat(id_or_ap_id, user) do with %Activity{} = activity <- get_by_id_or_ap_id(id_or_ap_id), - object <- Object.normalize(activity.data["object"]["id"]) do + object <- Object.normalize(activity) do ActivityPub.unannounce(user, object) else _ -> @@ -96,7 +95,7 @@ defmodule Pleroma.Web.CommonAPI do def favorite(id_or_ap_id, user) do with %Activity{} = activity <- get_by_id_or_ap_id(id_or_ap_id), - object <- Object.normalize(activity.data["object"]["id"]), + object <- Object.normalize(activity), nil <- Utils.get_existing_like(user.ap_id, object) do ActivityPub.like(user, object) else @@ -107,7 +106,7 @@ defmodule Pleroma.Web.CommonAPI do def unfavorite(id_or_ap_id, user) do with %Activity{} = activity <- get_by_id_or_ap_id(id_or_ap_id), - object <- Object.normalize(activity.data["object"]["id"]) do + object <- Object.normalize(activity) do ActivityPub.unlike(user, object) else _ -> diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex index 3e807a5b7..6a94c2d26 100644 --- a/lib/pleroma/web/common_api/utils.ex +++ b/lib/pleroma/web/common_api/utils.ex @@ -17,13 +17,13 @@ defmodule Pleroma.Web.CommonAPI.Utils do # This is a hack for twidere. def get_by_id_or_ap_id(id) do - activity = Repo.get(Activity, id) || Activity.get_create_by_object_ap_id(id) + activity = Activity.get_by_id_with_object(id) || Activity.get_create_by_object_ap_id_with_object(id) activity && if activity.data["type"] == "Create" do activity else - Activity.get_create_by_object_ap_id(activity.data["object"]) + Activity.get_create_by_object_ap_id_with_object(activity.data["object"]) end end @@ -302,10 +302,10 @@ defmodule Pleroma.Web.CommonAPI.Utils do def maybe_notify_mentioned_recipients( recipients, - %Activity{data: %{"to" => _to, "type" => type} = data} = _activity + %Activity{data: %{"to" => _to, "type" => type} = data} = activity ) when type == "Create" do - object = Object.normalize(data["object"]) + object = Object.normalize(activity) object_data = cond do From b3bf523c0967e27616b78a42ba50a6b4c432749e Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sat, 23 Mar 2019 00:22:57 +0000 Subject: [PATCH 18/55] rich media: use optimized Object.normalize() --- lib/pleroma/web/rich_media/helpers.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/rich_media/helpers.ex b/lib/pleroma/web/rich_media/helpers.ex index 92c61ff51..f97d1863c 100644 --- a/lib/pleroma/web/rich_media/helpers.ex +++ b/lib/pleroma/web/rich_media/helpers.ex @@ -23,7 +23,7 @@ defmodule Pleroma.Web.RichMedia.Helpers do def fetch_data_for_activity(%Activity{} = activity) do with true <- Pleroma.Config.get([:rich_media, :enabled]), - %Object{} = object <- Object.normalize(activity.data["object"]), + %Object{} = object <- Object.normalize(activity), {:ok, page_url} <- HTML.extract_first_external_url(object, object.data["content"]), :ok <- validate_page_url(page_url), {:ok, rich_media} <- Parser.parse(page_url) do From 59518cbcd8bcc0fe98f2d05ef88d3e2ff68a256f Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sat, 23 Mar 2019 00:24:23 +0000 Subject: [PATCH 19/55] activity: fix credo nitpick --- lib/pleroma/activity.ex | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex index 370721070..f041d0e53 100644 --- a/lib/pleroma/activity.ex +++ b/lib/pleroma/activity.ex @@ -41,7 +41,8 @@ defmodule Pleroma.Activity do # # ``` # |> join(:inner, [activity], o in Object, - # fragment("(?->>'id') = COALESCE((? -> 'object'::text) ->> 'id'::text)", o.data, activity.data)) + # on: fragment("(?->>'id') = COALESCE((? -> 'object'::text) ->> 'id'::text)", + # o.data, activity.data)) # |> preload([activity, object], [object: object]) # ``` # @@ -58,7 +59,7 @@ defmodule Pleroma.Activity do :inner, [activity], o in Object, - fragment( + on: fragment( "(?->>'id') = COALESCE((? -> 'object'::text) ->> 'id'::text)", o.data, activity.data From a6973a668e40645f9c0940a4fb5aeee45003b66f Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sat, 23 Mar 2019 00:28:16 +0000 Subject: [PATCH 20/55] formatting --- lib/pleroma/activity.ex | 11 ++++++----- lib/pleroma/web/common_api/common_api.ex | 3 ++- lib/pleroma/web/common_api/utils.ex | 3 ++- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex index f041d0e53..a762d66ef 100644 --- a/lib/pleroma/activity.ex +++ b/lib/pleroma/activity.ex @@ -59,11 +59,12 @@ defmodule Pleroma.Activity do :inner, [activity], o in Object, - on: fragment( - "(?->>'id') = COALESCE((? -> 'object'::text) ->> 'id'::text)", - o.data, - activity.data - ) + on: + fragment( + "(?->>'id') = COALESCE((? -> 'object'::text) ->> 'id'::text)", + o.data, + activity.data + ) ) |> preload([activity, object], object: object) end diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex index 8625f6621..25b990677 100644 --- a/lib/pleroma/web/common_api/common_api.ex +++ b/lib/pleroma/web/common_api/common_api.ex @@ -63,7 +63,8 @@ defmodule Pleroma.Web.CommonAPI do end def delete(activity_id, user) do - with %Activity{data: %{"object" => _}} = activity <- Activity.get_by_id_with_object(activity_id), + with %Activity{data: %{"object" => _}} = activity <- + Activity.get_by_id_with_object(activity_id), %Object{} = object <- Object.normalize(activity), true <- User.superuser?(user) || user.ap_id == object.data["actor"], {:ok, _} <- unpin(activity_id, user), diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex index 6a94c2d26..f596f703b 100644 --- a/lib/pleroma/web/common_api/utils.ex +++ b/lib/pleroma/web/common_api/utils.ex @@ -17,7 +17,8 @@ defmodule Pleroma.Web.CommonAPI.Utils do # This is a hack for twidere. def get_by_id_or_ap_id(id) do - activity = Activity.get_by_id_with_object(id) || Activity.get_create_by_object_ap_id_with_object(id) + activity = + Activity.get_by_id_with_object(id) || Activity.get_create_by_object_ap_id_with_object(id) activity && if activity.data["type"] == "Create" do From e4307cadc86018914cbe1298bcd06299e81006e5 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sat, 23 Mar 2019 00:40:08 +0000 Subject: [PATCH 21/55] activitypub: splice in the child object if we have one --- lib/pleroma/web/activity_pub/activity_pub.ex | 10 +++++++++- lib/pleroma/web/activity_pub/utils.ex | 6 +++--- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 7d38a46e5..9cca7ec6f 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -95,7 +95,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do :ok <- check_actor_is_active(map["actor"]), {_, true} <- {:remote_limit_error, check_remote_limit(map)}, {:ok, map} <- MRF.filter(map), - :ok <- insert_full_object(map) do + {:ok, object} <- insert_full_object(map) do {recipients, _, _} = get_recipients(map) {:ok, activity} = @@ -106,6 +106,14 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do recipients: recipients }) + # Splice in the child object if we have one. + activity = + if !is_nil(object) do + Map.put(activity, :object, object) + else + activity + end + Task.start(fn -> Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity) end) diff --git a/lib/pleroma/web/activity_pub/utils.ex b/lib/pleroma/web/activity_pub/utils.ex index af317245f..2e9ffe41c 100644 --- a/lib/pleroma/web/activity_pub/utils.ex +++ b/lib/pleroma/web/activity_pub/utils.ex @@ -209,12 +209,12 @@ defmodule Pleroma.Web.ActivityPub.Utils do """ def insert_full_object(%{"object" => %{"type" => type} = object_data}) when is_map(object_data) and type in @supported_object_types do - with {:ok, _} <- Object.create(object_data) do - :ok + with {:ok, object} <- Object.create(object_data) do + {:ok, object} end end - def insert_full_object(_), do: :ok + def insert_full_object(_), do: {:ok, nil} def update_object_in_activities(%{data: %{"id" => id}} = object) do # TODO From ba7299fc875adc95102ddb1332bec2e6e89b6155 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sat, 23 Mar 2019 00:53:35 +0000 Subject: [PATCH 22/55] activitypub: add missing with_preloaded_object() --- lib/pleroma/web/activity_pub/activity_pub.ex | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 9cca7ec6f..95bbadec1 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -438,6 +438,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do ), order_by: [desc: :id] ) + |> Activity.with_preloaded_object() Repo.all(query) end From 73efe95368dfc910c965e4025f47b36b6eb37aaa Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sat, 23 Mar 2019 01:09:12 +0000 Subject: [PATCH 23/55] activitypub: allow skipping preload in some cases (like certain tests where the preload is obnoxious) --- lib/pleroma/web/activity_pub/activity_pub.ex | 9 +++++- test/web/activity_pub/activity_pub_test.exs | 30 ++++++++++++++------ 2 files changed, 29 insertions(+), 10 deletions(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 95bbadec1..0441376e3 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -718,6 +718,13 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do defp restrict_muted_reblogs(query, _), do: query + defp maybe_preload_objects(query, %{"skip_preload" => true}), do: query + + defp maybe_preload_objects(query, _) do + query + |> Activity.with_preloaded_object() + end + def fetch_activities_query(recipients, opts \\ %{}) do base_query = from( @@ -725,9 +732,9 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do limit: 20, order_by: [fragment("? desc nulls last", activity.id)] ) - |> Activity.with_preloaded_object() base_query + |> maybe_preload_objects(opts) |> restrict_recipients(recipients, opts["user"]) |> restrict_tag(opts) |> restrict_tag_reject(opts) diff --git a/test/web/activity_pub/activity_pub_test.exs b/test/web/activity_pub/activity_pub_test.exs index 035778218..c11140f1c 100644 --- a/test/web/activity_pub/activity_pub_test.exs +++ b/test/web/activity_pub/activity_pub_test.exs @@ -270,7 +270,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do booster = insert(:user) {:ok, user} = User.block(user, %{ap_id: activity_one.data["actor"]}) - activities = ActivityPub.fetch_activities([], %{"blocking_user" => user}) + activities = + ActivityPub.fetch_activities([], %{"blocking_user" => user, "skip_preload" => true}) assert Enum.member?(activities, activity_two) assert Enum.member?(activities, activity_three) @@ -278,7 +279,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do {:ok, user} = User.unblock(user, %{ap_id: activity_one.data["actor"]}) - activities = ActivityPub.fetch_activities([], %{"blocking_user" => user}) + activities = + ActivityPub.fetch_activities([], %{"blocking_user" => user, "skip_preload" => true}) assert Enum.member?(activities, activity_two) assert Enum.member?(activities, activity_three) @@ -289,14 +291,16 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do %Activity{} = boost_activity = Activity.get_create_by_object_ap_id(id) activity_three = Repo.get(Activity, activity_three.id) - activities = ActivityPub.fetch_activities([], %{"blocking_user" => user}) + activities = + ActivityPub.fetch_activities([], %{"blocking_user" => user, "skip_preload" => true}) assert Enum.member?(activities, activity_two) refute Enum.member?(activities, activity_three) refute Enum.member?(activities, boost_activity) assert Enum.member?(activities, activity_one) - activities = ActivityPub.fetch_activities([], %{"blocking_user" => nil}) + activities = + ActivityPub.fetch_activities([], %{"blocking_user" => nil, "skip_preload" => true}) assert Enum.member?(activities, activity_two) assert Enum.member?(activities, activity_three) @@ -312,14 +316,20 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do booster = insert(:user) {:ok, user} = User.mute(user, %User{ap_id: activity_one.data["actor"]}) - activities = ActivityPub.fetch_activities([], %{"muting_user" => user}) + activities = + ActivityPub.fetch_activities([], %{"muting_user" => user, "skip_preload" => true}) assert Enum.member?(activities, activity_two) assert Enum.member?(activities, activity_three) refute Enum.member?(activities, activity_one) # Calling with 'with_muted' will deliver muted activities, too. - activities = ActivityPub.fetch_activities([], %{"muting_user" => user, "with_muted" => true}) + activities = + ActivityPub.fetch_activities([], %{ + "muting_user" => user, + "with_muted" => true, + "skip_preload" => true + }) assert Enum.member?(activities, activity_two) assert Enum.member?(activities, activity_three) @@ -327,7 +337,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do {:ok, user} = User.unmute(user, %User{ap_id: activity_one.data["actor"]}) - activities = ActivityPub.fetch_activities([], %{"muting_user" => user}) + activities = + ActivityPub.fetch_activities([], %{"muting_user" => user, "skip_preload" => true}) assert Enum.member?(activities, activity_two) assert Enum.member?(activities, activity_three) @@ -338,14 +349,15 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do %Activity{} = boost_activity = Activity.get_create_by_object_ap_id(id) activity_three = Repo.get(Activity, activity_three.id) - activities = ActivityPub.fetch_activities([], %{"muting_user" => user}) + activities = + ActivityPub.fetch_activities([], %{"muting_user" => user, "skip_preload" => true}) assert Enum.member?(activities, activity_two) refute Enum.member?(activities, activity_three) refute Enum.member?(activities, boost_activity) assert Enum.member?(activities, activity_one) - activities = ActivityPub.fetch_activities([], %{"muting_user" => nil}) + activities = ActivityPub.fetch_activities([], %{"muting_user" => nil, "skip_preload" => true}) assert Enum.member?(activities, activity_two) assert Enum.member?(activities, activity_three) From e430a71d373a15b105a52771551b0ec4ed436ba4 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sat, 23 Mar 2019 01:17:26 +0000 Subject: [PATCH 24/55] ostatus: fetch preloaded object in note handler for testsuite --- lib/pleroma/web/ostatus/handlers/note_handler.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/ostatus/handlers/note_handler.ex b/lib/pleroma/web/ostatus/handlers/note_handler.ex index 770a71a0a..db995ec77 100644 --- a/lib/pleroma/web/ostatus/handlers/note_handler.ex +++ b/lib/pleroma/web/ostatus/handlers/note_handler.ex @@ -106,7 +106,7 @@ defmodule Pleroma.Web.OStatus.NoteHandler do # TODO: Clean this up a bit. def handle_note(entry, doc \\ nil) do with id <- XML.string_from_xpath("//id", entry), - activity when is_nil(activity) <- Activity.get_create_by_object_ap_id(id), + activity when is_nil(activity) <- Activity.get_create_by_object_ap_id_with_object(id), [author] <- :xmerl_xpath.string('//author[1]', doc), {:ok, actor} <- OStatus.find_make_or_update_user(author), content_html <- OStatus.get_content(entry), From aaec91b9a126a4f36c996485c3a3149b3ba8745f Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sat, 23 Mar 2019 01:23:02 +0000 Subject: [PATCH 25/55] relay test: don't do preloading (since follow objects are activities, it's a mess) --- test/tasks/relay_test.exs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/tasks/relay_test.exs b/test/tasks/relay_test.exs index c9d90fa2e..535dc3756 100644 --- a/test/tasks/relay_test.exs +++ b/test/tasks/relay_test.exs @@ -60,7 +60,8 @@ defmodule Mix.Tasks.Pleroma.RelayTest do ActivityPub.fetch_activities([], %{ "type" => "Undo", "actor_id" => follower_id, - "limit" => 1 + "limit" => 1, + "skip_preload" => true }) assert undo_activity.data["type"] == "Undo" From 4cedf454236c73b9337e58f5bf0e20ae65c65313 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sat, 23 Mar 2019 01:33:41 +0000 Subject: [PATCH 26/55] relay: use preloaded object since we always have it --- lib/pleroma/web/activity_pub/relay.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/activity_pub/relay.ex b/lib/pleroma/web/activity_pub/relay.ex index 01fef71b9..a7a20ca37 100644 --- a/lib/pleroma/web/activity_pub/relay.ex +++ b/lib/pleroma/web/activity_pub/relay.ex @@ -41,7 +41,7 @@ defmodule Pleroma.Web.ActivityPub.Relay do def publish(%Activity{data: %{"type" => "Create"}} = activity) do with %User{} = user <- get_actor(), - %Object{} = object <- Object.normalize(activity.data["object"]["id"]) do + %Object{} = object <- Object.normalize(activity) do ActivityPub.announce(user, object, nil, true, false) else e -> Logger.error("error: #{inspect(e)}") From 9a06d9f6e87c948997b908456ff8838e6110757e Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sat, 23 Mar 2019 02:19:34 +0000 Subject: [PATCH 27/55] notification: preload child objects --- lib/pleroma/notification.ex | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex index a98649b63..ef8e0b4c1 100644 --- a/lib/pleroma/notification.ex +++ b/lib/pleroma/notification.ex @@ -7,6 +7,7 @@ defmodule Pleroma.Notification do alias Pleroma.Activity alias Pleroma.Notification + alias Pleroma.Object alias Pleroma.Pagination alias Pleroma.Repo alias Pleroma.User @@ -33,7 +34,10 @@ defmodule Pleroma.Notification do Notification |> where(user_id: ^user.id) |> join(:inner, [n], activity in assoc(n, :activity)) - |> preload(:activity) + |> join(:left, [n, a], object in Object, + on: fragment("(?->>'id') = COALESCE((? -> 'object'::text) ->> 'id'::text)", object.data, a.data) + ) + |> preload([n, a, o], activity: {a, object: o}) end def for_user(user, opts \\ %{}) do From c62220c50078c648c33d7aad7a615b9c7e1f27c6 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sat, 23 Mar 2019 02:26:49 +0000 Subject: [PATCH 28/55] rich media: helpers: only crawl Create activities --- lib/pleroma/web/rich_media/helpers.ex | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/rich_media/helpers.ex b/lib/pleroma/web/rich_media/helpers.ex index f97d1863c..f67aaf58b 100644 --- a/lib/pleroma/web/rich_media/helpers.ex +++ b/lib/pleroma/web/rich_media/helpers.ex @@ -21,7 +21,7 @@ defmodule Pleroma.Web.RichMedia.Helpers do defp validate_page_url(%URI{}), do: :ok defp validate_page_url(_), do: :error - def fetch_data_for_activity(%Activity{} = activity) do + def fetch_data_for_activity(%Activity{data: %{"type" => "Create"}} = activity) do with true <- Pleroma.Config.get([:rich_media, :enabled]), %Object{} = object <- Object.normalize(activity), {:ok, page_url} <- HTML.extract_first_external_url(object, object.data["content"]), @@ -32,4 +32,6 @@ defmodule Pleroma.Web.RichMedia.Helpers do _ -> %{} end end + + def fetch_data_for_activity(_), do: %{} end From 07cdd9ed02838069b58c3b6cf9aec73b1d1852c3 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sat, 23 Mar 2019 02:27:52 +0000 Subject: [PATCH 29/55] streamer: use the preloaded object if possible --- lib/pleroma/web/streamer.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/streamer.ex b/lib/pleroma/web/streamer.ex index 7425bfb54..592749b42 100644 --- a/lib/pleroma/web/streamer.ex +++ b/lib/pleroma/web/streamer.ex @@ -202,7 +202,7 @@ defmodule Pleroma.Web.Streamer do mutes = user.info.mutes || [] reblog_mutes = user.info.muted_reblogs || [] - parent = Object.normalize(item.data["object"]) + parent = Object.normalize(item) unless is_nil(parent) or item.actor in blocks or item.actor in mutes or item.actor in reblog_mutes or not ActivityPub.contain_activity(item, user) or From aabcecb26935475b5985b6438774ee1d3cc14514 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sat, 23 Mar 2019 02:30:53 +0000 Subject: [PATCH 30/55] notification: formatting --- lib/pleroma/notification.ex | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex index ef8e0b4c1..cac10f24a 100644 --- a/lib/pleroma/notification.ex +++ b/lib/pleroma/notification.ex @@ -35,7 +35,12 @@ defmodule Pleroma.Notification do |> where(user_id: ^user.id) |> join(:inner, [n], activity in assoc(n, :activity)) |> join(:left, [n, a], object in Object, - on: fragment("(?->>'id') = COALESCE((? -> 'object'::text) ->> 'id'::text)", object.data, a.data) + on: + fragment( + "(?->>'id') = COALESCE((? -> 'object'::text) ->> 'id'::text)", + object.data, + a.data + ) ) |> preload([n, a, o], activity: {a, object: o}) end From ce47eb8b29aaba8f59720597485d3c586fbb7831 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sat, 23 Mar 2019 02:38:59 +0000 Subject: [PATCH 31/55] activitypub: when fetching objects, use the preloaded object from the synthesized activity --- lib/pleroma/web/activity_pub/activity_pub.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 0441376e3..80c64ae04 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -957,7 +957,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do }, :ok <- Transmogrifier.contain_origin(id, params), {:ok, activity} <- Transmogrifier.handle_incoming(params) do - {:ok, Object.normalize(activity.data["object"])} + {:ok, Object.normalize(activity)} else {:error, {:reject, nil}} -> {:reject, nil} @@ -969,7 +969,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do Logger.info("Couldn't get object via AP, trying out OStatus fetching...") case OStatus.fetch_activity_from_url(id) do - {:ok, [activity | _]} -> {:ok, Object.normalize(activity.data["object"])} + {:ok, [activity | _]} -> {:ok, Object.normalize(activity)} e -> e end end From f9d5c13b21905b6839f489d4c8d75bc1e95d3875 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sat, 23 Mar 2019 02:49:10 +0000 Subject: [PATCH 32/55] activity: add get_by_ap_id_with_object() --- lib/pleroma/activity.ex | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex index a762d66ef..fcdac3a3f 100644 --- a/lib/pleroma/activity.ex +++ b/lib/pleroma/activity.ex @@ -78,6 +78,23 @@ defmodule Pleroma.Activity do ) end + def get_by_ap_id_with_object(ap_id) do + Repo.one( + from( + activity in Activity, + where: fragment("(?)->>'id' = ?", activity.data, ^to_string(ap_id)), + inner_join: o in Object, + on: + fragment( + "(?->>'id') = COALESCE((? -> 'object'::text) ->> 'id'::text)", + o.data, + activity.data + ), + preload: [object: o] + ) + ) + end + def get_by_id(id) do Repo.get(Activity, id) end @@ -181,8 +198,8 @@ defmodule Pleroma.Activity do |> Repo.one() end - def normalize(obj) when is_map(obj), do: Activity.get_by_ap_id(obj["id"]) - def normalize(ap_id) when is_binary(ap_id), do: Activity.get_by_ap_id(ap_id) + def normalize(obj) when is_map(obj), do: get_by_ap_id_with_object(obj["id"]) + def normalize(ap_id) when is_binary(ap_id), do: get_by_ap_id_with_object(ap_id) def normalize(_), do: nil def get_in_reply_to_activity(%Activity{data: %{"object" => %{"inReplyTo" => ap_id}}}) do From 8c70156157fcb2e0091c21e566324bcb24886f6f Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sat, 23 Mar 2019 02:49:40 +0000 Subject: [PATCH 33/55] activitypub: object view: use preloaded object when possible --- lib/pleroma/web/activity_pub/views/object_view.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/activity_pub/views/object_view.ex b/lib/pleroma/web/activity_pub/views/object_view.ex index 84fa94e32..6028b773c 100644 --- a/lib/pleroma/web/activity_pub/views/object_view.ex +++ b/lib/pleroma/web/activity_pub/views/object_view.ex @@ -17,7 +17,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectView do def render("object.json", %{object: %Activity{data: %{"type" => "Create"}} = activity}) do base = Pleroma.Web.ActivityPub.Utils.make_json_ld_header() - object = Object.normalize(activity.data["object"]) + object = Object.normalize(activity) additional = Transmogrifier.prepare_object(activity.data) @@ -28,7 +28,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectView do def render("object.json", %{object: %Activity{} = activity}) do base = Pleroma.Web.ActivityPub.Utils.make_json_ld_header() - object = Object.normalize(activity.data["object"]) + object = Object.normalize(activity) additional = Transmogrifier.prepare_object(activity.data) From 3c2350cbee95df3b6a31945d54bf7c46a8afff25 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sat, 23 Mar 2019 03:00:04 +0000 Subject: [PATCH 34/55] object: downgrade normalize warning to debug severity --- lib/pleroma/object.ex | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/object.ex b/lib/pleroma/object.ex index 0f5c532ec..193ae3fa8 100644 --- a/lib/pleroma/object.ex +++ b/lib/pleroma/object.ex @@ -47,18 +47,22 @@ defmodule Pleroma.Object do # Catch and log Object.normalize() calls where the Activity's child object is not # preloaded. def normalize(%Activity{data: %{"object" => %{"id" => ap_id}}}) do - Logger.info( + Logger.debug( "Object.normalize() called without preloaded object (#{ap_id}). Consider preloading the object!" ) + Logger.debug("Backtrace: #{inspect(Process.info(:erlang.self(), :current_stacktrace))}") + normalize(ap_id) end def normalize(%Activity{data: %{"object" => ap_id}}) do - Logger.info( + Logger.debug( "Object.normalize() called without preloaded object (#{ap_id}). Consider preloading the object!" ) + Logger.debug("Backtrace: #{inspect(Process.info(:erlang.self(), :current_stacktrace))}") + normalize(ap_id) end From debf7f016d5f09b5878ddb24051a28ddb2cf1e4a Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sat, 23 Mar 2019 03:03:06 +0000 Subject: [PATCH 35/55] ostatus: use preload objects with Object.normalize() when opportunistic --- lib/pleroma/web/ostatus/ostatus.ex | 10 +++++----- lib/pleroma/web/ostatus/ostatus_controller.ex | 13 +++++++------ 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/lib/pleroma/web/ostatus/ostatus.ex b/lib/pleroma/web/ostatus/ostatus.ex index 266f86bf4..9a34d7ad5 100644 --- a/lib/pleroma/web/ostatus/ostatus.ex +++ b/lib/pleroma/web/ostatus/ostatus.ex @@ -23,8 +23,8 @@ defmodule Pleroma.Web.OStatus do alias Pleroma.Web.WebFinger alias Pleroma.Web.Websub - def is_representable?(%Activity{data: data}) do - object = Object.normalize(data["object"]) + def is_representable?(%Activity{} = activity) do + object = Object.normalize(activity) cond do is_nil(object) -> @@ -119,7 +119,7 @@ defmodule Pleroma.Web.OStatus do def make_share(entry, doc, retweeted_activity) do with {:ok, actor} <- find_make_or_update_user(doc), - %Object{} = object <- Object.normalize(retweeted_activity.data["object"]), + %Object{} = object <- Object.normalize(retweeted_activity), id when not is_nil(id) <- string_from_xpath("/entry/id", entry), {:ok, activity, _object} = ActivityPub.announce(actor, object, id, false) do {:ok, activity} @@ -137,7 +137,7 @@ defmodule Pleroma.Web.OStatus do def make_favorite(entry, doc, favorited_activity) do with {:ok, actor} <- find_make_or_update_user(doc), - %Object{} = object <- Object.normalize(favorited_activity.data["object"]), + %Object{} = object <- Object.normalize(favorited_activity), id when not is_nil(id) <- string_from_xpath("/entry/id", entry), {:ok, activity, _object} = ActivityPub.like(actor, object, id, false) do {:ok, activity} @@ -159,7 +159,7 @@ defmodule Pleroma.Web.OStatus do Logger.debug("Trying to get entry from db") with id when not is_nil(id) <- string_from_xpath("//activity:object[1]/id", entry), - %Activity{} = activity <- Activity.get_create_by_object_ap_id(id) do + %Activity{} = activity <- Activity.get_create_by_object_ap_id_with_object(id) do {:ok, activity} else _ -> diff --git a/lib/pleroma/web/ostatus/ostatus_controller.ex b/lib/pleroma/web/ostatus/ostatus_controller.ex index 0579a5f3d..2fb6ce41b 100644 --- a/lib/pleroma/web/ostatus/ostatus_controller.ex +++ b/lib/pleroma/web/ostatus/ostatus_controller.ex @@ -102,7 +102,8 @@ defmodule Pleroma.Web.OStatus.OStatusController do ActivityPubController.call(conn, :object) else with id <- o_status_url(conn, :object, uuid), - {_, %Activity{} = activity} <- {:activity, Activity.get_create_by_object_ap_id(id)}, + {_, %Activity{} = activity} <- + {:activity, Activity.get_create_by_object_ap_id_with_object(id)}, {_, true} <- {:public?, Visibility.is_public?(activity)}, %User{} = user <- User.get_cached_by_ap_id(activity.data["actor"]) do case get_format(conn) do @@ -148,13 +149,13 @@ defmodule Pleroma.Web.OStatus.OStatusController do end def notice(conn, %{"id" => id}) do - with {_, %Activity{} = activity} <- {:activity, Activity.get_by_id(id)}, + with {_, %Activity{} = activity} <- {:activity, Activity.get_by_id_with_object(id)}, {_, true} <- {:public?, Visibility.is_public?(activity)}, %User{} = user <- User.get_cached_by_ap_id(activity.data["actor"]) do case format = get_format(conn) do "html" -> if activity.data["type"] == "Create" do - %Object{} = object = Object.normalize(activity.data["object"]) + %Object{} = object = Object.normalize(activity) Fallback.RedirectController.redirector_with_meta(conn, %{ activity_id: activity.id, @@ -191,9 +192,9 @@ defmodule Pleroma.Web.OStatus.OStatusController do # Returns an HTML embedded @shp" - - content = HtmlSanitizeEx.strip_tags(content_html) - date = DateTime.from_naive!(~N[2016-05-24 13:26:08.003], "Etc/UTC") |> DateTime.to_iso8601() - - {:ok, convo_object} = Object.context_mapping("2hu") |> Repo.insert() - - to = [ - User.ap_followers(user), - "https://www.w3.org/ns/activitystreams#Public", - mentioned_user.ap_id - ] - - activity = %Activity{ - id: 1, - data: %{ - "type" => "Create", - "id" => "id", - "to" => to, - "actor" => User.ap_id(user), - "object" => %{ - "published" => date, - "type" => "Note", - "content" => content_html, - "summary" => "2hu :2hu:", - "inReplyToStatusId" => 213_123, - "attachment" => [ - object - ], - "external_url" => "some url", - "like_count" => 5, - "announcement_count" => 3, - "context" => "2hu", - "tag" => ["content", "mentioning", "nsfw"], - "emoji" => %{ - "2hu" => "corndog.png" - } - }, - "published" => date, - "context" => "2hu" - }, - local: false, - recipients: to - } - - corndog_emojo = ~s(2hu) - - expected_html = - ~s(

    2hu ) <> - corndog_emojo <> - ~s(

    alert\('YAY'\)Some ) <> - corndog_emojo <> - ~s( content mentioning     @shp) - - expected_status = %{ - "id" => activity.id, - "user" => UserView.render("show.json", %{user: user, for: follower}), - "is_local" => false, - "statusnet_html" => expected_html, - "text" => "2hu :2hu:" <> content, - "is_post_verb" => true, - "created_at" => "Tue May 24 13:26:08 +0000 2016", - "in_reply_to_status_id" => 213_123, - "in_reply_to_screen_name" => nil, - "in_reply_to_user_id" => nil, - "in_reply_to_profileurl" => nil, - "in_reply_to_ostatus_uri" => nil, - "statusnet_conversation_id" => convo_object.id, - "attachments" => [ - ObjectRepresenter.to_map(object) - ], - "attentions" => [ - UserView.render("show.json", %{user: mentioned_user, for: follower}) - ], - "fave_num" => 5, - "repeat_num" => 3, - "favorited" => false, - "repeated" => false, - "pinned" => false, - "external_url" => "some url", - "tags" => ["nsfw", "content", "mentioning"], - "activity_type" => "post", - "possibly_sensitive" => true, - "uri" => activity.data["object"]["id"], - "visibility" => "direct", - "card" => nil, - "muted" => false, - "summary" => "2hu :2hu:", - "summary_html" => - "2hu \"2hu\"" - } - - assert ActivityRepresenter.to_map(activity, %{ - user: user, - for: follower, - mentioned: [mentioned_user] - }) == expected_status - end - - test "a delete activity" do - object = insert(:note) - user = User.get_by_ap_id(object.data["actor"]) - - {:ok, delete} = ActivityPub.delete(object) - - map = ActivityRepresenter.to_map(delete, %{user: user}) - - assert map["is_post_verb"] == false - assert map["activity_type"] == "delete" - assert map["uri"] == object.data["id"] - end -end diff --git a/test/web/twitter_api/twitter_api_controller_test.exs b/test/web/twitter_api/twitter_api_controller_test.exs index 1b810c9a0..ac481ca14 100644 --- a/test/web/twitter_api/twitter_api_controller_test.exs +++ b/test/web/twitter_api/twitter_api_controller_test.exs @@ -18,7 +18,7 @@ defmodule Pleroma.Web.TwitterAPI.ControllerTest do alias Pleroma.Web.OAuth.Token alias Pleroma.Web.TwitterAPI.Controller alias Pleroma.Web.TwitterAPI.NotificationView - alias Pleroma.Web.TwitterAPI.Representers.ActivityRepresenter + alias Pleroma.Web.TwitterAPI.ActivityView alias Pleroma.Web.TwitterAPI.TwitterAPI alias Pleroma.Web.TwitterAPI.UserView @@ -116,7 +116,11 @@ defmodule Pleroma.Web.TwitterAPI.ControllerTest do |> post(request_path, %{status: "Nice meme.", visibility: "private"}) assert json_response(conn, 200) == - ActivityRepresenter.to_map(Repo.one(Activity), %{user: user, for: user}) + ActivityView.render("activity.json", %{ + activity: Repo.one(Activity), + user: user, + for: user + }) end end @@ -273,7 +277,7 @@ defmodule Pleroma.Web.TwitterAPI.ControllerTest do response = json_response(conn, 200) - assert response == ActivityRepresenter.to_map(activity, %{user: actor}) + assert response == ActivityView.render("activity.json", %{activity: activity, user: actor}) end end @@ -372,7 +376,8 @@ defmodule Pleroma.Web.TwitterAPI.ControllerTest do assert response == Enum.map(returned_activities, fn activity -> - ActivityRepresenter.to_map(activity, %{ + ActivityView.render("activity.json", %{ + activity: activity, user: User.get_cached_by_ap_id(activity.data["actor"]), for: current_user }) @@ -469,10 +474,10 @@ defmodule Pleroma.Web.TwitterAPI.ControllerTest do assert length(response) == 1 assert Enum.at(response, 0) == - ActivityRepresenter.to_map(activity, %{ + ActivityView.render("activity.json", %{ user: current_user, for: current_user, - mentioned: [current_user] + activity: activity }) end @@ -594,7 +599,9 @@ defmodule Pleroma.Web.TwitterAPI.ControllerTest do conn = get(conn, "/api/statuses/user_timeline.json", %{"user_id" => user.id}) response = json_response(conn, 200) assert length(response) == 1 - assert Enum.at(response, 0) == ActivityRepresenter.to_map(activity, %{user: user}) + + assert Enum.at(response, 0) == + ActivityView.render("activity.json", %{user: user, activity: activity}) end test "with screen_name", %{conn: conn} do @@ -604,7 +611,9 @@ defmodule Pleroma.Web.TwitterAPI.ControllerTest do conn = get(conn, "/api/statuses/user_timeline.json", %{"screen_name" => user.nickname}) response = json_response(conn, 200) assert length(response) == 1 - assert Enum.at(response, 0) == ActivityRepresenter.to_map(activity, %{user: user}) + + assert Enum.at(response, 0) == + ActivityView.render("activity.json", %{user: user, activity: activity}) end test "with credentials", %{conn: conn, user: current_user} do @@ -620,7 +629,11 @@ defmodule Pleroma.Web.TwitterAPI.ControllerTest do assert length(response) == 1 assert Enum.at(response, 0) == - ActivityRepresenter.to_map(activity, %{user: current_user, for: current_user}) + ActivityView.render("activity.json", %{ + user: current_user, + for: current_user, + activity: activity + }) end test "with credentials with user_id", %{conn: conn, user: current_user} do @@ -635,7 +648,9 @@ defmodule Pleroma.Web.TwitterAPI.ControllerTest do response = json_response(conn, 200) assert length(response) == 1 - assert Enum.at(response, 0) == ActivityRepresenter.to_map(activity, %{user: user}) + + assert Enum.at(response, 0) == + ActivityView.render("activity.json", %{user: user, activity: activity}) end test "with credentials screen_name", %{conn: conn, user: current_user} do @@ -650,7 +665,9 @@ defmodule Pleroma.Web.TwitterAPI.ControllerTest do response = json_response(conn, 200) assert length(response) == 1 - assert Enum.at(response, 0) == ActivityRepresenter.to_map(activity, %{user: user}) + + assert Enum.at(response, 0) == + ActivityView.render("activity.json", %{user: user, activity: activity}) end test "with credentials with user_id, excluding RTs", %{conn: conn, user: current_user} do @@ -669,7 +686,9 @@ defmodule Pleroma.Web.TwitterAPI.ControllerTest do response = json_response(conn, 200) assert length(response) == 1 - assert Enum.at(response, 0) == ActivityRepresenter.to_map(activity, %{user: user}) + + assert Enum.at(response, 0) == + ActivityView.render("activity.json", %{user: user, activity: activity}) conn = conn @@ -678,7 +697,9 @@ defmodule Pleroma.Web.TwitterAPI.ControllerTest do response = json_response(conn, 200) assert length(response) == 1 - assert Enum.at(response, 0) == ActivityRepresenter.to_map(activity, %{user: user}) + + assert Enum.at(response, 0) == + ActivityView.render("activity.json", %{user: user, activity: activity}) end end @@ -937,7 +958,7 @@ defmodule Pleroma.Web.TwitterAPI.ControllerTest do activity_user = Repo.get_by(User, ap_id: note_activity.data["actor"]) assert json_response(response, 200) == - ActivityRepresenter.to_map(activity, %{user: activity_user, for: current_user}) + ActivityView.render("activity.json", %{user: activity_user, for: current_user, activity: activity}) end end @@ -971,7 +992,7 @@ defmodule Pleroma.Web.TwitterAPI.ControllerTest do activity_user = Repo.get_by(User, ap_id: note_activity.data["actor"]) assert json_response(response, 200) == - ActivityRepresenter.to_map(activity, %{user: activity_user, for: current_user}) + ActivityView.render("activity.json", %{user: activity_user, for: current_user, activity: activity}) end end @@ -1955,7 +1976,7 @@ defmodule Pleroma.Web.TwitterAPI.ControllerTest do user = refresh_record(user) assert json_response(response, 200) == - ActivityRepresenter.to_map(activity, %{user: user, for: user}) + ActivityView.render("activity.json", %{user: user, for: user, activity: activity}) end end @@ -1985,7 +2006,7 @@ defmodule Pleroma.Web.TwitterAPI.ControllerTest do user = refresh_record(user) assert json_response(response, 200) == - ActivityRepresenter.to_map(activity, %{user: user, for: user}) + ActivityView.render("activity.json", %{user: user, for: user, activity: activity}) end end From 087662d4fb91e35497de90bb568a1d7cba441b34 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Tue, 26 Mar 2019 17:18:18 +0300 Subject: [PATCH 51/55] Fix formatting --- test/web/twitter_api/twitter_api_controller_test.exs | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/test/web/twitter_api/twitter_api_controller_test.exs b/test/web/twitter_api/twitter_api_controller_test.exs index ac481ca14..9e008b5a0 100644 --- a/test/web/twitter_api/twitter_api_controller_test.exs +++ b/test/web/twitter_api/twitter_api_controller_test.exs @@ -958,7 +958,11 @@ defmodule Pleroma.Web.TwitterAPI.ControllerTest do activity_user = Repo.get_by(User, ap_id: note_activity.data["actor"]) assert json_response(response, 200) == - ActivityView.render("activity.json", %{user: activity_user, for: current_user, activity: activity}) + ActivityView.render("activity.json", %{ + user: activity_user, + for: current_user, + activity: activity + }) end end @@ -992,7 +996,11 @@ defmodule Pleroma.Web.TwitterAPI.ControllerTest do activity_user = Repo.get_by(User, ap_id: note_activity.data["actor"]) assert json_response(response, 200) == - ActivityView.render("activity.json", %{user: activity_user, for: current_user, activity: activity}) + ActivityView.render("activity.json", %{ + user: activity_user, + for: current_user, + activity: activity + }) end end From aacdcac1be43aa31fb61407719ba7fb62cee4d79 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Tue, 26 Mar 2019 17:35:45 +0300 Subject: [PATCH 52/55] Credo is upset about me not remembering the alphabet --- test/web/twitter_api/twitter_api_controller_test.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/web/twitter_api/twitter_api_controller_test.exs b/test/web/twitter_api/twitter_api_controller_test.exs index 9e008b5a0..083540017 100644 --- a/test/web/twitter_api/twitter_api_controller_test.exs +++ b/test/web/twitter_api/twitter_api_controller_test.exs @@ -16,9 +16,9 @@ defmodule Pleroma.Web.TwitterAPI.ControllerTest do alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.CommonAPI alias Pleroma.Web.OAuth.Token + alias Pleroma.Web.TwitterAPI.ActivityView alias Pleroma.Web.TwitterAPI.Controller alias Pleroma.Web.TwitterAPI.NotificationView - alias Pleroma.Web.TwitterAPI.ActivityView alias Pleroma.Web.TwitterAPI.TwitterAPI alias Pleroma.Web.TwitterAPI.UserView From caf0e9cf331dc70b8f055d741bb40bb8bf564f59 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Tue, 26 Mar 2019 18:13:24 +0300 Subject: [PATCH 53/55] Test for MastoAPI /api/v1/instance response structure. --- .../mastodon_api_controller_test.exs | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index b2302422b..aa9abe97d 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -1808,6 +1808,27 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do end test "get instance information", %{conn: conn} do + conn = get(conn, "/api/v1/instance") + assert result = json_response(conn, 200) + + # Note: not checking for "max_toot_chars" since it's optional + assert %{ + "uri" => _, + "title" => _, + "description" => _, + "version" => _, + "email" => _, + "urls" => %{ + "streaming_api" => _ + }, + "stats" => _, + "thumbnail" => _, + "languages" => _, + "registrations" => _ + } = result + end + + test "get instance stats", %{conn: conn} do user = insert(:user, %{local: true}) user2 = insert(:user, %{local: true}) From 47b49ab1a1453ac2068cca336ad6db9877c3e64b Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Tue, 26 Mar 2019 18:18:36 +0300 Subject: [PATCH 54/55] Test for MastoAPI /api/v1/instance response structure (formatting fix). --- .../mastodon_api_controller_test.exs | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index aa9abe97d..21e88eda9 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -1813,19 +1813,19 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do # Note: not checking for "max_toot_chars" since it's optional assert %{ - "uri" => _, - "title" => _, - "description" => _, - "version" => _, - "email" => _, - "urls" => %{ - "streaming_api" => _ - }, - "stats" => _, - "thumbnail" => _, - "languages" => _, - "registrations" => _ - } = result + "uri" => _, + "title" => _, + "description" => _, + "version" => _, + "email" => _, + "urls" => %{ + "streaming_api" => _ + }, + "stats" => _, + "thumbnail" => _, + "languages" => _, + "registrations" => _ + } = result end test "get instance stats", %{conn: conn} do From 10a7a4a868fbc5ff09516892b99e11100d5d3660 Mon Sep 17 00:00:00 2001 From: lain Date: Tue, 26 Mar 2019 16:40:09 +0100 Subject: [PATCH 55/55] AP UserView: Refactor banner / avatar display code, add test. --- lib/pleroma/user.ex | 16 ++++--------- lib/pleroma/web/activity_pub/utils.ex | 15 ------------ .../web/activity_pub/views/user_view.ex | 20 ++++++++++++---- .../web/activity_pub/views/user_view_test.exs | 23 +++++++++++++++++++ 4 files changed, 42 insertions(+), 32 deletions(-) diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index ee5eb8efa..efb4540b9 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -83,25 +83,17 @@ defmodule Pleroma.User do def superuser?(%User{local: true, info: %User.Info{is_moderator: true}}), do: true def superuser?(_), do: false - def avatar_url(user) do + def avatar_url(user, options \\ []) do case user.avatar do %{"url" => [%{"href" => href} | _]} -> href - _ -> "#{Web.base_url()}/images/avi.png" + _ -> !options[:no_default] && "#{Web.base_url()}/images/avi.png" end end - # Do not return instance default avatar for federation - def avatar_url_ap(user) do - case user.avatar do - %{"url" => [%{"href" => href} | _]} -> href - _ -> nil - end - end - - def banner_url(user) do + def banner_url(user, options \\ []) do case user.info.banner do %{"url" => [%{"href" => href} | _]} -> href - _ -> "#{Web.base_url()}/images/banner.png" + _ -> !options[:no_default] && "#{Web.base_url()}/images/banner.png" end end diff --git a/lib/pleroma/web/activity_pub/utils.ex b/lib/pleroma/web/activity_pub/utils.ex index 86b6e7029..2e9ffe41c 100644 --- a/lib/pleroma/web/activity_pub/utils.ex +++ b/lib/pleroma/web/activity_pub/utils.ex @@ -230,21 +230,6 @@ defmodule Pleroma.Web.ActivityPub.Utils do end) end - # Only federate user icon if not nil - # Prevents federating instance default avatars - def maybe_make_icon(user) do - if User.avatar_url_ap(user) do - %{ - "icon" => %{ - "type" => "Image", - "url" => User.avatar_url_ap(user) - } - } - else - %{} - end - end - #### Like-related helpers @doc """ diff --git a/lib/pleroma/web/activity_pub/views/user_view.ex b/lib/pleroma/web/activity_pub/views/user_view.ex index f5c86d360..5926a3294 100644 --- a/lib/pleroma/web/activity_pub/views/user_view.ex +++ b/lib/pleroma/web/activity_pub/views/user_view.ex @@ -87,13 +87,10 @@ defmodule Pleroma.Web.ActivityPub.UserView do "publicKeyPem" => public_key }, "endpoints" => endpoints, - "image" => %{ - "type" => "Image", - "url" => User.banner_url(user) - }, "tag" => user.info.source_data["tag"] || [] } - |> Map.merge(Utils.maybe_make_icon(user)) + |> Map.merge(maybe_make_image(&User.avatar_url/2, "icon", user)) + |> Map.merge(maybe_make_image(&User.banner_url/2, "image", user)) |> Map.merge(Utils.make_json_ld_header()) end @@ -291,4 +288,17 @@ defmodule Pleroma.Web.ActivityPub.UserView do map end end + + defp maybe_make_image(func, key, user) do + if image = func.(user, no_default: true) do + %{ + key => %{ + "type" => "Image", + "url" => image + } + } + else + %{} + end + end end diff --git a/test/web/activity_pub/views/user_view_test.exs b/test/web/activity_pub/views/user_view_test.exs index 0bc1d4728..9fb9455d2 100644 --- a/test/web/activity_pub/views/user_view_test.exs +++ b/test/web/activity_pub/views/user_view_test.exs @@ -16,6 +16,29 @@ defmodule Pleroma.Web.ActivityPub.UserViewTest do assert String.contains?(result["publicKey"]["publicKeyPem"], "BEGIN PUBLIC KEY") end + test "Does not add an avatar image if the user hasn't set one" do + user = insert(:user) + {:ok, user} = Pleroma.Web.WebFinger.ensure_keys_present(user) + + result = UserView.render("user.json", %{user: user}) + refute result["icon"] + refute result["image"] + + user = + insert(:user, + avatar: %{"url" => [%{"href" => "https://someurl"}]}, + info: %{ + banner: %{"url" => [%{"href" => "https://somebanner"}]} + } + ) + + {:ok, user} = Pleroma.Web.WebFinger.ensure_keys_present(user) + + result = UserView.render("user.json", %{user: user}) + assert result["icon"]["url"] == "https://someurl" + assert result["image"]["url"] == "https://somebanner" + end + describe "endpoints" do test "local users have a usable endpoints structure" do user = insert(:user)