Merge remote-tracking branch 'origin/develop' into shigusegubu

* origin/develop:
  Update AdminFE build, re-enables emoji packs
  Migrations: Make user_id index on notifications better for query.
  Notification: Actually preload objects.
  ActivityPub: Change ordering to `nulls last` in favorites query
  Activity.Queries: Use correct actor restriction.
  Migrations: Add index on client_id and client_secret for apps.
  Config: Restore old new background image
  Fix Oban warnings
  Mastodon API Controllers: Use the correct params for rate limiting.
  [#1501] Made user feed contain public and unlisted activities.
  Skip failing `:crypt` test on mac
  Docs: sql -> SQL
  Apply suggestion to docs/installation/otp_en.md
  Docs: Document reasonable Postgres settings.
  Transmogrifier: Use a simpler way to get mentions.
  User: Change signature of get_users_from_set
  StealEmojiPolicyTest: Fix flaky test.
  Factory: Set users to be ap_enabled by default.
  UserTest: Add test for user refreshing.
  User: Don't error out if we want to refresh a user but can't

docs/configuration/postgresql.md

@@ -0,0 +1,31 @@
+# Optimizing your PostgreSQL performance
+
+Pleroma performance depends to a large extent on good database performance. The default PostgreSQL settings are mostly fine, but often you can get better performance by changing a few settings.
+
+You can use [PGTune](https://pgtune.leopard.in.ua) to get recommendations for your setup. If you do, set the "Number of Connections" field to 20, as Pleroma will only use 10 concurrent connections anyway. If you don't, it will give you advice that might even hurt your performance.
+
+We also recommend not using the "Network Storage" option.
+
+## Example configurations
+
+Here are some configuration suggestions for PostgreSQL 10+.
+
+### 1GB RAM, 1 CPU
+```
+shared_buffers = 256MB
+effective_cache_size = 768MB
+maintenance_work_mem = 64MB
+work_mem = 13107kB
+```
+
+### 2GB RAM, 2 CPU
+```
+shared_buffers = 512MB
+effective_cache_size = 1536MB
+maintenance_work_mem = 128MB
+work_mem = 26214kB
+max_worker_processes = 2
+max_parallel_workers_per_gather = 1
+max_parallel_workers = 2
+```
+

docs/installation/otp_en.md

@@ -63,7 +63,7 @@ apt install postgresql-11-rum
 ```
 
 #### (Optional) Performance configuration
-For optimal performance, you may use [PGTune](https://pgtune.leopard.in.ua), don't forget to restart postgresql after editing the configuration
+It is encouraged to check [Optimizing your PostgreSQL performance](../configuration/postgresql.md) document, for tips on PostgreSQL tuning.
 
 ```sh tab="Alpine"
 rc-service postgresql restart

lib/pleroma/activity/queries.ex

@@ -24,10 +24,7 @@ defmodule Pleroma.Activity.Queries do
 
   @spec by_actor(query, String.t()) :: query
   def by_actor(query \\ Activity, actor) do
-    from(
-      activity in query,
-      where: fragment("(?)->>'actor' = ?", activity.data, ^actor)
-    )
+    from(a in query, where: a.actor == ^actor)
   end
 
   @spec by_author(query, User.t()) :: query

lib/pleroma/conversation.ex

@@ -63,7 +63,7 @@ defmodule Pleroma.Conversation do
          ap_id when is_binary(ap_id) and byte_size(ap_id) > 0 <- object.data["context"] do
       {:ok, conversation} = create_for_ap_id(ap_id)
 
-      users = User.get_users_from_set(activity.recipients, false)
+      users = User.get_users_from_set(activity.recipients, local_only: false)
 
       participations =
         Enum.map(users, fn user ->

lib/pleroma/notification.ex

@@ -92,8 +92,9 @@ defmodule Pleroma.Notification do
     |> join(:left, [n, a], object in Object,
       on:
         fragment(
-          "(?->>'id') = COALESCE((? -> 'object'::text) ->> 'id'::text)",
+          "(?->>'id') = COALESCE(?->'object'->>'id', ?->>'object')",
           object.data,
+          a.data,
           a.data
         )
     )
@@ -224,18 +225,8 @@ defmodule Pleroma.Notification do
       |> Marker.multi_set_last_read_id(user, "notifications")
       |> Repo.transaction()
 
-    Notification
+    for_user_query(user)
     |> where([n], n.id in ^notification_ids)
-    |> join(:inner, [n], activity in assoc(n, :activity))
-    |> join(:left, [n, a], object in Object,
-      on:
-        fragment(
-          "(?->>'id') = COALESCE((? -> 'object'::text) ->> 'id'::text)",
-          object.data,
-          a.data
-        )
-    )
-    |> preload([n, a, o], activity: {a, object: o})
     |> Repo.all()
   end
 
@@ -370,7 +361,8 @@ defmodule Pleroma.Notification do
       when type in ["Create", "Like", "Announce", "Follow", "Move", "EmojiReact"] do
     potential_receiver_ap_ids = get_potential_receiver_ap_ids(activity)
 
-    potential_receivers = User.get_users_from_set(potential_receiver_ap_ids, local_only)
+    potential_receivers =
+      User.get_users_from_set(potential_receiver_ap_ids, local_only: local_only)
 
     notification_enabled_ap_ids =
       potential_receiver_ap_ids

lib/pleroma/user.ex

@@ -1208,8 +1208,9 @@ defmodule Pleroma.User do
 
   def increment_unread_conversation_count(_, user), do: {:ok, user}
 
-  @spec get_users_from_set([String.t()], boolean()) :: [User.t()]
-  def get_users_from_set(ap_ids, local_only \\ true) do
+  @spec get_users_from_set([String.t()], keyword()) :: [User.t()]
+  def get_users_from_set(ap_ids, opts \\ []) do
+    local_only = Keyword.get(opts, :local_only, true)
     criteria = %{ap_id: ap_ids, deactivated: false}
     criteria = if local_only, do: Map.put(criteria, :local, true), else: criteria
 
@@ -1618,12 +1619,19 @@ defmodule Pleroma.User do
   def fetch_by_ap_id(ap_id), do: ActivityPub.make_user_from_ap_id(ap_id)
 
   def get_or_fetch_by_ap_id(ap_id) do
-    user = get_cached_by_ap_id(ap_id)
+    cached_user = get_cached_by_ap_id(ap_id)
 
-    if !is_nil(user) and !needs_update?(user) do
+    maybe_fetched_user = needs_update?(cached_user) && fetch_by_ap_id(ap_id)
+
+    case {cached_user, maybe_fetched_user} do
+      {_, {:ok, %User{} = user}} ->
         {:ok, user}
-    else
-      fetch_by_ap_id(ap_id)
+
+      {%User{} = user, _} ->
+        {:ok, user}
+
+      _ ->
+        {:error, :not_found}
     end
   end
 

lib/pleroma/web/activity_pub/activity_pub.ex

@@ -538,14 +538,27 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     |> Repo.one()
   end
 
-  @spec fetch_public_activities(map(), Pagination.type()) :: [Activity.t()]
-  def fetch_public_activities(opts \\ %{}, pagination \\ :keyset) do
+  @spec fetch_public_or_unlisted_activities(map(), Pagination.type()) :: [Activity.t()]
+  def fetch_public_or_unlisted_activities(opts \\ %{}, pagination \\ :keyset) do
     opts = Map.drop(opts, ["user"])
 
-    [Constants.as_public()]
-    |> fetch_activities_query(opts)
-    |> restrict_unlisted()
-    |> Pagination.fetch_paginated(opts, pagination)
+    query = fetch_activities_query([Constants.as_public()], opts)
+
+    query =
+      if opts["restrict_unlisted"] do
+        restrict_unlisted(query)
+      else
+        query
+      end
+
+    Pagination.fetch_paginated(query, opts, pagination)
+  end
+
+  @spec fetch_public_activities(map(), Pagination.type()) :: [Activity.t()]
+  def fetch_public_activities(opts \\ %{}, pagination \\ :keyset) do
+    opts
+    |> Map.put("restrict_unlisted", true)
+    |> fetch_public_or_unlisted_activities(pagination)
   end
 
   @valid_visibilities ~w[direct unlisted public private]
@@ -1145,7 +1158,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     |> Activity.with_joined_object()
     |> Object.with_joined_activity()
     |> select([_like, object, activity], %{activity | object: object})
-    |> order_by([like, _, _], desc: like.id)
+    |> order_by([like, _, _], desc_nulls_last: like.id)
     |> Pagination.fetch_paginated(
       Map.merge(params, %{"skip_order" => true}),
       pagination,

lib/pleroma/web/activity_pub/transmogrifier.ex

@@ -1045,10 +1045,14 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
     Map.put(object, "tag", tags)
   end
 
+  # TODO These should be added on our side on insertion, it doesn't make much
+  # sense to regenerate these all the time
   def add_mention_tags(object) do
-    {enabled_receivers, disabled_receivers} = Utils.get_notified_from_object(object)
-    potential_receivers = enabled_receivers ++ disabled_receivers
-    mentions = Enum.map(potential_receivers, &build_mention_tag/1)
+    to = object["to"] || []
+    cc = object["cc"] || []
+    mentioned = User.get_users_from_set(to ++ cc, local_only: false)
+
+    mentions = Enum.map(mentioned, &build_mention_tag/1)
 
     tags = object["tag"] || []
     Map.put(object, "tag", tags ++ mentions)

lib/pleroma/web/feed/user_controller.ex

@@ -56,7 +56,7 @@ defmodule Pleroma.Web.Feed.UserController do
           "actor_id" => user.ap_id
         }
         |> put_if_exist("max_id", params["max_id"])
-        |> ActivityPub.fetch_public_activities()
+        |> ActivityPub.fetch_public_or_unlisted_activities()
 
       conn
       |> put_resp_content_type("application/#{format}+xml")

lib/pleroma/web/mastodon_api/controllers/account_controller.ex

@@ -81,7 +81,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountController do
 
   plug(
     RateLimiter,
-    [name: :relation_id_action, params: ["id", "uri"]] when action in @relationship_actions
+    [name: :relation_id_action, params: [:id, :uri]] when action in @relationship_actions
   )
 
   plug(RateLimiter, [name: :relations_actions] when action in @relationship_actions)

lib/pleroma/web/mastodon_api/controllers/status_controller.ex

@@ -84,13 +84,13 @@ defmodule Pleroma.Web.MastodonAPI.StatusController do
 
   plug(
     RateLimiter,
-    [name: :status_id_action, bucket_name: "status_id_action:reblog_unreblog", params: ["id"]]
+    [name: :status_id_action, bucket_name: "status_id_action:reblog_unreblog", params: [:id]]
     when action in ~w(reblog unreblog)a
   )
 
   plug(
     RateLimiter,
-    [name: :status_id_action, bucket_name: "status_id_action:fav_unfav", params: ["id"]]
+    [name: :status_id_action, bucket_name: "status_id_action:fav_unfav", params: [:id]]
     when action in ~w(favourite unfavourite)a
   )
 

lib/pleroma/workers/cron/clear_oauth_token_worker.ex

@@ -16,6 +16,8 @@ defmodule Pleroma.Workers.Cron.ClearOauthTokenWorker do
   def perform(_opts, _job) do
     if Config.get([:oauth2, :clean_expired_tokens], false) do
       Token.delete_expired_tokens()
+    else
+      :ok
     end
   end
 end

lib/pleroma/workers/cron/digest_emails_worker.ex

@@ -37,6 +37,8 @@ defmodule Pleroma.Workers.Cron.DigestEmailsWorker do
       )
       |> Repo.all()
       |> send_emails
+    else
+      :ok
     end
   end
 

lib/pleroma/workers/cron/new_users_digest_worker.ex

@@ -55,7 +55,11 @@ defmodule Pleroma.Workers.Cron.NewUsersDigestWorker do
         |> Repo.all()
         |> Enum.map(&Pleroma.Emails.NewUsersDigestEmail.new_users(&1, users_and_statuses))
         |> Enum.each(&Pleroma.Emails.Mailer.deliver/1)
+      else
+        :ok
       end
+    else
+      :ok
     end
   end
 end

lib/pleroma/workers/cron/purge_expired_activities_worker.ex

@@ -23,6 +23,8 @@ defmodule Pleroma.Workers.Cron.PurgeExpiredActivitiesWorker do
   def perform(_opts, _job) do
     if Config.get([ActivityExpiration, :enabled]) do
       Enum.each(ActivityExpiration.due_expirations(@interval), &delete_activity/1)
+    else
+      :ok
     end
   end
 

priv/repo/migrations/20200526144426_add_apps_indexes.exs

@@ -0,0 +1,7 @@
+defmodule Pleroma.Repo.Migrations.AddAppsIndexes do
+  use Ecto.Migration
+
+  def change do
+    create(index(:apps, [:client_id, :client_secret]))
+  end
+end

priv/repo/migrations/20200527104138_change_notification_user_index.exs

@@ -0,0 +1,8 @@
+defmodule Pleroma.Repo.Migrations.ChangeNotificationUserIndex do
+  use Ecto.Migration
+
+  def change do
+    drop_if_exists(index(:notifications, [:user_id]))
+    create_if_not_exists(index(:notifications, [:user_id, "id desc nulls last"]))
+  end
+end

priv/static/adminfe/chunk-e458.6c0703cb.css

@@ -0,0 +1 @@
+.status-card{margin-bottom:10px}.status-card .account{text-decoration:underline;line-height:26px;font-size:13px}.status-card .image{width:20%}.status-card .image img{width:100%}.status-card .show-more-button{margin-left:5px}.status-card .status-account{display:-webkit-box;display:-ms-flexbox;display:flex;-webkit-box-align:center;-ms-flex-align:center;align-items:center}.status-card .status-avatar-img{display:inline-block;width:15px;height:15px;margin-right:5px}.status-card .status-account-name{display:inline-block;margin:0;height:22px}.status-card .status-body{display:-webkit-box;display:-ms-flexbox;display:flex;-webkit-box-orient:vertical;-webkit-box-direction:normal;-ms-flex-direction:column;flex-direction:column}.status-card .status-checkbox{margin-right:7px}.status-card .status-content{font-size:15px;line-height:26px}.status-card .status-deleted{font-style:italic;margin-top:3px}.status-card .status-header{display:-webkit-box;display:-ms-flexbox;display:flex;-webkit-box-pack:justify;-ms-flex-pack:justify;justify-content:space-between}.status-card .status-without-content{font-style:italic}@media only screen and (max-width:480px){.el-message{min-width:80%}.el-message-box{width:80%}.status-card .el-card__header{padding:10px 17px}.status-card .el-tag{margin:3px 4px 3px 0}.status-card .status-account-container{margin-bottom:5px}.status-card .status-actions-button{margin:3px 0}.status-card .status-actions{display:-webkit-box;display:-ms-flexbox;display:flex;-ms-flex-wrap:wrap;flex-wrap:wrap}.status-card .status-header{display:-webkit-box;display:-ms-flexbox;display:flex;-webkit-box-orient:vertical;-webkit-box-direction:normal;-ms-flex-direction:column;flex-direction:column}}.statuses-container{padding:0 15px}.statuses-container h1{margin:10px 0 15px}.statuses-container .status-container{margin:0 0 10px}.statuses-header-container .el-button.is-plain:focus,.statuses-header-container .el-button.is-plain:hover{border-color:#dcdfe6;color:#606266;cursor:default}.checkbox-container{margin-bottom:15px}.filter-container{display:-webkit-box;display:-ms-flexbox;display:flex;height:36px;-webkit-box-pack:justify;-ms-flex-pack:justify;justify-content:space-between;-webkit-box-align:center;-ms-flex-align:center;align-items:center;margin:22px 0 15px}.reboot-button{padding:10px;margin:0;width:145px}.select-instance{width:396px}.statuses-header,.statuses-header-container{display:-webkit-box;display:-ms-flexbox;display:flex;-webkit-box-align:center;-ms-flex-align:center;align-items:center;-webkit-box-pack:justify;-ms-flex-pack:justify;justify-content:space-between}.statuses-pagination{padding:15px 0;text-align:center}@media only screen and (max-width:480px){.checkbox-container{margin-bottom:10px}.filter-container{display:-webkit-box;display:-ms-flexbox;display:flex;height:36px;-webkit-box-orient:vertical;-webkit-box-direction:normal;-ms-flex-direction:column;flex-direction:column;margin:10px 0}.select-field{width:100%;margin-bottom:5px}.select-instance{width:100%}.statuses-header-container{-webkit-box-orient:vertical;-webkit-box-direction:normal;-ms-flex-direction:column;flex-direction:column;-webkit-box-align:start;-ms-flex-align:start;align-items:flex-start}.statuses-header-container .el-button-group{width:100%}.statuses-header-container .el-button{padding:10px 6.5px;width:50%}.statuses-header-container .el-button-group>.el-button:first-child{border-bottom-left-radius:0}.statuses-header-container .el-button-group>.el-button:not(:first-child):not(:last-child).private-button{border-top-right-radius:4px}.statuses-header-container .el-button-group>.el-button:not(:first-child):not(:last-child).public-button{border-bottom-left-radius:4px;border-top:#fff}.statuses-header-container .el-button-group>.el-button:last-child{border-top-right-radius:0;border-top:#fff}.statuses-header-container .reboot-button{margin:10px 0 0}}

priv/static/adminfe/chunk-e458.f88bafea.css

@@ -1 +0,0 @@
-.status-card{margin-bottom:10px}.status-card .account{text-decoration:underline;line-height:26px;font-size:13px}.status-card .image{width:20%}.status-card .image img{width:100%}.status-card .show-more-button{margin-left:5px}.status-card .status-account{display:-webkit-box;display:-ms-flexbox;display:flex;-webkit-box-align:center;-ms-flex-align:center;align-items:center}.status-card .status-avatar-img{display:inline-block;width:15px;height:15px;margin-right:5px}.status-card .status-account-name{display:inline-block;margin:0;height:22px}.status-card .status-body{display:-webkit-box;display:-ms-flexbox;display:flex;-webkit-box-orient:vertical;-webkit-box-direction:normal;-ms-flex-direction:column;flex-direction:column}.status-card .status-checkbox{margin-right:7px}.status-card .status-content{font-size:15px;line-height:26px}.status-card .status-deleted{font-style:italic;margin-top:3px}.status-card .status-header{display:-webkit-box;display:-ms-flexbox;display:flex;-webkit-box-pack:justify;-ms-flex-pack:justify;justify-content:space-between}.status-card .status-without-content{font-style:italic}@media only screen and (max-width:480px){.el-message{min-width:80%}.el-message-box{width:80%}.status-card .el-card__header{padding:10px 17px}.status-card .el-tag{margin:3px 4px 3px 0}.status-card .status-account-container{margin-bottom:5px}.status-card .status-actions-button{margin:3px 0}.status-card .status-actions{display:-webkit-box;display:-ms-flexbox;display:flex;-ms-flex-wrap:wrap;flex-wrap:wrap}.status-card .status-header{display:-webkit-box;display:-ms-flexbox;display:flex;-webkit-box-orient:vertical;-webkit-box-direction:normal;-ms-flex-direction:column;flex-direction:column}}.statuses-container{padding:0 15px}.statuses-container h1{margin:10px 0 15px}.statuses-container .status-container{margin:0 0 10px}.checkbox-container{margin-bottom:15px}.filter-container{display:-webkit-box;display:-ms-flexbox;display:flex;height:36px;-webkit-box-pack:justify;-ms-flex-pack:justify;justify-content:space-between;-webkit-box-align:center;-ms-flex-align:center;align-items:center;margin:22px 0 15px}.reboot-button{padding:10px;margin:0;width:145px}.select-instance{width:396px}.statuses-header,.statuses-header-container{display:-webkit-box;display:-ms-flexbox;display:flex;-webkit-box-align:center;-ms-flex-align:center;align-items:center;-webkit-box-pack:justify;-ms-flex-pack:justify;justify-content:space-between}.statuses-pagination{padding:15px 0;text-align:center}@media only screen and (max-width:480px){.checkbox-container{margin-bottom:10px}.filter-container{display:-webkit-box;display:-ms-flexbox;display:flex;height:36px;-webkit-box-orient:vertical;-webkit-box-direction:normal;-ms-flex-direction:column;flex-direction:column;margin:10px 0}.select-field{width:100%;margin-bottom:5px}.select-instance{width:100%}.statuses-header-container{-webkit-box-orient:vertical;-webkit-box-direction:normal;-ms-flex-direction:column;flex-direction:column;-webkit-box-align:start;-ms-flex-align:start;align-items:flex-start}.statuses-header-container .el-button{padding:10px 6.5px}.statuses-header-container .reboot-button{margin:10px 0 0}}

priv/static/adminfe/index.html

@@ -1 +1 @@
-<!DOCTYPE html><html><head><meta charset=utf-8><meta http-equiv=X-UA-Compatible content="IE=edge,chrome=1"><meta name=renderer content=webkit><meta name=viewport content="width=device-width,initial-scale=1,maximum-scale=1,user-scalable=no"><title>Admin FE</title><link rel="shortcut icon" href=favicon.ico><link href=chunk-elementUI.1abbc9b8.css rel=stylesheet><link href=chunk-libs.686b5876.css rel=stylesheet><link href=app.796ca6d4.css rel=stylesheet></head><body><div id=app></div><script type=text/javascript src=static/js/runtime.1b4f6ce0.js></script><script type=text/javascript src=static/js/chunk-elementUI.fba0efec.js></script><script type=text/javascript src=static/js/chunk-libs.b8c453ab.js></script><script type=text/javascript src=static/js/app.203f69f8.js></script></body></html>
+<!DOCTYPE html><html><head><meta charset=utf-8><meta http-equiv=X-UA-Compatible content="IE=edge,chrome=1"><meta name=renderer content=webkit><meta name=viewport content="width=device-width,initial-scale=1,maximum-scale=1,user-scalable=no"><title>Admin FE</title><link rel="shortcut icon" href=favicon.ico><link href=chunk-elementUI.1abbc9b8.css rel=stylesheet><link href=chunk-libs.686b5876.css rel=stylesheet><link href=app.796ca6d4.css rel=stylesheet></head><body><div id=app></div><script type=text/javascript src=static/js/runtime.b08eb412.js></script><script type=text/javascript src=static/js/chunk-elementUI.fba0efec.js></script><script type=text/javascript src=static/js/chunk-libs.b8c453ab.js></script><script type=text/javascript src=static/js/app.0146039c.js></script></body></html>

priv/static/adminfe/static/js/runtime.b08eb412.js

@@ -1,2 +1,2 @@
-!function(e){function n(n){for(var r,c,a=n[0],f=n[1],h=n[2],i=0,l=[];i<a.length;i++)c=a[i],u[c]&&l.push(u[c][0]),u[c]=0;for(r in f)Object.prototype.hasOwnProperty.call(f,r)&&(e[r]=f[r]);for(d&&d(n);l.length;)l.shift()();return o.push.apply(o,h||[]),t()}function t(){for(var e,n=0;n<o.length;n++){for(var t=o[n],r=!0,c=1;c<t.length;c++){var f=t[c];0!==u[f]&&(r=!1)}r&&(o.splice(n--,1),e=a(a.s=t[0]))}return e}var r={},c={runtime:0},u={runtime:0},o=[];function a(n){if(r[n])return r[n].exports;var t=r[n]={i:n,l:!1,exports:{}};return e[n].call(t.exports,t,t.exports,a),t.l=!0,t.exports}a.e=function(e){var n=[];c[e]?n.push(c[e]):0!==c[e]&&{"chunk-6b68":1,"chunk-d38a":1,"chunk-0558":1,"chunk-0778":1,"chunk-3384":1,"chunk-6e81":1,"chunk-4011":1,"chunk-970d":1,"chunk-22d2":1,"chunk-e458":1,"chunk-7637":1,"chunk-0961":1}[e]&&n.push(c[e]=new Promise(function(n,t){for(var r=({}[e]||e)+"."+{"7zzA":"31d6cfe0",JEtC:"31d6cfe0",ZhIB:"31d6cfe0","chunk-6b68":"0cc00484","chunk-d38a":"cabdc22e","chunk-0558":"af0d89cd","chunk-0778":"d9e7180a","chunk-3384":"2278f87c","chunk-6e81":"0e80d020","chunk-7f9e":"31d6cfe0","chunk-4011":"c4799067","chunk-df62":"31d6cfe0","chunk-970d":"f59cca8c","chunk-22d2":"813009b9","chunk-e458":"f88bafea","chunk-7637":"941c4edb",oAJy:"31d6cfe0","chunk-0961":"d3692214","chunk-16d0":"31d6cfe0"}[e]+".css",c=a.p+r,u=document.getElementsByTagName("link"),o=0;o<u.length;o++){var f=(i=u[o]).getAttribute("data-href")||i.getAttribute("href");if("stylesheet"===i.rel&&(f===r||f===c))return n()}var h=document.getElementsByTagName("style");for(o=0;o<h.length;o++){var i;if((f=(i=h[o]).getAttribute("data-href"))===r||f===c)return n()}var d=document.createElement("link");d.rel="stylesheet",d.type="text/css",d.onload=n,d.onerror=function(n){var r=n&&n.target&&n.target.src||c,u=new Error("Loading CSS chunk "+e+" failed.\n("+r+")");u.request=r,t(u)},d.href=c,document.getElementsByTagName("head")[0].appendChild(d)}).then(function(){c[e]=0}));var t=u[e];if(0!==t)if(t)n.push(t[2]);else{var r=new Promise(function(n,r){t=u[e]=[n,r]});n.push(t[2]=r);var o,f=document.createElement("script");f.charset="utf-8",f.timeout=120,a.nc&&f.setAttribute("nonce",a.nc),f.src=function(e){return a.p+"static/js/"+({}[e]||e)+"."+{"7zzA":"e1ae1c94",JEtC:"f9ba4594",ZhIB:"861df339","chunk-6b68":"fbc0f684","chunk-d38a":"a851004a","chunk-0558":"75954137","chunk-0778":"b17650df","chunk-3384":"458ffaf1","chunk-6e81":"3733ace2","chunk-7f9e":"c49aa694","chunk-4011":"67fb1692","chunk-df62":"6c5105a6","chunk-970d":"2457e066","chunk-22d2":"a0cf7976","chunk-e458":"4e5aad44","chunk-7637":"8f5fb36e",oAJy:"840fb1c2","chunk-0961":"ef33e81b","chunk-16d0":"6ce78978"}[e]+".js"}(e),o=function(n){f.onerror=f.onload=null,clearTimeout(h);var t=u[e];if(0!==t){if(t){var r=n&&("load"===n.type?"missing":n.type),c=n&&n.target&&n.target.src,o=new Error("Loading chunk "+e+" failed.\n("+r+": "+c+")");o.type=r,o.request=c,t[1](o)}u[e]=void 0}};var h=setTimeout(function(){o({type:"timeout",target:f})},12e4);f.onerror=f.onload=o,document.head.appendChild(f)}return Promise.all(n)},a.m=e,a.c=r,a.d=function(e,n,t){a.o(e,n)||Object.defineProperty(e,n,{enumerable:!0,get:t})},a.r=function(e){"undefined"!=typeof Symbol&&Symbol.toStringTag&&Object.defineProperty(e,Symbol.toStringTag,{value:"Module"}),Object.defineProperty(e,"__esModule",{value:!0})},a.t=function(e,n){if(1&n&&(e=a(e)),8&n)return e;if(4&n&&"object"==typeof e&&e&&e.__esModule)return e;var t=Object.create(null);if(a.r(t),Object.defineProperty(t,"default",{enumerable:!0,value:e}),2&n&&"string"!=typeof e)for(var r in e)a.d(t,r,function(n){return e[n]}.bind(null,r));return t},a.n=function(e){var n=e&&e.__esModule?function(){return e.default}:function(){return e};return a.d(n,"a",n),n},a.o=function(e,n){return Object.prototype.hasOwnProperty.call(e,n)},a.p="",a.oe=function(e){throw console.error(e),e};var f=window.webpackJsonp=window.webpackJsonp||[],h=f.push.bind(f);f.push=n,f=f.slice();for(var i=0;i<f.length;i++)n(f[i]);var d=h;t()}([]);
-//# sourceMappingURL=runtime.1b4f6ce0.js.map
+!function(e){function n(n){for(var r,c,a=n[0],f=n[1],h=n[2],i=0,l=[];i<a.length;i++)c=a[i],u[c]&&l.push(u[c][0]),u[c]=0;for(r in f)Object.prototype.hasOwnProperty.call(f,r)&&(e[r]=f[r]);for(d&&d(n);l.length;)l.shift()();return o.push.apply(o,h||[]),t()}function t(){for(var e,n=0;n<o.length;n++){for(var t=o[n],r=!0,c=1;c<t.length;c++){var f=t[c];0!==u[f]&&(r=!1)}r&&(o.splice(n--,1),e=a(a.s=t[0]))}return e}var r={},c={runtime:0},u={runtime:0},o=[];function a(n){if(r[n])return r[n].exports;var t=r[n]={i:n,l:!1,exports:{}};return e[n].call(t.exports,t,t.exports,a),t.l=!0,t.exports}a.e=function(e){var n=[];c[e]?n.push(c[e]):0!==c[e]&&{"chunk-6b68":1,"chunk-d38a":1,"chunk-0558":1,"chunk-0778":1,"chunk-3384":1,"chunk-6e81":1,"chunk-7e30":1,"chunk-e458":1,"chunk-970d":1,"chunk-22d2":1,"chunk-7637":1,"chunk-0961":1}[e]&&n.push(c[e]=new Promise(function(n,t){for(var r=({}[e]||e)+"."+{"7zzA":"31d6cfe0",JEtC:"31d6cfe0",ZhIB:"31d6cfe0","chunk-6b68":"0cc00484","chunk-d38a":"cabdc22e","chunk-0558":"af0d89cd","chunk-0778":"d9e7180a","chunk-3384":"d50ed383","chunk-6e81":"0e80d020","chunk-7f9e":"31d6cfe0","chunk-7e30":"f2b9674a","chunk-df62":"31d6cfe0","chunk-e458":"6c0703cb","chunk-970d":"f59cca8c","chunk-22d2":"813009b9","chunk-7637":"941c4edb",oAJy:"31d6cfe0","chunk-0961":"d3692214","chunk-16d0":"31d6cfe0"}[e]+".css",c=a.p+r,u=document.getElementsByTagName("link"),o=0;o<u.length;o++){var f=(i=u[o]).getAttribute("data-href")||i.getAttribute("href");if("stylesheet"===i.rel&&(f===r||f===c))return n()}var h=document.getElementsByTagName("style");for(o=0;o<h.length;o++){var i;if((f=(i=h[o]).getAttribute("data-href"))===r||f===c)return n()}var d=document.createElement("link");d.rel="stylesheet",d.type="text/css",d.onload=n,d.onerror=function(n){var r=n&&n.target&&n.target.src||c,u=new Error("Loading CSS chunk "+e+" failed.\n("+r+")");u.request=r,t(u)},d.href=c,document.getElementsByTagName("head")[0].appendChild(d)}).then(function(){c[e]=0}));var t=u[e];if(0!==t)if(t)n.push(t[2]);else{var r=new Promise(function(n,r){t=u[e]=[n,r]});n.push(t[2]=r);var o,f=document.createElement("script");f.charset="utf-8",f.timeout=120,a.nc&&f.setAttribute("nonce",a.nc),f.src=function(e){return a.p+"static/js/"+({}[e]||e)+"."+{"7zzA":"e1ae1c94",JEtC:"f9ba4594",ZhIB:"861df339","chunk-6b68":"fbc0f684","chunk-d38a":"a851004a","chunk-0558":"75954137","chunk-0778":"b17650df","chunk-3384":"b2ebeeca","chunk-6e81":"3733ace2","chunk-7f9e":"c49aa694","chunk-7e30":"ec42e302","chunk-df62":"6c5105a6","chunk-e458":"bb460d81","chunk-970d":"2457e066","chunk-22d2":"a0cf7976","chunk-7637":"8f5fb36e",oAJy:"840fb1c2","chunk-0961":"ef33e81b","chunk-16d0":"6ce78978"}[e]+".js"}(e),o=function(n){f.onerror=f.onload=null,clearTimeout(h);var t=u[e];if(0!==t){if(t){var r=n&&("load"===n.type?"missing":n.type),c=n&&n.target&&n.target.src,o=new Error("Loading chunk "+e+" failed.\n("+r+": "+c+")");o.type=r,o.request=c,t[1](o)}u[e]=void 0}};var h=setTimeout(function(){o({type:"timeout",target:f})},12e4);f.onerror=f.onload=o,document.head.appendChild(f)}return Promise.all(n)},a.m=e,a.c=r,a.d=function(e,n,t){a.o(e,n)||Object.defineProperty(e,n,{enumerable:!0,get:t})},a.r=function(e){"undefined"!=typeof Symbol&&Symbol.toStringTag&&Object.defineProperty(e,Symbol.toStringTag,{value:"Module"}),Object.defineProperty(e,"__esModule",{value:!0})},a.t=function(e,n){if(1&n&&(e=a(e)),8&n)return e;if(4&n&&"object"==typeof e&&e&&e.__esModule)return e;var t=Object.create(null);if(a.r(t),Object.defineProperty(t,"default",{enumerable:!0,value:e}),2&n&&"string"!=typeof e)for(var r in e)a.d(t,r,function(n){return e[n]}.bind(null,r));return t},a.n=function(e){var n=e&&e.__esModule?function(){return e.default}:function(){return e};return a.d(n,"a",n),n},a.o=function(e,n){return Object.prototype.hasOwnProperty.call(e,n)},a.p="",a.oe=function(e){throw console.error(e),e};var f=window.webpackJsonp=window.webpackJsonp||[],h=f.push.bind(f);f.push=n,f=f.slice();for(var i=0;i<f.length;i++)n(f[i]);var d=h;t()}([]);
+//# sourceMappingURL=runtime.b08eb412.js.map

test/notification_test.exs

@@ -454,8 +454,7 @@ defmodule Pleroma.NotificationTest do
           status: "hey again @#{other_user.nickname}!"
         })
 
-      [n2, n1] = notifs = Notification.for_user(other_user)
-      assert length(notifs) == 2
+      [n2, n1] = Notification.for_user(other_user)
 
       assert n2.id > n1.id
 
@@ -464,7 +463,9 @@ defmodule Pleroma.NotificationTest do
           status: "hey yet again @#{other_user.nickname}!"
         })
 
-      Notification.set_read_up_to(other_user, n2.id)
+      [_, read_notification] = Notification.set_read_up_to(other_user, n2.id)
+
+      assert read_notification.activity.object
 
       [n3, n2, n1] = Notification.for_user(other_user)
 
@@ -972,7 +973,9 @@ defmodule Pleroma.NotificationTest do
 
       {:ok, _activity} = CommonAPI.post(muted, %{status: "hey @#{user.nickname}"})
 
-      assert length(Notification.for_user(user)) == 1
+      [notification] = Notification.for_user(user)
+
+      assert notification.activity.object
     end
 
     test "it doesn't return notifications for muted user with notifications" do

test/plugs/authentication_plug_test.exs

@@ -68,6 +68,7 @@ defmodule Pleroma.Plugs.AuthenticationPlugTest do
     assert "$pbkdf2" <> _ = user.password_hash
   end
 
+  @tag :skip_on_mac
   test "with a crypt hash, it updates to a pkbdf2 hash", %{conn: conn} do
     user =
       insert(:user,

test/support/factory.ex

@@ -34,7 +34,8 @@ defmodule Pleroma.Factory do
       last_digest_emailed_at: NaiveDateTime.utc_now(),
       last_refreshed_at: NaiveDateTime.utc_now(),
       notification_settings: %Pleroma.User.NotificationSetting{},
-      multi_factor_authentication_settings: %Pleroma.MFA.Settings{}
+      multi_factor_authentication_settings: %Pleroma.MFA.Settings{},
+      ap_enabled: true
     }
 
     %{

test/user_test.exs

@@ -586,6 +586,26 @@ defmodule Pleroma.UserTest do
 
       refute user.last_refreshed_at == orig_user.last_refreshed_at
     end
+
+    @tag capture_log: true
+    test "it returns the old user if stale, but unfetchable" do
+      a_week_ago = NaiveDateTime.add(NaiveDateTime.utc_now(), -604_800)
+
+      orig_user =
+        insert(
+          :user,
+          local: false,
+          nickname: "admin@mastodon.example.org",
+          ap_id: "http://mastodon.example.org/users/raymoo",
+          last_refreshed_at: a_week_ago
+        )
+
+      assert orig_user.last_refreshed_at == a_week_ago
+
+      {:ok, user} = User.get_or_fetch_by_ap_id("http://mastodon.example.org/users/raymoo")
+
+      assert user.last_refreshed_at == orig_user.last_refreshed_at
+    end
   end
 
   test "returns an ap_id for a user" do

test/web/activity_pub/activity_pub_controller_test.exs

@@ -451,6 +451,36 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do
       assert Activity.get_by_ap_id(data["id"])
     end
 
+    @tag capture_log: true
+    test "it inserts an incoming activity into the database" <>
+           "even if we can't fetch the user but have it in our db",
+         %{conn: conn} do
+      user =
+        insert(:user,
+          ap_id: "https://mastodon.example.org/users/raymoo",
+          ap_enabled: true,
+          local: false,
+          last_refreshed_at: nil
+        )
+
+      data =
+        File.read!("test/fixtures/mastodon-post-activity.json")
+        |> Poison.decode!()
+        |> Map.put("actor", user.ap_id)
+        |> put_in(["object", "attridbutedTo"], user.ap_id)
+
+      conn =
+        conn
+        |> assign(:valid_signature, true)
+        |> put_req_header("content-type", "application/activity+json")
+        |> post("/inbox", data)
+
+      assert "ok" == json_response(conn, 200)
+
+      ObanHelpers.perform(all_enqueued(worker: ReceiverWorker))
+      assert Activity.get_by_ap_id(data["id"])
+    end
+
     test "it clears `unreachable` federation status of the sender", %{conn: conn} do
       data = File.read!("test/fixtures/mastodon-post-activity.json") |> Poison.decode!()
 

test/web/activity_pub/transmogrifier_test.exs

@@ -1094,7 +1094,10 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do
       {:ok, activity} =
         CommonAPI.post(user, %{status: "hey, @#{other_user.nickname}, how are ya? #2hu"})
 
+      with_mock Pleroma.Notification,
+        get_notified_from_activity: fn _, _ -> [] end do
         {:ok, modified} = Transmogrifier.prepare_outgoing(activity.data)
+
         object = modified["object"]
 
         expected_mention = %{
@@ -1109,9 +1112,11 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do
           "name" => "#2hu"
         }
 
+        refute called(Pleroma.Notification.get_notified_from_activity(:_, :_))
         assert Enum.member?(object["tag"], expected_tag)
         assert Enum.member?(object["tag"], expected_mention)
       end
+    end
 
     test "it adds the sensitive property" do
       user = insert(:user)

test/web/feed/user_controller_test.exs

@@ -11,13 +11,14 @@ defmodule Pleroma.Web.Feed.UserControllerTest do
   alias Pleroma.Config
   alias Pleroma.Object
   alias Pleroma.User
+  alias Pleroma.Web.CommonAPI
 
   setup do: clear_config([:instance, :federating], true)
 
   describe "feed" do
     setup do: clear_config([:feed])
 
-    test "gets a feed", %{conn: conn} do
+    test "gets an atom feed", %{conn: conn} do
       Config.put(
         [:feed, :post_title],
         %{max_length: 10, omission: "..."}
@@ -157,6 +158,29 @@ defmodule Pleroma.Web.Feed.UserControllerTest do
 
       assert response(conn, 404)
     end
+
+    test "returns feed with public and unlisted activities", %{conn: conn} do
+      user = insert(:user)
+
+      {:ok, _} = CommonAPI.post(user, %{status: "public", visibility: "public"})
+      {:ok, _} = CommonAPI.post(user, %{status: "direct", visibility: "direct"})
+      {:ok, _} = CommonAPI.post(user, %{status: "unlisted", visibility: "unlisted"})
+      {:ok, _} = CommonAPI.post(user, %{status: "private", visibility: "private"})
+
+      resp =
+        conn
+        |> put_req_header("accept", "application/atom+xml")
+        |> get(user_feed_path(conn, :feed, user.nickname))
+        |> response(200)
+
+      activity_titles =
+        resp
+        |> SweetXml.parse()
+        |> SweetXml.xpath(~x"//entry/title/text()"l)
+        |> Enum.sort()
+
+      assert activity_titles == ['public', 'unlisted']
+    end
   end
 
   # Note: see ActivityPubControllerTest for JSON format tests