Merge remote-tracking branch 'origin/develop' into shigusegubu

* origin/develop: (218 commits)
  http: bump connection timeout to 10 seconds
  Fix prometheus-ecto error when not configured
  Document MRF.Simple :report_removal
  Add virtual :thread_muted? field
  Move default mascot configuration to `config/`
  Add changelog entry for mascot config
  Use string map for default mascot
  Format mascot tests
  Add mascot get/set tests
  Add API endpoints for a custom user mascot
  Add report filtering to MRF.SimplePolicy
  add Changelog entry
  also suppress link previews from posts marked #nsfw
  add CHANGELOG entry
  config: make sending the user agent configurable, disable sending the user agent in tests
  http: request builder: send user-agent when making requests
  rich media: suppress link previews if post is marked as sensitive
  mrf: simple policy: mark all posts instead of posts with media as sensitive if they match media_nsfw
  CI: Use the correct image with the correct hostname.
  RUM: Set rum status by the environment.
  ...

.gitignore

@@ -3,7 +3,6 @@
 /db
 /deps
 /*.ez
-/uploads
 /test/uploads
 /.elixir_ls
 /test/fixtures/test_tmp.txt
@@ -11,6 +10,7 @@
 /test/tmp/
 /doc
 /instance
+/priv/ssh_keys
 
 # Prevent committing custom emojis
 /priv/static/emoji/custom/*
@@ -38,3 +38,7 @@ erl_crash.dump
 
 # Prevent committing docs files
 /priv/static/doc/*
+
+# Code test coverage
+/cover
+/Elixir.*.coverdata

.gitlab-ci.yml

@@ -45,12 +45,30 @@ docs-build:
 unit-testing:
   stage: test
   services:
-  - name: postgres:9.6.2
+  - name: lainsoykaf/postgres-with-rum
+    alias: postgres
     command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"]
   script:
+    - mix deps.get
     - mix ecto.create
     - mix ecto.migrate
     - mix test --trace --preload-modules
+    - mix coveralls
+
+unit-testing-rum:
+  stage: test
+  services:
+  - name: lainsoykaf/postgres-with-rum
+    alias: postgres
+    command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"]
+  variables:
+    RUM_ENABLED: "true"
+  script:
+    - mix deps.get
+    - mix ecto.create
+    - mix ecto.migrate
+    - "mix ecto.migrate --migrations-path priv/repo/optional_migrations/rum_indexing/"
+    - mix test --trace --preload-modules
 
 lint:
   stage: test
@@ -63,7 +81,6 @@ analysis:
     - mix deps.get
     - mix credo --strict --only=warnings,todo,fixme,consistency,readability
 
-
 docs-deploy:
   stage: deploy
   image: alpine:3.9
@@ -77,4 +94,4 @@ docs-deploy:
     - echo "${SSH_HOST_KEY}" > ~/.ssh/known_hosts
     - eval $(ssh-agent -s)
     - echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add -
-    - rsync -hrvz --delete -e "ssh -p ${SSH_PORT}" priv/static/doc/ "${SSH_USER_HOST_LOCATION}/${CI_COMMIT_REF_NAME}" 
+    - rsync -hrvz --delete -e "ssh -p ${SSH_PORT}" priv/static/doc/ "${SSH_USER_HOST_LOCATION}/${CI_COMMIT_REF_NAME}"

CHANGELOG.md

@@ -10,23 +10,34 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - A [job queue](https://git.pleroma.social/pleroma/pleroma_job_queue) for federation, emails, web push, etc.
 - [Prometheus](https://prometheus.io/) metrics
 - Support for Mastodon's remote interaction
+- Mix Tasks: `mix pleroma.database bump_all_conversations`
 - Mix Tasks: `mix pleroma.database remove_embedded_objects`
+- Mix Tasks: `mix pleroma.database update_users_following_followers_counts`
+- Mix Tasks: `mix pleroma.user toggle_confirmed`
 - Federation: Support for reports
 - Configuration: `safe_dm_mentions` option
 - Configuration: `link_name` option
 - Configuration: `fetch_initial_posts` option
 - Configuration: `notify_email` option
 - Configuration: Media proxy `whitelist` option
+- Configuration: `report_uri` option
 - Pleroma API: User subscriptions
 - Pleroma API: Healthcheck endpoint
+- Pleroma API: `/api/v1/pleroma/mascot` per-user frontend mascot configuration endpoints
 - Admin API: Endpoints for listing/revoking invite tokens
 - Admin API: Endpoints for making users follow/unfollow each other
+- Admin API: added filters (role, tags, email, name) for users endpoint
+- Admin API: Endpoints for managing reports
+- Admin API: Endpoints for deleting and changing the scope of individual reported statuses
+- AdminFE: initial release with basic user management accessible at /pleroma/admin/
 - Mastodon API: [Scheduled statuses](https://docs.joinmastodon.org/api/rest/scheduled-statuses/)
 - Mastodon API: `/api/v1/notifications/destroy_multiple` (glitch-soc extension)
 - Mastodon API: `/api/v1/pleroma/accounts/:id/favourites` (API extension)
 - Mastodon API: [Reports](https://docs.joinmastodon.org/api/rest/reports/)
+- Mastodon API: `POST /api/v1/accounts` (account creation API)
 - ActivityPub C2S: OAuth endpoints
-- Metadata RelMe provider
+- Metadata: RelMe provider
+- OAuth: added support for refresh tokens
 - Emoji packs and emoji pack manager
 
 ### Changed
@@ -40,8 +51,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Federation: Removed `inReplyToStatusId` from objects
 - Configuration: Dedupe enabled by default
 - Configuration: Added `extra_cookie_attrs` for setting non-standard cookie attributes. Defaults to ["SameSite=Lax"] so that remote follows work.
-- Pleroma API: Support for emoji tags in `/api/pleroma/emoji` resulting in a breaking API change
 - Timelines: Messages involving people you have blocked will be excluded from the timeline in all cases instead of just repeats.
+- Admin API: Move the user related API to `api/pleroma/admin/users`
+- Pleroma API: Support for emoji tags in `/api/pleroma/emoji` resulting in a breaking API change
 - Mastodon API: Support for `exclude_types`, `limit` and `min_id` in `/api/v1/notifications`
 - Mastodon API: Add `languages` and `registrations` to `/api/v1/instance`
 - Mastodon API: Provide plaintext versions of cw/content in the Status entity
@@ -55,16 +67,21 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Mastodon API: Add `with_muted` parameter to timeline endpoints
 - Mastodon API: Actual reblog hiding instead of a dummy
 - Mastodon API: Remove attachment limit in the Status entity
+- Mastodon API: Added support max_id & since_id for bookmark timeline endpoints.
 - Deps: Updated Cowboy to 2.6
 - Deps: Updated Ecto to 3.0.7
 - Don't ship finmoji by default, they can be installed as an emoji pack
-- Mastodon API: Added support max_id & since_id for bookmark timeline endpoints.
+- Hide deactivated users and their statuses
+- Posts which are marked sensitive or tagged nsfw no longer have link previews.
+- HTTP connection timeout is now set to 10 seconds.
 
 ### Fixed
+- Added an FTS index on objects. Running `vacuum analyze` and setting a larger `work_mem` is recommended.
 - Followers counter not being updated when a follower is blocked
 - Deactivated users being able to request an access token
 - Limit on request body in rich media/relme parsers being ignored resulting in a possible memory leak
-- proper Twitter Card generation instead of a dummy
+- Proper Twitter Card generation instead of a dummy
+- Deletions failing for users with a large number of posts
 - NodeInfo: Include admins in `staffAccounts`
 - ActivityPub: Crashing when requesting empty local user's outbox
 - Federation: Handling of objects without `summary` property
@@ -87,6 +104,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Mastodon API: Handling of `reblogs` in `/api/v1/accounts/:id/follow`
 - Mastodon API: Correct `reblogged`, `favourited`, and `bookmarked` values in the reblog status JSON
 - Mastodon API: Exposing default scope of the user to anyone
+- Mastodon API: Make `irreversible` field default to `false` [`POST /api/v1/filters`]
+- User-Agent is now sent correctly for all HTTP requests.
+
+## Removed
+- Configuration: `config :pleroma, :fe` in favor of the more flexible `config :pleroma, :frontend_configurations`
 
 ## [0.9.9999] - 2019-04-05
 ### Security

COPYING

@@ -15,6 +15,14 @@ priv/static/images/pleroma-tan.png
 
 ---
 
+The following files are copyright © 2019 shitposter.club, and are distributed
+under the Creative Commons Attribution 4.0 International license, you should
+have received a copy of the license file as CC-BY-4.0.
+
+priv/static/images/pleroma-fox-tan-shy.png
+
+---
+
 The following files are copyright © 2017-2019 Pleroma Authors
 <https://pleroma.social/>, and are distributed under the Creative Commons
 Attribution-ShareAlike 4.0 International license, you should have received

README.md

@@ -12,7 +12,7 @@ For clients it supports both the [GNU Social API with Qvitter extensions](https:
 
 - [Client Applications for Pleroma](https://docs-develop.pleroma.social/clients.html)
 
-No release has been made yet, but several servers have been online for months already. If you want to run your own server, feel free to contact us at @lain@pleroma.soykaf.com or in our dev chat at #pleroma on freenode or via matrix at <https://matrix.heldscal.la/#/room/#freenode_#pleroma:matrix.org>.
+If you want to run your own server, feel free to contact us at @lain@pleroma.soykaf.com or in our dev chat at #pleroma on freenode or via matrix at <https://matrix.heldscal.la/#/room/#freenode_#pleroma:matrix.org>.
 
 ## Installation
 

config/config.exs

@@ -48,7 +48,8 @@ config :pleroma, ecto_repos: [Pleroma.Repo]
 
 config :pleroma, Pleroma.Repo,
   types: Pleroma.PostgresTypes,
-  telemetry_event: [Pleroma.Repo.Instrumenter]
+  telemetry_event: [Pleroma.Repo.Instrumenter],
+  migration_lock: nil
 
 config :pleroma, Pleroma.Captcha,
   enabled: false,
@@ -191,6 +192,7 @@ config :tesla, adapter: Tesla.Adapter.Hackney
 # Configures http settings, upstream proxy etc.
 config :pleroma, :http,
   proxy_url: nil,
+  send_user_agent: true,
   adapter: [
     ssl_options: [
       # We don't support TLS v1.3 yet
@@ -212,6 +214,11 @@ config :pleroma, :instance,
   registrations_open: true,
   federating: true,
   federation_reachability_timeout_days: 7,
+  federation_publisher_modules: [
+    Pleroma.Web.ActivityPub.Publisher,
+    Pleroma.Web.Websub,
+    Pleroma.Web.Salmon
+  ],
   allow_relay: true,
   rewrite_policy: [
     Pleroma.Web.ActivityPub.MRF.SimplePolicy,
@@ -236,6 +243,8 @@ config :pleroma, :instance,
   safe_dm_mentions: false,
   healthcheck: false
 
+config :pleroma, :app_account_creation, enabled: true, max_requests: 25, interval: 1800
+
 config :pleroma, :markup,
   # XXX - unfortunately, inline images must be enabled by default right now, because
   # of custom emoji.  Issue #275 discusses defanging that somehow.
@@ -271,6 +280,19 @@ config :pleroma, :frontend_configurations,
     showInstanceSpecificPanel: true
   }
 
+config :pleroma, :assets,
+  mascots: [
+    pleroma_fox_tan: %{
+      url: "/images/pleroma-fox-tan-smol.png",
+      mime_type: "image/png"
+    },
+    pleroma_fox_tan_shy: %{
+      url: "/images/pleroma-fox-tan-shy.png",
+      mime_type: "image/png"
+    }
+  ],
+  default_mascot: :pleroma_fox_tan
+
 config :pleroma, :activitypub,
   accept_blocks: true,
   unfollow_blocked: true,
@@ -302,6 +324,7 @@ config :pleroma, :mrf_simple,
     "preteengirls.biz",
     "melalandia.tk"
   ],
+  report_removal: [],
   reject: [],
   accept: []
 
@@ -410,7 +433,8 @@ config :pleroma_job_queue, :queues,
   web_push: 50,
   mailer: 10,
   transmogrifier: 20,
-  scheduled_activities: 10
+  scheduled_activities: 10,
+  background: 5
 
 config :pleroma, :fetch_initial_posts,
   enabled: false,
@@ -437,6 +461,9 @@ config :pleroma, :ldap,
   base: System.get_env("LDAP_BASE") || "dc=example,dc=com",
   uid: System.get_env("LDAP_UID") || "cn"
 
+config :esshd,
+  enabled: false
+
 oauth_consumer_strategies = String.split(System.get_env("OAUTH_CONSUMER_STRATEGIES") || "")
 
 ueberauth_providers =
@@ -462,6 +489,15 @@ config :pleroma, Pleroma.ScheduledActivity,
   total_user_limit: 300,
   enabled: true
 
+config :pleroma, :oauth2,
+  token_expires_in: 600,
+  issue_new_refresh_token: true
+
+config :pleroma, :database, rum_enabled: false
+
+config :http_signatures,
+  adapter: Pleroma.Signature
+
 # Import environment specific config. This must remain at the bottom
 # of this file so it overrides the configuration defined above.
 import_config "#{Mix.env()}.exs"

config/test.exs

@@ -59,6 +59,16 @@ config :pleroma, Pleroma.ScheduledActivity,
   total_user_limit: 3,
   enabled: false
 
+config :pleroma, :app_account_creation, max_requests: 5
+
+config :pleroma, :http_security, report_uri: "https://endpoint.com"
+
+config :pleroma, :http, send_user_agent: false
+
+rum_enabled = System.get_env("RUM_ENABLED") == "true"
+config :pleroma, :database, rum_enabled: rum_enabled
+IO.puts("RUM enabled: #{rum_enabled}")
+
 try do
   import_config "test.secret.exs"
 rescue

docs/api/admin_api.md

@@ -8,18 +8,23 @@ Authentication is required and the user must be an admin.
 
 - Method `GET`
 - Query Params:
-  - *optional* `query`: **string** search term
+  - *optional* `query`: **string** search term (e.g. nickname, domain, nickname@domain)
   - *optional* `filters`: **string** comma-separated string of filters:
     - `local`: only local users
     - `external`: only external users
     - `active`: only active users
     - `deactivated`: only deactivated users
+    - `is_admin`: users with admin role
+    - `is_moderator`: users with moderator role
   - *optional* `page`: **integer** page number
   - *optional* `page_size`: **integer** number of users per page (default is `50`)
-- Example: `https://mypleroma.org/api/pleroma/admin/users?query=john&filters=local,active&page=1&page_size=10`
+  - *optional* `tags`: **[string]** tags list
+  - *optional* `name`: **string** user display name
+  - *optional* `email`: **string** user email
+- Example: `https://mypleroma.org/api/pleroma/admin/users?query=john&filters=local,active&page=1&page_size=10&tags[]=some_tag&tags[]=another_tag&name=display_name&email=email@example.com`
 - Response:
 
-```JSON
+```json
 {
   "page_size": integer,
   "count": integer,
@@ -40,7 +45,7 @@ Authentication is required and the user must be an admin.
 }
 ```
 
-## `/api/pleroma/admin/user`
+## `/api/pleroma/admin/users`
 
 ### Remove a user
 
@@ -58,7 +63,7 @@ Authentication is required and the user must be an admin.
   - `password`
 - Response: User’s nickname
 
-## `/api/pleroma/admin/user/follow`
+## `/api/pleroma/admin/users/follow`
 ### Make a user follow another user
 
 - Methods: `POST`
@@ -68,7 +73,7 @@ Authentication is required and the user must be an admin.
 - Response:
  - "ok"
 
-## `/api/pleroma/admin/user/unfollow`
+## `/api/pleroma/admin/users/unfollow`
 ### Make a user unfollow another user
 
 - Methods: `POST`
@@ -87,7 +92,7 @@ Authentication is required and the user must be an admin.
   - `nickname`
 - Response: User’s object
 
-```JSON
+```json
 {
   "deactivated": bool,
   "id": integer,
@@ -101,17 +106,17 @@ Authentication is required and the user must be an admin.
 
 - Method: `PUT`
 - Params:
-  - `nickname`
-  - `tags`
+  - `nicknames` (array)
+  - `tags` (array)
 
 ### Untag a list of users
 
 - Method: `DELETE`
 - Params:
-  - `nickname`
-  - `tags`
+  - `nicknames` (array)
+  - `tags` (array)
 
-## `/api/pleroma/admin/permission_group/:nickname`
+## `/api/pleroma/admin/users/:nickname/permission_group`
 
 ### Get user user permission groups membership
 
@@ -119,14 +124,14 @@ Authentication is required and the user must be an admin.
 - Params: none
 - Response:
 
-```JSON
+```json
 {
   "is_moderator": bool,
   "is_admin": bool
 }
 ```
 
-## `/api/pleroma/admin/permission_group/:nickname/:permission_group`
+## `/api/pleroma/admin/users/:nickname/permission_group/:permission_group`
 
 Note: Available `:permission_group` is currently moderator and admin. 404 is returned when the permission group doesn’t exist.
 
@@ -136,7 +141,7 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret
 - Params: none
 - Response:
 
-```JSON
+```json
 {
   "is_moderator": bool,
   "is_admin": bool
@@ -160,7 +165,7 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret
   - On success: JSON of the `user.info`
 - Note: An admin cannot revoke their own admin status.
 
-## `/api/pleroma/admin/activation_status/:nickname`
+## `/api/pleroma/admin/users/:nickname/activation_status`
 
 ### Active or deactivate a user
 
@@ -198,7 +203,7 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret
 - Response:
   - On success: URL of the unfollowed relay
 
-## `/api/pleroma/admin/invite_token`
+## `/api/pleroma/admin/users/invite_token`
 
 ### Get an account registration invite token
 
@@ -210,7 +215,7 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret
   ]
 - Response: invite token (base64 string)
 
-## `/api/pleroma/admin/invites`
+## `/api/pleroma/admin/users/invites`
 
 ### Get a list of generated invites
 
@@ -218,7 +223,7 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret
 - Params: none
 - Response:
 
-```JSON
+```json
 {
 
   "invites": [
@@ -236,7 +241,7 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret
 }
 ```
 
-## `/api/pleroma/admin/revoke_invite`
+## `/api/pleroma/admin/users/revoke_invite`
 
 ### Revoke invite by token
 
@@ -245,7 +250,7 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret
   - `token`
 - Response:
 
-```JSON
+```json
 {
   "id": integer,
   "token": string,
@@ -259,7 +264,7 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret
 ```
 
 
-## `/api/pleroma/admin/email_invite`
+## `/api/pleroma/admin/users/email_invite`
 
 ### Sends registration invite via email
 
@@ -268,10 +273,287 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret
   - `email`
   - `name`, optional
 
-## `/api/pleroma/admin/password_reset`
+## `/api/pleroma/admin/users/:nickname/password_reset`
 
 ### Get a password reset token for a given nickname
 
 - Methods: `GET`
 - Params: none
 - Response: password reset token (base64 string)
+
+## `/api/pleroma/admin/reports`
+### Get a list of reports
+- Method `GET`
+- Params:
+  - `state`: optional, the state of reports. Valid values are `open`, `closed` and `resolved`
+  - `limit`: optional, the number of records to retrieve
+  - `since_id`: optional, returns results that are more recent than the specified id
+  - `max_id`: optional, returns results that are older than the specified id
+- Response: 
+  - On failure: 403 Forbidden error `{"error": "error_msg"}` when requested by anonymous or non-admin
+  - On success: JSON, returns a list of reports, where:
+    - `account`: the user who has been reported
+    - `actor`: the user who has sent the report
+    - `statuses`: list of statuses that have been included to the report
+
+```json
+{
+  "reports": [
+    {
+      "account": {
+        "acct": "user",
+        "avatar": "https://pleroma.example.org/images/avi.png",
+        "avatar_static": "https://pleroma.example.org/images/avi.png",
+        "bot": false,
+        "created_at": "2019-04-23T17:32:04.000Z",
+        "display_name": "User",
+        "emojis": [],
+        "fields": [],
+        "followers_count": 1,
+        "following_count": 1,
+        "header": "https://pleroma.example.org/images/banner.png",
+        "header_static": "https://pleroma.example.org/images/banner.png",
+        "id": "9i6dAJqSGSKMzLG2Lo",
+        "locked": false,
+        "note": "",
+        "pleroma": {
+          "confirmation_pending": false,
+          "hide_favorites": true,
+          "hide_followers": false,
+          "hide_follows": false,
+          "is_admin": false,
+          "is_moderator": false,
+          "relationship": {},
+          "tags": []
+        },
+        "source": {
+          "note": "",
+          "pleroma": {},
+          "sensitive": false
+        },
+        "statuses_count": 3,
+        "url": "https://pleroma.example.org/users/user",
+        "username": "user"
+      },
+      "actor": {
+        "acct": "lain",
+        "avatar": "https://pleroma.example.org/images/avi.png",
+        "avatar_static": "https://pleroma.example.org/images/avi.png",
+        "bot": false,
+        "created_at": "2019-03-28T17:36:03.000Z",
+        "display_name": "Roger Braun",
+        "emojis": [],
+        "fields": [],
+        "followers_count": 1,
+        "following_count": 1,
+        "header": "https://pleroma.example.org/images/banner.png",
+        "header_static": "https://pleroma.example.org/images/banner.png",
+        "id": "9hEkA5JsvAdlSrocam",
+        "locked": false,
+        "note": "",
+        "pleroma": {
+          "confirmation_pending": false,
+          "hide_favorites": false,
+          "hide_followers": false,
+          "hide_follows": false,
+          "is_admin": false,
+          "is_moderator": false,
+          "relationship": {},
+          "tags": []
+        },
+        "source": {
+          "note": "",
+          "pleroma": {},
+          "sensitive": false
+        },
+        "statuses_count": 1,
+        "url": "https://pleroma.example.org/users/lain",
+        "username": "lain"
+      },
+      "content": "Please delete it",
+      "created_at": "2019-04-29T19:48:15.000Z",
+      "id": "9iJGOv1j8hxuw19bcm",
+      "state": "open",
+      "statuses": [
+        {
+          "account": { ... },
+          "application": {
+            "name": "Web",
+            "website": null
+          },
+          "bookmarked": false,
+          "card": null,
+          "content": "<span class=\"h-card\"><a data-user=\"9hEkA5JsvAdlSrocam\" class=\"u-url mention\" href=\"https://pleroma.example.org/users/lain\">@<span>lain</span></a></span> click on my link <a href=\"https://www.google.com/\">https://www.google.com/</a>",
+          "created_at": "2019-04-23T19:15:47.000Z",
+          "emojis": [],
+          "favourited": false,
+          "favourites_count": 0,
+          "id": "9i6mQ9uVrrOmOime8m",
+          "in_reply_to_account_id": null,
+          "in_reply_to_id": null,
+          "language": null,
+          "media_attachments": [],
+          "mentions": [
+            {
+              "acct": "lain",
+              "id": "9hEkA5JsvAdlSrocam",
+              "url": "https://pleroma.example.org/users/lain",
+              "username": "lain"
+            },
+            {
+              "acct": "user",
+              "id": "9i6dAJqSGSKMzLG2Lo",
+              "url": "https://pleroma.example.org/users/user",
+              "username": "user"
+            }
+          ],
+          "muted": false,
+          "pinned": false,
+          "pleroma": {
+            "content": {
+              "text/plain": "@lain click on my link https://www.google.com/"
+            },
+            "conversation_id": 28,
+            "in_reply_to_account_acct": null,
+            "local": true,
+            "spoiler_text": {
+              "text/plain": ""
+            }
+          },
+          "reblog": null,
+          "reblogged": false,
+          "reblogs_count": 0,
+          "replies_count": 0,
+          "sensitive": false,
+          "spoiler_text": "",
+          "tags": [],
+          "uri": "https://pleroma.example.org/objects/8717b90f-8e09-4b58-97b0-e3305472b396",
+          "url": "https://pleroma.example.org/notice/9i6mQ9uVrrOmOime8m",
+          "visibility": "direct"
+        }
+      ]
+    }
+  ]
+}
+```
+
+## `/api/pleroma/admin/reports/:id`
+### Get an individual report
+- Method `GET`
+- Params:
+  - `id`
+- Response:
+  - On failure: 
+    - 403 Forbidden `{"error": "error_msg"}`
+    - 404 Not Found `"Not found"`
+  - On success: JSON, Report object (see above)
+
+## `/api/pleroma/admin/reports/:id`
+### Change the state of the report
+- Method `PUT`
+- Params:
+  - `id`
+  - `state`: required, the new state. Valid values are `open`, `closed` and `resolved`
+- Response: 
+  - On failure: 
+    - 400 Bad Request `"Unsupported state"`
+    - 403 Forbidden `{"error": "error_msg"}`
+    - 404 Not Found `"Not found"`
+  - On success: JSON, Report object (see above)
+
+## `/api/pleroma/admin/reports/:id/respond`
+### Respond to a report
+- Method `POST`
+- Params:
+  - `id`
+  - `status`: required, the message
+- Response: 
+  - On failure: 
+    - 400 Bad Request `"Invalid parameters"` when `status` is missing 
+    - 403 Forbidden `{"error": "error_msg"}` 
+    - 404 Not Found `"Not found"`
+  - On success: JSON, created Mastodon Status entity
+
+```json
+{
+  "account": { ... },
+  "application": {
+    "name": "Web",
+    "website": null
+  },
+  "bookmarked": false,
+  "card": null,
+  "content": "Your claim is going to be closed",
+  "created_at": "2019-05-11T17:13:03.000Z",
+  "emojis": [],
+  "favourited": false,
+  "favourites_count": 0,
+  "id": "9ihuiSL1405I65TmEq",
+  "in_reply_to_account_id": null,
+  "in_reply_to_id": null,
+  "language": null,
+  "media_attachments": [],
+  "mentions": [
+    {
+      "acct": "user",
+      "id": "9i6dAJqSGSKMzLG2Lo",
+      "url": "https://pleroma.example.org/users/user",
+      "username": "user"
+    },
+    {
+      "acct": "admin",
+      "id": "9hEkA5JsvAdlSrocam",
+      "url": "https://pleroma.example.org/users/admin",
+      "username": "admin"
+    }
+  ],
+  "muted": false,
+  "pinned": false,
+  "pleroma": {
+    "content": {
+      "text/plain": "Your claim is going to be closed"
+    },
+    "conversation_id": 35,
+    "in_reply_to_account_acct": null,
+    "local": true,
+    "spoiler_text": {
+      "text/plain": ""
+    }
+  },
+  "reblog": null,
+  "reblogged": false,
+  "reblogs_count": 0,
+  "replies_count": 0,
+  "sensitive": false,
+  "spoiler_text": "",
+  "tags": [],
+  "uri": "https://pleroma.example.org/objects/cab0836d-9814-46cd-a0ea-529da9db5fcb",
+  "url": "https://pleroma.example.org/notice/9ihuiSL1405I65TmEq",
+  "visibility": "direct"
+}
+```
+
+## `/api/pleroma/admin/statuses/:id`
+### Change the scope of an individual reported status
+- Method `PUT`
+- Params:
+  - `id`
+  - `sensitive`: optional, valid values are `true` or `false`
+  - `visibility`: optional, valid values are `public`, `private` and `unlisted`
+- Response: 
+  - On failure: 
+    - 400 Bad Request `"Unsupported visibility"`
+    - 403 Forbidden `{"error": "error_msg"}` 
+    - 404 Not Found `"Not found"`
+  - On success: JSON, Mastodon Status entity
+
+## `/api/pleroma/admin/statuses/:id`
+### Delete an individual reported status
+- Method `DELETE`
+- Params:
+  - `id`
+- Response: 
+  - On failure: 
+    - 403 Forbidden `{"error": "error_msg"}` 
+    - 404 Not Found `"Not found"`
+  - On success: 200 OK `{}`

docs/api/differences_in_mastoapi_responses.md

@@ -1,6 +1,6 @@
 # Differences in Mastodon API responses from vanilla Mastodon
 
-A Pleroma instance can be identified by "<Mastodon version> (compatible; Pleroma <version>)" present in `version` field in response from `/api/v1/instance` 
+A Pleroma instance can be identified by "<Mastodon version> (compatible; Pleroma <version>)" present in `version` field in response from `/api/v1/instance`
 
 ## Flake IDs
 
@@ -79,3 +79,21 @@ Additional parameters can be added to the JSON body/Form data:
 - `hide_follows` - if true, user's follows will be hidden
 - `hide_favorites` - if true, user's favorites timeline will be hidden
 - `show_role` - if true, user's role (e.g admin, moderator) will be exposed to anyone in the API
+- `default_scope` - the scope returned under `privacy` key in Source subentity
+
+## Authentication
+
+*Pleroma supports refreshing tokens.
+
+`POST /oauth/token`
+Post here request with grant_type=refresh_token to obtain new access token. Returns an access token.
+
+## Account Registration
+`POST /api/v1/accounts`
+
+Has theses additionnal parameters (which are the same as in Pleroma-API):
+    * `fullname`: optional
+    * `bio`: optional
+    * `captcha_solution`: optional, contains provider-specific captcha solution,
+    * `captcha_token`: optional, contains provider-specific captcha token
+    * `token`: invite token required when the registerations aren't public.

docs/api/pleroma_api.md

@@ -61,6 +61,15 @@ Request parameters can be passed via [query strings](https://en.wikipedia.org/wi
 * Response: JSON. Returns `{"status": "success"}` if the deletion was successful, `{"error": "[error message]"}` otherwise
 * Example response: `{"error": "Invalid password."}`
 
+## `/api/pleroma/disable_account`
+### Disable an account
+* Method `POST`
+* Authentication: required
+* Params:
+    * `password`: user's password
+* Response: JSON. Returns `{"status": "success"}` if the account was successfully disabled, `{"error": "[error message]"}` otherwise
+* Example response: `{"error": "Invalid password."}`
+
 ## `/api/account/register`
 ### Register a new user
 * Method `POST`
@@ -243,6 +252,45 @@ See [Admin-API](Admin-API.md)
 ]
 ```
 
+## `/api/v1/pleroma/mascot`
+### Gets user mascot image
+* Method `GET`
+* Authentication: required
+
+* Response: JSON. Returns a mastodon media attachment entity.
+* Example response:
+```json
+{
+    "id": "abcdefg",
+    "url": "https://pleroma.example.org/media/abcdefg.png",
+    "type": "image",
+    "pleroma": {
+        "mime_type": "image/png"
+    }
+}
+```
+
+### Updates user mascot image
+* Method `PUT`
+* Authentication: required
+* Params:
+    * `image`: Multipart image
+* Response: JSON. Returns a mastodon media attachment entity
+  when successful, otherwise returns HTTP 415 `{"error": "error_msg"}`
+* Example response:
+```json
+{
+    "id": "abcdefg",
+    "url": "https://pleroma.example.org/media/abcdefg.png",
+    "type": "image",
+    "pleroma": {
+        "mime_type": "image/png"
+    }
+}
+```
+* Note: Behaves exactly the same as `POST /api/v1/upload`.
+  Can only accept images - any attempt to upload non-image files will be met with `HTTP 415 Unsupported Media Type`.
+
 ## `/api/pleroma/notification_settings`
 ### Updates user notification settings
 * Method `PUT`

docs/config.md

@@ -37,7 +37,7 @@ This filter replaces the filename (not the path) of an upload. For complete obfu
 
 An example for Sendgrid adapter:
 
-```exs
+```elixir
 config :pleroma, Pleroma.Emails.Mailer,
   adapter: Swoosh.Adapters.Sendgrid,
   api_key: "YOUR_API_KEY"
@@ -45,7 +45,7 @@ config :pleroma, Pleroma.Emails.Mailer,
 
 An example for SMTP adapter:
 
-```exs
+```elixir
 config :pleroma, Pleroma.Emails.Mailer,
   adapter: Swoosh.Adapters.SMTP,
   relay: "smtp.gmail.com",
@@ -105,11 +105,17 @@ config :pleroma, Pleroma.Emails.Mailer,
 * `safe_dm_mentions`: If set to true, only mentions at the beginning of a post will be used to address people in direct messages. This is to prevent accidental mentioning of people when talking about them (e.g. "@friend hey i really don't like @enemy"). (Default: `false`)
 * `healthcheck`: if set to true, system data will be shown on ``/api/pleroma/healthcheck``.
 
+## :app_account_creation
+REST API for creating an account settings
+* `enabled`: Enable/disable registration
+* `max_requests`: Number of requests allowed for creating accounts
+* `interval`: Interval for restricting requests for one ip (seconds)
+
 ## :logger
 * `backends`: `:console` is used to send logs to stdout, `{ExSyslogger, :ex_syslogger}` to log to syslog, and `Quack.Logger` to log to Slack
 
 An example to enable ONLY ExSyslogger (f/ex in ``prod.secret.exs``) with info and debug suppressed:
-```
+```elixir
 config :logger,
   backends: [{ExSyslogger, :ex_syslogger}]
 
@@ -118,7 +124,7 @@ config :logger, :ex_syslogger,
 ```
 
 Another example, keeping console output and adding the pid to syslog output:
-```
+```elixir
 config :logger,
   backends: [:console, {ExSyslogger, :ex_syslogger}]
 
@@ -130,7 +136,7 @@ config :logger, :ex_syslogger,
 See: [logger’s documentation](https://hexdocs.pm/logger/Logger.html) and [ex_syslogger’s documentation](https://hexdocs.pm/ex_syslogger/)
 
 An example of logging info to local syslog, but warn to a Slack channel:
-```
+```elixir
 config :logger,
   backends: [ {ExSyslogger, :ex_syslogger}, Quack.Logger ],
   level: :info
@@ -156,14 +162,30 @@ Frontends can access these settings at `/api/pleroma/frontend_configurations`
 
 To add your own configuration for PleromaFE, use it like this:
 
-`config :pleroma, :frontend_configurations, pleroma_fe: %{redirectRootNoLogin: "/main/all", ...}`
+```elixir
+config :pleroma, :frontend_configurations,
+  pleroma_fe: %{
+    theme: "pleroma-dark",
+    # ... see /priv/static/static/config.json for the available keys.
+},
+  masto_fe: %{
+    showInstanceSpecificPanel: true
+  }
+```
 
-These settings need to be complete, they will override the defaults. See `priv/static/static/config.json` for the available keys.
+These settings **need to be complete**, they will override the defaults.
+
+NOTE: for versions < 1.0, you need to set [`:fe`](#fe) to false, as shown a few lines below.
 
 ## :fe
 __THIS IS DEPRECATED__
 
-If you are using this method, please change it to the `frontend_configurations` method. Please set this option to false in your config like this: `config :pleroma, :fe, false`.
+If you are using this method, please change it to the [`frontend_configurations`](#frontend_configurations) method.
+Please **set this option to false** in your config like this:
+
+```elixir
+config :pleroma, :fe, false
+```
 
 This section is used to configure Pleroma-FE, unless ``:managed_config`` in ``:instance`` is set to false.
 
@@ -181,6 +203,16 @@ This section is used to configure Pleroma-FE, unless ``:managed_config`` in ``:i
 * `hide_post_stats`: Hide notices statistics(repeats, favorites, …)
 * `hide_user_stats`: Hide profile statistics(posts, posts per day, followers, followings, …)
 
+## :assets
+
+This section configures assets to be used with various frontends. Currently the only option
+relates to mascots on the mastodon frontend
+
+* `mascots`: KeywordList of mascots, each element __MUST__ contain both a `url` and a
+  `mime_type` key.
+* `default_mascot`: An element from `mascots` - This will be used as the default mascot
+  on MastoFE (default: `:pleroma_fox_tan`)
+
 ## :mrf_simple
 * `media_removal`: List of instances to remove medias from
 * `media_nsfw`: List of instances to put medias as NSFW(sensitive) from
@@ -264,7 +296,8 @@ This will make Pleroma listen on `127.0.0.1` port `8080` and generate urls start
 * ``sts``: Whether to additionally send a `Strict-Transport-Security` header
 * ``sts_max_age``: The maximum age for the `Strict-Transport-Security` header if sent
 * ``ct_max_age``: The maximum age for the `Expect-CT` header if sent
-* ``referrer_policy``: The referrer policy to use, either `"same-origin"` or `"no-referrer"`.
+* ``referrer_policy``: The referrer policy to use, either `"same-origin"` or `"no-referrer"`
+* ``report_uri``: Adds the specified url to `report-uri` and `report-to` group in CSP header.
 
 ## :mrf_user_allowlist
 
@@ -274,7 +307,7 @@ their ActivityPub ID.
 
 An example:
 
-```exs
+```elixir
 config :pleroma, :mrf_user_allowlist,
   "example.org": ["https://example.org/users/admin"]
 ```
@@ -303,7 +336,7 @@ the source code is here: https://github.com/koto-bank/kocaptcha. The default end
 
 Allows to set a token that can be used to authenticate with the admin api without using an actual user by giving it as the 'admin_token' parameter. Example:
 
-```exs
+```elixir
 config :pleroma, :admin_token, "somerandomtoken"
 ```
 
@@ -387,7 +420,7 @@ Configuration for the `auto_linker` library:
 
 Example:
 
-```exs
+```elixir
 config :auto_linker,
   opts: [
     scheme: true,
@@ -428,15 +461,36 @@ Pleroma account will be created with the same name as the LDAP user name.
 * `base`: LDAP base, e.g. "dc=example,dc=com"
 * `uid`: LDAP attribute name to authenticate the user, e.g. when "cn", the filter will be "cn=username,base"
 
+## BBS / SSH access
+
+To enable simple command line interface accessible over ssh, add a setting like this to your configuration file:
+
+```exs
+app_dir = File.cwd!
+priv_dir = Path.join([app_dir, "priv/ssh_keys"])
+
+config :esshd,
+  enabled: true,
+  priv_dir: priv_dir,
+  handler: "Pleroma.BBS.Handler",
+  port: 10_022,
+  password_authenticator: "Pleroma.BBS.Authenticator"
+```
+
+Feel free to adjust the priv_dir and port number. Then you will have to create the key for the keys (in the example `priv/ssh_keys`) and create the host keys with `ssh-keygen -N "" -b 2048 -t rsa -f ssh_host_rsa_key`. After restarting, you should be able to connect to your Pleroma instance with `ssh username@server -p $PORT`
+
 ## :auth
 
+* `Pleroma.Web.Auth.PleromaAuthenticator`: default database authenticator
+* `Pleroma.Web.Auth.LDAPAuthenticator`: LDAP authentication
+
 Authentication / authorization settings.
 
 * `auth_template`: authentication form template. By default it's `show.html` which corresponds to `lib/pleroma/web/templates/o_auth/o_auth/show.html.eex`.
 * `oauth_consumer_template`: OAuth consumer mode authentication form template. By default it's `consumer.html` which corresponds to `lib/pleroma/web/templates/o_auth/o_auth/consumer.html.eex`.
 * `oauth_consumer_strategies`: the list of enabled OAuth consumer strategies; by default it's set by OAUTH_CONSUMER_STRATEGIES environment variable.
 
-# OAuth consumer mode
+## OAuth consumer mode
 
 OAuth consumer mode allows sign in / sign up via external OAuth providers (e.g. Twitter, Facebook, Google, Microsoft, etc.).
 Implementation is based on Ueberauth; see the list of [available strategies](https://github.com/ueberauth/ueberauth/wiki/List-of-Strategies).
@@ -460,7 +514,7 @@ Note: make sure that `"SameSite=Lax"` is set in `extra_cookie_attrs` when you ha
 Once the app is configured on external OAuth provider side, add app's credentials and strategy-specific settings (if any — e.g. see Microsoft below) to `config/prod.secret.exs`,
 per strategy's documentation (e.g. [ueberauth_twitter](https://github.com/ueberauth/ueberauth_twitter)). Example config basing on environment variables:
 
-```
+```elixir
 # Twitter
 config :ueberauth, Ueberauth.Strategy.Twitter.OAuth,
   consumer_key: System.get_env("TWITTER_CONSUMER_KEY"),
@@ -489,7 +543,29 @@ config :ueberauth, Ueberauth,
   ]
 ```
 
+## OAuth 2.0 provider - :oauth2
+
+Configure OAuth 2 provider capabilities:
+
+* `token_expires_in` - The lifetime in seconds of the access token.
+* `issue_new_refresh_token` - Keeps old refresh token or generate new refresh token when to obtain an access token.
+
 ## :emoji
 * `shortcode_globs`: Location of custom emoji files. `*` can be used as a wildcard. Example `["/emoji/custom/**/*.png"]`
 * `groups`: Emojis are ordered in groups (tags). This is an array of key-value pairs where the key is the groupname and the value the location or array of locations. `*` can be used as a wildcard. Example `[Custom: ["/emoji/*.png", "/emoji/custom/*.png"]]`
 * `default_manifest`: Location of the JSON-manifest. This manifest contains information about the emoji-packs you can download. Currently only one manifest can be added (no arrays).
+
+## Database options
+
+### RUM indexing for full text search
+* `rum_enabled`: If RUM indexes should be used. Defaults to `false`.
+
+RUM indexes are an alternative indexing scheme that is not included in PostgreSQL by default. While they may eventually be mainlined, for now they have to be installed as a PostgreSQL extension from https://github.com/postgrespro/rum.
+
+Their advantage over the standard GIN indexes is that they allow efficient ordering of search results by timestamp, which makes search queries a lot faster on larger servers, by one or two orders of magnitude. They take up around 3 times as much space as GIN indexes.
+
+To enable them, both the `rum_enabled` flag has to be set and the following special migration has to be run:
+
+`mix ecto.migrate --migrations-path priv/repo/optional_migrations/rum_indexing/`
+
+This will probably take a long time.

docs/config/mrf.md

@@ -5,11 +5,12 @@ Possible uses include:
 
 * marking incoming messages with media from a given account or instance as sensitive
 * rejecting messages from a specific instance
+* rejecting reports (flags) from a specific instance
 * removing/unlisting messages from the public timelines
 * removing media from messages
 * sending only public messages to a specific instance
 
-The MRF provides user-configurable policies.  The default policy is `NoOpPolicy`, which disables the MRF functionality.  Pleroma also includes an easy to use policy called `SimplePolicy` which maps messages matching certain pre-defined criterion to actions built into the policy module.  
+The MRF provides user-configurable policies.  The default policy is `NoOpPolicy`, which disables the MRF functionality.  Pleroma also includes an easy to use policy called `SimplePolicy` which maps messages matching certain pre-defined criterion to actions built into the policy module.
 It is possible to use multiple, active MRF policies at the same time.
 
 ## Quarantine Instances
@@ -41,12 +42,13 @@ Once `SimplePolicy` is enabled, you can configure various groups in the `:mrf_si
 * `media_nsfw`: Servers in this group will have the #nsfw tag and sensitive setting injected into incoming messages which contain media.
 * `reject`: Servers in this group will have their messages rejected.
 * `federated_timeline_removal`: Servers in this group will have their messages unlisted from the public timelines by flipping the `to` and `cc` fields.
+* `report_removal`: Servers in this group will have their reports (flags) rejected.
 
 Servers should be configured as lists.
 
 ### Example
 
-This example will enable `SimplePolicy`, block media from `illegalporn.biz`, mark media as NSFW from `porn.biz` and `porn.business`, reject messages from `spam.com` and remove messages from `spam.university` from the federated timeline:
+This example will enable `SimplePolicy`, block media from `illegalporn.biz`, mark media as NSFW from `porn.biz` and `porn.business`, reject messages from `spam.com`, remove messages from `spam.university` from the federated timeline and block reports (flags) from `whiny.whiner`:
 
 ```
 config :pleroma, :instance,
@@ -56,7 +58,8 @@ config :pleroma, :mrf_simple,
   media_removal: ["illegalporn.biz"],
   media_nsfw: ["porn.biz", "porn.business"],
   reject: ["spam.com"],
-  federated_timeline_removal: ["spam.university"]
+  federated_timeline_removal: ["spam.university"],
+  report_removal: ["whiny.whiner"]
 
 ```
 

docs/installation/debian_based_en.md

@@ -12,6 +12,7 @@ This guide will assume you are on Debian Stretch. This guide should also work wi
 * `erlang-tools`
 * `erlang-parsetools`
 * `erlang-eldap`, if you want to enable ldap authenticator
+* `erlang-ssh`
 * `erlang-xmerl`
 * `git`
 * `build-essential`
@@ -49,7 +50,7 @@ sudo dpkg -i /tmp/erlang-solutions_1.0_all.deb
 
 ```shell
 sudo apt update
-sudo apt install elixir erlang-dev erlang-parsetools erlang-xmerl erlang-tools
+sudo apt install elixir erlang-dev erlang-parsetools erlang-xmerl erlang-tools erlang-ssh
 ```
 
 ### Install PleromaBE

docs/installation/debian_based_jp.md

@@ -14,6 +14,7 @@
 - erlang-dev
 - erlang-tools
 - erlang-parsetools
+- erlang-ssh
 - erlang-xmerl (Jessieではバックポートからインストールすること！)
 - git
 - build-essential
@@ -44,7 +45,7 @@ wget -P /tmp/ https://packages.erlang-solutions.com/erlang-solutions_1.0_all.deb
 
 * ElixirとErlangをインストールします、
 ```
-apt update && apt install elixir erlang-dev erlang-parsetools erlang-xmerl erlang-tools
+apt update && apt install elixir erlang-dev erlang-parsetools erlang-xmerl erlang-tools erlang-ssh
 ```
 
 ### Pleroma BE (バックエンド) をインストールします

docs/introduction.md

@@ -1,30 +1,30 @@
 # Introduction to Pleroma
 ## What is Pleroma?
-Pleroma is a federated social networking platform, compatible with GNU social, Mastodon and other OStatus and ActivityPub implementations. It is free software licensed under the AGPLv3.  
-It actually consists of two components: a backend, named simply Pleroma, and a user-facing frontend, named Pleroma-FE. It also includes the Mastodon frontend, if that's your thing.  
-It's part of what we call the fediverse, a federated network of instances which speak common protocols and can communicate with each other.  
+Pleroma is a federated social networking platform, compatible with GNU social, Mastodon and other OStatus and ActivityPub implementations. It is free software licensed under the AGPLv3.
+It actually consists of two components: a backend, named simply Pleroma, and a user-facing frontend, named Pleroma-FE. It also includes the Mastodon frontend, if that's your thing.
+It's part of what we call the fediverse, a federated network of instances which speak common protocols and can communicate with each other.
 One account on a instance is enough to talk to the entire fediverse!
-  
+
 ## How can I use it?
 
-Pleroma instances are already widely deployed, a list can be found here:  
+Pleroma instances are already widely deployed, a list can be found here:
 http://distsn.org/pleroma-instances.html
 
-If you don't feel like joining an existing instance, but instead prefer to deploy your own instance, that's easy too!  
-Installation instructions can be found here:  
+If you don't feel like joining an existing instance, but instead prefer to deploy your own instance, that's easy too!
+Installation instructions can be found here:
 [main Pleroma wiki](/)
-  
+
 ## I got an account, now what?
-Great! Now you can explore the fediverse!  
-- Open the login page for your Pleroma instance (for ex. https://pleroma.soykaf.com) and login with your username and password.  
-(If you don't have one yet, click on Register) :slightly_smiling_face:  
+Great! Now you can explore the fediverse!
+- Open the login page for your Pleroma instance (for ex. https://pleroma.soykaf.com) and login with your username and password.
+(If you don't have one yet, click on Register) :slightly_smiling_face:
 
 At this point you will have two columns in front of you.
 
 ### Left column
-- first block: here you can see your avatar, your nickname a bio, and statistics (Statuses, Following, Followers).  
-Under that you have a text form which allows you to post new statuses. The icon on the left is for uploading media files and attach them to your post. The number under the text form is a character counter, every instance can have a different character limit (the default is 5000).  
-If you want to mention someone, type @ + name of the person. A drop-down menu will help you in finding the right person. :slight_smile:   
+- first block: here you can see your avatar, your nickname a bio, and statistics (Statuses, Following, Followers).
+Under that you have a text form which allows you to post new statuses. The icon on the left is for uploading media files and attach them to your post. The number under the text form is a character counter, every instance can have a different character limit (the default is 5000).
+If you want to mention someone, type @ + name of the person. A drop-down menu will help you in finding the right person. :slight_smile:
 To post your status, simply press Submit.
 
 - second block: Here you can switch between the different timelines:
@@ -38,7 +38,7 @@ To post your status, simply press Submit.
 - fourth block: This is the Notifications block, here you will get notified whenever somebody mentions you, follows you, repeats or favorites one of your statuses.
 
 ### Right column
-This is where the interesting stuff happens! :slight_smile:   
+This is where the interesting stuff happens! :slight_smile:
 Depending on the timeline you will see different statuses, but each status has a standard structure:
 - Icon + name + link to profile. An optional left-arrow if it's a reply to another status (hovering will reveal the replied-to status).
 - A + button on the right allows you to Expand/Collapse an entire discussion thread. It also updates in realtime!
@@ -47,9 +47,9 @@ Depending on the timeline you will see different statuses, but each status has a
 - Four buttons (left to right): Reply, Repeat, Favorite, Delete.
 
 ## Mastodon interface
-If the Pleroma interface isn't your thing, or you're just trying something new but you want to keep using the familiar Mastodon interface, we got that too! :smile:  
-Just add a "/web" after your instance url (for ex. https://pleroma.soycaf.com/web) and you'll end on the Mastodon web interface, but with a Pleroma backend! MAGIC! :fireworks:  
-For more information on the Mastodon interface, please look here:  
+If the Pleroma interface isn't your thing, or you're just trying something new but you want to keep using the familiar Mastodon interface, we got that too! :smile:
+Just add a "/web" after your instance url (for ex. https://pleroma.soycaf.com/web) and you'll end on the Mastodon web interface, but with a Pleroma backend! MAGIC! :fireworks:
+For more information on the Mastodon interface, please look here:
 https://github.com/tootsuite/documentation/blob/master/Using-Mastodon/User-guide.md
 
 Remember, what you see is only the frontend part of Mastodon, the backend is still Pleroma.

installation/download-mastofe-build.sh

@@ -0,0 +1,45 @@
+#!/bin/sh
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+project_id="74"
+project_branch="rebase/glitch-soc"
+static_dir="instance/static"
+# For bundling:
+# project_branch="pleroma"
+# static_dir="priv/static"
+
+if [[ ! -d "${static_dir}" ]]
+then
+	echo "Error: ${static_dir} directory is missing, are you sure you are running this script at the root of pleroma’s repository?"
+	exit 1
+fi
+
+last_modified="$(curl -s -I 'https://git.pleroma.social/api/v4/projects/'${project_id}'/jobs/artifacts/'${project_branch}'/download?job=build' | grep '^Last-Modified:' | cut -d: -f2-)"
+
+echo "branch:${project_branch}"
+echo "Last-Modified:${last_modified}"
+
+artifact="mastofe.zip"
+
+if [[ -e mastofe.timestamp ]] && [[ "${last_modified}" != "" ]]
+then
+	if [[ "$(cat mastofe.timestamp)" == "${last_modified}" ]]
+	then
+		echo "MastoFE is up-to-date, exiting…"
+		exit 0
+	fi
+fi
+
+curl -c - "https://git.pleroma.social/api/v4/projects/${project_id}/jobs/artifacts/${project_branch}/download?job=build" -o "${artifact}" || exit
+
+# TODO: Update the emoji as well
+rm -fr "${static_dir}/sw.js" "${static_dir}/packs" || exit
+unzip -q "${artifact}" || exit
+
+cp public/assets/sw.js "${static_dir}/sw.js" || exit
+cp -r public/packs "${static_dir}/packs" || exit
+
+echo "${last_modified}" > mastofe.timestamp
+rm -fr public
+rm -i "${artifact}"

lib/mix/tasks/benchmark.ex

@@ -0,0 +1,25 @@
+defmodule Mix.Tasks.Pleroma.Benchmark do
+  use Mix.Task
+  alias Mix.Tasks.Pleroma.Common
+
+  def run(["search"]) do
+    Common.start_pleroma()
+
+    Benchee.run(%{
+      "search" => fn ->
+        Pleroma.Web.MastodonAPI.MastodonAPIController.status_search(nil, "cofe")
+      end
+    })
+  end
+
+  def run(["tag"]) do
+    Common.start_pleroma()
+
+    Benchee.run(%{
+      "tag" => fn ->
+        %{"type" => "Create", "tag" => "cofe"}
+        |> Pleroma.Web.ActivityPub.ActivityPub.fetch_public_activities()
+      end
+    })
+  end
+end

lib/mix/tasks/pleroma/database.ex

@@ -4,6 +4,9 @@
 
 defmodule Mix.Tasks.Pleroma.Database do
   alias Mix.Tasks.Pleroma.Common
+  alias Pleroma.Conversation
+  alias Pleroma.Repo
+  alias Pleroma.User
   require Logger
   use Mix.Task
 
@@ -19,6 +22,14 @@ defmodule Mix.Tasks.Pleroma.Database do
 
     Options:
     - `--vacuum` - run `VACUUM FULL` after the embedded objects are replaced with their references
+
+  ## Create a conversation for all existing DMs. Can be safely re-run.
+
+      mix pleroma.database bump_all_conversations
+
+  ## Remove duplicated items from following and update followers count for all users
+
+      mix pleroma.database update_users_following_followers_counts
   """
   def run(["remove_embedded_objects" | args]) do
     {options, [], []} =
@@ -32,7 +43,7 @@ defmodule Mix.Tasks.Pleroma.Database do
     Common.start_pleroma()
     Logger.info("Removing embedded objects")
 
-    Pleroma.Repo.query!(
+    Repo.query!(
       "update activities set data = jsonb_set(data, '{object}'::text[], data->'object'->'id') where data->'object'->>'id' is not null;",
       [],
       timeout: :infinity
@@ -41,11 +52,24 @@ defmodule Mix.Tasks.Pleroma.Database do
     if Keyword.get(options, :vacuum) do
       Logger.info("Runnning VACUUM FULL")
 
-      Pleroma.Repo.query!(
+      Repo.query!(
         "vacuum full;",
         [],
         timeout: :infinity
       )
     end
   end
+
+  def run(["bump_all_conversations"]) do
+    Common.start_pleroma()
+    Conversation.bump_for_all_activities()
+  end
+
+  def run(["update_users_following_followers_counts"]) do
+    Common.start_pleroma()
+
+    users = Repo.all(User)
+    Enum.each(users, &User.remove_duplicated_following/1)
+    Enum.each(users, &User.update_follower_count/1)
+  end
 end

lib/mix/tasks/pleroma/emoji.ex

@@ -109,7 +109,7 @@ defmodule Mix.Tasks.Pleroma.Emoji do
           ])
         )
 
-        binary_archive = Tesla.get!(src_url).body
+        binary_archive = Tesla.get!(client(), src_url).body
         archive_sha = :crypto.hash(:sha256, binary_archive) |> Base.encode16()
 
         sha_status_text = ["SHA256 of ", :bright, pack_name, :normal, " source file is ", :bright]
@@ -137,7 +137,7 @@ defmodule Mix.Tasks.Pleroma.Emoji do
           ])
         )
 
-        files = Tesla.get!(files_url).body |> Poison.decode!()
+        files = Tesla.get!(client(), files_url).body |> Jason.decode!()
 
         IO.puts(IO.ANSI.format(["Unpacking ", :bright, pack_name]))
 
@@ -213,7 +213,7 @@ defmodule Mix.Tasks.Pleroma.Emoji do
 
     IO.puts("Downloading the pack and generating SHA256")
 
-    binary_archive = Tesla.get!(src).body
+    binary_archive = Tesla.get!(client(), src).body
     archive_sha = :crypto.hash(:sha256, binary_archive) |> Base.encode16()
 
     IO.puts("SHA256 is #{archive_sha}")
@@ -239,7 +239,7 @@ defmodule Mix.Tasks.Pleroma.Emoji do
 
     emoji_map = Pleroma.Emoji.make_shortcode_to_file_map(tmp_pack_dir, exts)
 
-    File.write!(files_name, Poison.encode!(emoji_map, pretty: true))
+    File.write!(files_name, Jason.encode!(emoji_map, pretty: true))
 
     IO.puts("""
 
@@ -248,11 +248,11 @@ defmodule Mix.Tasks.Pleroma.Emoji do
     """)
 
     if File.exists?("index.json") do
-      existing_data = File.read!("index.json") |> Poison.decode!()
+      existing_data = File.read!("index.json") |> Jason.decode!()
 
       File.write!(
         "index.json",
-        Poison.encode!(
+        Jason.encode!(
           Map.merge(
             existing_data,
             pack_json
@@ -263,16 +263,16 @@ defmodule Mix.Tasks.Pleroma.Emoji do
 
       IO.puts("index.json file has been update with the #{name} pack")
     else
-      File.write!("index.json", Poison.encode!(pack_json, pretty: true))
+      File.write!("index.json", Jason.encode!(pack_json, pretty: true))
 
       IO.puts("index.json has been created with the #{name} pack")
     end
   end
 
   defp fetch_manifest(from) do
-    Poison.decode!(
+    Jason.decode!(
       if String.starts_with?(from, "http") do
-        Tesla.get!(from).body
+        Tesla.get!(client(), from).body
       else
         File.read!(from)
       end
@@ -290,4 +290,12 @@ defmodule Mix.Tasks.Pleroma.Emoji do
       ]
     )
   end
+
+  defp client do
+    middleware = [
+      {Tesla.Middleware.FollowRedirects, [max_redirects: 3]}
+    ]
+
+    Tesla.client(middleware)
+  end
 end

lib/mix/tasks/pleroma/user.ex

@@ -77,6 +77,10 @@ defmodule Mix.Tasks.Pleroma.User do
   ## Delete tags from a user.
 
       mix pleroma.user untag NICKNAME TAGS
+
+  ## Toggle confirmation of the user's account.
+
+      mix pleroma.user toggle_confirmed NICKNAME
   """
   def run(["new", nickname, email | rest]) do
     {options, [], []} =
@@ -126,7 +130,7 @@ defmodule Mix.Tasks.Pleroma.User do
 
     proceed? = assume_yes? or Mix.shell().yes?("Continue?")
 
-    unless not proceed? do
+    if proceed? do
       Common.start_pleroma()
 
       params = %{
@@ -138,7 +142,7 @@ defmodule Mix.Tasks.Pleroma.User do
         bio: bio
       }
 
-      changeset = User.register_changeset(%User{}, params, confirmed: true)
+      changeset = User.register_changeset(%User{}, params, need_confirmation: false)
       {:ok, _user} = User.register(changeset)
 
       Mix.shell().info("User #{nickname} created")
@@ -163,7 +167,7 @@ defmodule Mix.Tasks.Pleroma.User do
     Common.start_pleroma()
 
     with %User{local: true} = user <- User.get_cached_by_nickname(nickname) do
-      User.delete(user)
+      User.perform(:delete, user)
       Mix.shell().info("User #{nickname} deleted.")
     else
       _ ->
@@ -380,7 +384,7 @@ defmodule Mix.Tasks.Pleroma.User do
     Common.start_pleroma()
 
     with %User{local: true} = user <- User.get_cached_by_nickname(nickname) do
-      User.delete_user_activities(user)
+      {:ok, _} = User.delete_user_activities(user)
       Mix.shell().info("User #{nickname} statuses deleted.")
     else
       _ ->
@@ -388,6 +392,21 @@ defmodule Mix.Tasks.Pleroma.User do
     end
   end
 
+  def run(["toggle_confirmed", nickname]) do
+    Common.start_pleroma()
+
+    with %User{} = user <- User.get_cached_by_nickname(nickname) do
+      {:ok, user} = User.toggle_confirmation(user)
+
+      message = if user.info.confirmation_pending, do: "needs", else: "doesn't need"
+
+      Mix.shell().info("#{nickname} #{message} confirmation.")
+    else
+      _ ->
+        Mix.shell().error("No local user #{nickname}")
+    end
+  end
+
   defp set_moderator(user, value) do
     info_cng = User.Info.admin_api_update(user.info, %{is_moderator: value})
 

lib/pleroma/activity.ex

@@ -6,14 +6,19 @@ defmodule Pleroma.Activity do
   use Ecto.Schema
 
   alias Pleroma.Activity
+  alias Pleroma.Bookmark
   alias Pleroma.Notification
   alias Pleroma.Object
   alias Pleroma.Repo
+  alias Pleroma.ThreadMute
+  alias Pleroma.User
 
   import Ecto.Changeset
   import Ecto.Query
 
   @type t :: %__MODULE__{}
+  @type actor :: String.t()
+
   @primary_key {:id, Pleroma.FlakeId, autogenerate: true}
 
   # https://github.com/tootsuite/mastodon/blob/master/app/models/notification.rb#L19
@@ -33,6 +38,9 @@ defmodule Pleroma.Activity do
     field(:local, :boolean, default: true)
     field(:actor, :string)
     field(:recipients, {:array, :string}, default: [])
+    field(:thread_muted?, :boolean, virtual: true)
+    # This is a fake relation, do not use outside of with_preloaded_bookmark/get_bookmark
+    has_one(:bookmark, Bookmark)
     has_many(:notifications, Notification, on_delete: :delete_all)
 
     # Attention: this is a fake relation, don't try to preload it blindly and expect it to work!
@@ -54,23 +62,46 @@ defmodule Pleroma.Activity do
     timestamps()
   end
 
-  def with_preloaded_object(query) do
-    query
-    |> join(
-      :inner,
-      [activity],
-      o in Object,
+  def with_joined_object(query) do
+    join(query, :inner, [activity], o in Object,
       on:
         fragment(
           "(?->>'id') = COALESCE(?->'object'->>'id', ?->>'object')",
           o.data,
           activity.data,
           activity.data
-        )
+        ),
+      as: :object
     )
-    |> preload([activity, object], object: object)
   end
 
+  def with_preloaded_object(query) do
+    query
+    |> has_named_binding?(:object)
+    |> if(do: query, else: with_joined_object(query))
+    |> preload([activity, object: object], object: object)
+  end
+
+  def with_preloaded_bookmark(query, %User{} = user) do
+    from([a] in query,
+      left_join: b in Bookmark,
+      on: b.user_id == ^user.id and b.activity_id == a.id,
+      preload: [bookmark: b]
+    )
+  end
+
+  def with_preloaded_bookmark(query, _), do: query
+
+  def with_set_thread_muted_field(query, %User{} = user) do
+    from([a] in query,
+      left_join: tm in ThreadMute,
+      on: tm.user_id == ^user.id and tm.context == fragment("?->>'context'", a.data),
+      select: %Activity{a | thread_muted?: not is_nil(tm.id)}
+    )
+  end
+
+  def with_set_thread_muted_field(query, _), do: query
+
   def get_by_ap_id(ap_id) do
     Repo.one(
       from(
@@ -80,9 +111,19 @@ defmodule Pleroma.Activity do
     )
   end
 
+  def get_bookmark(%Activity{} = activity, %User{} = user) do
+    if Ecto.assoc_loaded?(activity.bookmark) do
+      activity.bookmark
+    else
+      Bookmark.get(user.id, activity.id)
+    end
+  end
+
+  def get_bookmark(_, _), do: nil
+
   def change(struct, params \\ %{}) do
     struct
-    |> cast(params, [:data])
+    |> cast(params, [:data, :recipients])
     |> validate_required([:data])
     |> unique_constraint(:ap_id, name: :activities_unique_apid_index)
   end
@@ -106,7 +147,10 @@ defmodule Pleroma.Activity do
   end
 
   def get_by_id(id) do
-    Repo.get(Activity, id)
+    Activity
+    |> where([a], a.id == ^id)
+    |> restrict_deactivated_users()
+    |> Repo.one()
   end
 
   def get_by_id_with_object(id) do
@@ -174,6 +218,7 @@ defmodule Pleroma.Activity do
 
   def get_create_by_object_ap_id(ap_id) when is_binary(ap_id) do
     create_by_object_ap_id(ap_id)
+    |> restrict_deactivated_users()
     |> Repo.one()
   end
 
@@ -260,4 +305,42 @@ defmodule Pleroma.Activity do
     |> where([s], s.actor == ^actor)
     |> Repo.all()
   end
+
+  def follow_requests_for_actor(%Pleroma.User{ap_id: ap_id}) do
+    from(
+      a in Activity,
+      where:
+        fragment(
+          "? ->> 'type' = 'Follow'",
+          a.data
+        ),
+      where:
+        fragment(
+          "? ->> 'state' = 'pending'",
+          a.data
+        ),
+      where:
+        fragment(
+          "coalesce((?)->'object'->>'id', (?)->>'object') = ?",
+          a.data,
+          a.data,
+          ^ap_id
+        )
+    )
+  end
+
+  @spec query_by_actor(actor()) :: Ecto.Query.t()
+  def query_by_actor(actor) do
+    from(a in Activity, where: a.actor == ^actor)
+  end
+
+  def restrict_deactivated_users(query) do
+    from(activity in query,
+      where:
+        fragment(
+          "? not in (SELECT ap_id FROM users WHERE info->'deactivated' @> 'true')",
+          activity.actor
+        )
+    )
+  end
 end

lib/pleroma/application.ex

@@ -131,19 +131,22 @@ defmodule Pleroma.Application do
   defp setup_instrumenters do
     require Prometheus.Registry
 
-    :ok =
-      :telemetry.attach(
-        "prometheus-ecto",
-        [:pleroma, :repo, :query],
-        &Pleroma.Repo.Instrumenter.handle_event/4,
-        %{}
-      )
+    if Application.get_env(:prometheus, Pleroma.Repo.Instrumenter) do
+      :ok =
+        :telemetry.attach(
+          "prometheus-ecto",
+          [:pleroma, :repo, :query],
+          &Pleroma.Repo.Instrumenter.handle_event/4,
+          %{}
+        )
+
+      Pleroma.Repo.Instrumenter.setup()
+    end
 
     Prometheus.Registry.register_collector(:prometheus_process_collector)
     Pleroma.Web.Endpoint.MetricsExporter.setup()
     Pleroma.Web.Endpoint.PipelineInstrumenter.setup()
     Pleroma.Web.Endpoint.Instrumenter.setup()
-    Pleroma.Repo.Instrumenter.setup()
   end
 
   def enabled_hackney_pools do

lib/pleroma/bbs/authenticator.ex

@@ -0,0 +1,16 @@
+defmodule Pleroma.BBS.Authenticator do
+  use Sshd.PasswordAuthenticator
+  alias Comeonin.Pbkdf2
+  alias Pleroma.User
+
+  def authenticate(username, password) do
+    username = to_string(username)
+    password = to_string(password)
+
+    with %User{} = user <- User.get_by_nickname(username) do
+      Pbkdf2.checkpw(password, user.password_hash)
+    else
+      _e -> false
+    end
+  end
+end

lib/pleroma/bbs/handler.ex

@@ -0,0 +1,146 @@
+defmodule Pleroma.BBS.Handler do
+  use Sshd.ShellHandler
+  alias Pleroma.Activity
+  alias Pleroma.Web.ActivityPub.ActivityPub
+  alias Pleroma.Web.CommonAPI
+
+  def on_shell(username, _pubkey, _ip, _port) do
+    :ok = IO.puts("Welcome to #{Pleroma.Config.get([:instance, :name])}!")
+    user = Pleroma.User.get_cached_by_nickname(to_string(username))
+    Logger.debug("#{inspect(user)}")
+    loop(run_state(user: user))
+  end
+
+  def on_connect(username, ip, port, method) do
+    Logger.debug(fn ->
+      """
+      Incoming SSH shell #{inspect(self())} requested for #{username} from #{inspect(ip)}:#{
+        inspect(port)
+      } using #{inspect(method)}
+      """
+    end)
+  end
+
+  def on_disconnect(username, ip, port) do
+    Logger.debug(fn ->
+      "Disconnecting SSH shell for #{username} from #{inspect(ip)}:#{inspect(port)}"
+    end)
+  end
+
+  defp loop(state) do
+    self_pid = self()
+    counter = state.counter
+    prefix = state.prefix
+    user = state.user
+
+    input = spawn(fn -> io_get(self_pid, prefix, counter, user.nickname) end)
+    wait_input(state, input)
+  end
+
+  def puts_activity(activity) do
+    status = Pleroma.Web.MastodonAPI.StatusView.render("status.json", %{activity: activity})
+    IO.puts("-- #{status.id} by #{status.account.display_name} (#{status.account.acct})")
+    IO.puts(HtmlSanitizeEx.strip_tags(status.content))
+    IO.puts("")
+  end
+
+  def handle_command(state, "help") do
+    IO.puts("Available commands:")
+    IO.puts("help - This help")
+    IO.puts("home - Show the home timeline")
+    IO.puts("p <text> - Post the given text")
+    IO.puts("r <id> <text> - Reply to the post with the given id")
+    IO.puts("quit - Quit")
+
+    state
+  end
+
+  def handle_command(%{user: user} = state, "r " <> text) do
+    text = String.trim(text)
+    [activity_id, rest] = String.split(text, " ", parts: 2)
+
+    with %Activity{} <- Activity.get_by_id(activity_id),
+         {:ok, _activity} <-
+           CommonAPI.post(user, %{"status" => rest, "in_reply_to_status_id" => activity_id}) do
+      IO.puts("Replied!")
+    else
+      _e -> IO.puts("Could not reply...")
+    end
+
+    state
+  end
+
+  def handle_command(%{user: user} = state, "p " <> text) do
+    text = String.trim(text)
+
+    with {:ok, _activity} <- CommonAPI.post(user, %{"status" => text}) do
+      IO.puts("Posted!")
+    else
+      _e -> IO.puts("Could not post...")
+    end
+
+    state
+  end
+
+  def handle_command(state, "home") do
+    user = state.user
+
+    params =
+      %{}
+      |> Map.put("type", ["Create"])
+      |> Map.put("blocking_user", user)
+      |> Map.put("muting_user", user)
+      |> Map.put("user", user)
+
+    activities =
+      [user.ap_id | user.following]
+      |> ActivityPub.fetch_activities(params)
+
+    Enum.each(activities, fn activity ->
+      puts_activity(activity)
+    end)
+
+    state
+  end
+
+  def handle_command(state, command) do
+    IO.puts("Unknown command '#{command}'")
+    state
+  end
+
+  defp wait_input(state, input) do
+    receive do
+      {:input, ^input, "quit\n"} ->
+        IO.puts("Exiting...")
+
+      {:input, ^input, code} when is_binary(code) ->
+        code = String.trim(code)
+
+        state = handle_command(state, code)
+
+        loop(%{state | counter: state.counter + 1})
+
+      {:error, :interrupted} ->
+        IO.puts("Caught Ctrl+C...")
+        loop(%{state | counter: state.counter + 1})
+
+      {:input, ^input, msg} ->
+        :ok = Logger.warn("received unknown message: #{inspect(msg)}")
+        loop(%{state | counter: state.counter + 1})
+    end
+  end
+
+  defp run_state(opts) do
+    %{prefix: "pleroma", counter: 1, user: opts[:user]}
+  end
+
+  defp io_get(pid, prefix, counter, username) do
+    prompt = prompt(prefix, counter, username)
+    send(pid, {:input, self(), IO.gets(:stdio, prompt)})
+  end
+
+  defp prompt(prefix, counter, username) do
+    prompt = "#{username}@#{prefix}:#{counter}>"
+    prompt <> " "
+  end
+end

lib/pleroma/captcha/kocaptcha.ex

@@ -15,7 +15,7 @@ defmodule Pleroma.Captcha.Kocaptcha do
         %{error: "Kocaptcha service unavailable"}
 
       {:ok, res} ->
-        json_resp = Poison.decode!(res.body)
+        json_resp = Jason.decode!(res.body)
 
         %{
           type: :kocaptcha,

lib/pleroma/config.ex

@@ -12,8 +12,12 @@ defmodule Pleroma.Config do
   def get([key], default), do: get(key, default)
 
   def get([parent_key | keys], default) do
-    Application.get_env(:pleroma, parent_key)
-    |> get_in(keys) || default
+    case :pleroma
+         |> Application.get_env(parent_key)
+         |> get_in(keys) do
+      nil -> default
+      any -> any
+    end
   end
 
   def get(key, default) do

lib/pleroma/config/deprecation_warnings.ex

@@ -5,15 +5,6 @@
 defmodule Pleroma.Config.DeprecationWarnings do
   require Logger
 
-  def check_frontend_config_mechanism do
-    if Pleroma.Config.get(:fe) do
-      Logger.warn("""
-      !!!DEPRECATION WARNING!!!
-      You are using the old configuration mechanism for the frontend. Please check config.md.
-      """)
-    end
-  end
-
   def check_hellthread_threshold do
     if Pleroma.Config.get([:mrf_hellthread, :threshold]) do
       Logger.warn("""
@@ -24,7 +15,6 @@ defmodule Pleroma.Config.DeprecationWarnings do
   end
 
   def warn do
-    check_frontend_config_mechanism()
     check_hellthread_threshold()
   end
 end

lib/pleroma/conversation.ex

@@ -0,0 +1,92 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Conversation do
+  alias Pleroma.Conversation.Participation
+  alias Pleroma.Repo
+  alias Pleroma.User
+  use Ecto.Schema
+  import Ecto.Changeset
+
+  schema "conversations" do
+    # This is the context ap id.
+    field(:ap_id, :string)
+    has_many(:participations, Participation)
+    has_many(:users, through: [:participations, :user])
+
+    timestamps()
+  end
+
+  def creation_cng(struct, params) do
+    struct
+    |> cast(params, [:ap_id])
+    |> validate_required([:ap_id])
+    |> unique_constraint(:ap_id)
+  end
+
+  def create_for_ap_id(ap_id) do
+    %__MODULE__{}
+    |> creation_cng(%{ap_id: ap_id})
+    |> Repo.insert(
+      on_conflict: [set: [updated_at: NaiveDateTime.utc_now()]],
+      returning: true,
+      conflict_target: :ap_id
+    )
+  end
+
+  def get_for_ap_id(ap_id) do
+    Repo.get_by(__MODULE__, ap_id: ap_id)
+  end
+
+  @doc """
+  This will
+  1. Create a conversation if there isn't one already
+  2. Create a participation for all the people involved who don't have one already
+  3. Bump all relevant participations to 'unread'
+  """
+  def create_or_bump_for(activity, opts \\ []) do
+    with true <- Pleroma.Web.ActivityPub.Visibility.is_direct?(activity),
+         "Create" <- activity.data["type"],
+         object <- Pleroma.Object.normalize(activity),
+         "Note" <- object.data["type"],
+         ap_id when is_binary(ap_id) and byte_size(ap_id) > 0 <- object.data["context"] do
+      {:ok, conversation} = create_for_ap_id(ap_id)
+
+      users = User.get_users_from_set(activity.recipients, false)
+
+      participations =
+        Enum.map(users, fn user ->
+          {:ok, participation} =
+            Participation.create_for_user_and_conversation(user, conversation, opts)
+
+          participation
+        end)
+
+      {:ok,
+       %{
+         conversation
+         | participations: participations
+       }}
+    else
+      e -> {:error, e}
+    end
+  end
+
+  @doc """
+  This is only meant to be run by a mix task. It creates conversations/participations for all direct messages in the database.
+  """
+  def bump_for_all_activities do
+    stream =
+      Pleroma.Web.ActivityPub.ActivityPub.fetch_direct_messages_query()
+      |> Repo.stream()
+
+    Repo.transaction(
+      fn ->
+        stream
+        |> Enum.each(fn a -> create_or_bump_for(a, read: true) end)
+      end,
+      timeout: :infinity
+    )
+  end
+end

lib/pleroma/conversation/participation.ex

@@ -0,0 +1,83 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Conversation.Participation do
+  use Ecto.Schema
+  alias Pleroma.Conversation
+  alias Pleroma.Repo
+  alias Pleroma.User
+  alias Pleroma.Web.ActivityPub.ActivityPub
+  import Ecto.Changeset
+  import Ecto.Query
+
+  schema "conversation_participations" do
+    belongs_to(:user, User, type: Pleroma.FlakeId)
+    belongs_to(:conversation, Conversation)
+    field(:read, :boolean, default: false)
+    field(:last_activity_id, Pleroma.FlakeId, virtual: true)
+
+    timestamps()
+  end
+
+  def creation_cng(struct, params) do
+    struct
+    |> cast(params, [:user_id, :conversation_id, :read])
+    |> validate_required([:user_id, :conversation_id])
+  end
+
+  def create_for_user_and_conversation(user, conversation, opts \\ []) do
+    read = !!opts[:read]
+
+    %__MODULE__{}
+    |> creation_cng(%{user_id: user.id, conversation_id: conversation.id, read: read})
+    |> Repo.insert(
+      on_conflict: [set: [read: read, updated_at: NaiveDateTime.utc_now()]],
+      returning: true,
+      conflict_target: [:user_id, :conversation_id]
+    )
+  end
+
+  def read_cng(struct, params) do
+    struct
+    |> cast(params, [:read])
+    |> validate_required([:read])
+  end
+
+  def mark_as_read(participation) do
+    participation
+    |> read_cng(%{read: true})
+    |> Repo.update()
+  end
+
+  def mark_as_unread(participation) do
+    participation
+    |> read_cng(%{read: false})
+    |> Repo.update()
+  end
+
+  def for_user(user, params \\ %{}) do
+    from(p in __MODULE__,
+      where: p.user_id == ^user.id,
+      order_by: [desc: p.updated_at]
+    )
+    |> Pleroma.Pagination.fetch_paginated(params)
+    |> Repo.preload(conversation: [:users])
+  end
+
+  def for_user_with_last_activity_id(user, params \\ %{}) do
+    for_user(user, params)
+    |> Enum.map(fn participation ->
+      activity_id =
+        ActivityPub.fetch_latest_activity_id_for_context(participation.conversation.ap_id, %{
+          "user" => user,
+          "blocking_user" => user
+        })
+
+      %{
+        participation
+        | last_activity_id: activity_id
+      }
+    end)
+  end
+end

lib/pleroma/emails/admin_email.ex

@@ -29,7 +29,7 @@ defmodule Pleroma.Emails.AdminEmail do
       end
 
     statuses_html =
-      if length(statuses) > 0 do
+      if is_list(statuses) && length(statuses) > 0 do
         statuses_list_html =
           statuses
           |> Enum.map(fn

lib/pleroma/filter.ex

@@ -38,7 +38,8 @@ defmodule Pleroma.Filter do
     query =
       from(
         f in Pleroma.Filter,
-        where: f.user_id == ^user_id
+        where: f.user_id == ^user_id,
+        order_by: [desc: :id]
       )
 
     Repo.all(query)

lib/pleroma/formatter.ex

@@ -113,9 +113,7 @@ defmodule Pleroma.Formatter do
 
       html =
         if not strip do
-          "<img height='32px' width='32px' alt='#{emoji}' title='#{emoji}' src='#{
-            MediaProxy.url(file)
-          }' />"
+          "<img class='emoji' alt='#{emoji}' title='#{emoji}' src='#{MediaProxy.url(file)}' />"
         else
           ""
         end
@@ -130,12 +128,23 @@ defmodule Pleroma.Formatter do
 
   def demojify(text, nil), do: text
 
+  @doc "Outputs a list of the emoji-shortcodes in a text"
   def get_emoji(text) when is_binary(text) do
     Enum.filter(Emoji.get_all(), fn {emoji, _, _} -> String.contains?(text, ":#{emoji}:") end)
   end
 
   def get_emoji(_), do: []
 
+  @doc "Outputs a list of the emoji-Maps in a text"
+  def get_emoji_map(text) when is_binary(text) do
+    get_emoji(text)
+    |> Enum.reduce(%{}, fn {name, file, _group}, acc ->
+      Map.put(acc, name, "#{Pleroma.Web.Endpoint.static_url()}#{file}")
+    end)
+  end
+
+  def get_emoji_map(_), do: []
+
   def html_escape({text, mentions, hashtags}, type) do
     {html_escape(text, type), mentions, hashtags}
   end

lib/pleroma/gopher/server.ex

@@ -77,13 +77,13 @@ defmodule Pleroma.Gopher.Server.ProtocolHandler do
       user = User.get_cached_by_ap_id(activity.data["actor"])
 
       object = Object.normalize(activity)
-      like_count = object["like_count"] || 0
-      announcement_count = object["announcement_count"] || 0
+      like_count = object.data["like_count"] || 0
+      announcement_count = object.data["announcement_count"] || 0
 
       link("Post ##{activity.id} by #{user.nickname}", "/notices/#{activity.id}") <>
         info("#{like_count} likes, #{announcement_count} repeats") <>
         "i\tfake\t(NULL)\t0\r\n" <>
-        info(HTML.strip_tags(String.replace(object["content"], "<br>", "\r")))
+        info(HTML.strip_tags(String.replace(object.data["content"], "<br>", "\r")))
     end)
     |> Enum.join("i\tfake\t(NULL)\t0\r\n")
   end

lib/pleroma/html.ex

@@ -28,12 +28,18 @@ defmodule Pleroma.HTML do
   def filter_tags(html), do: filter_tags(html, nil)
   def strip_tags(html), do: Scrubber.scrub(html, Scrubber.StripTags)
 
-  def get_cached_scrubbed_html_for_activity(content, scrubbers, activity, key \\ "") do
+  def get_cached_scrubbed_html_for_activity(
+        content,
+        scrubbers,
+        activity,
+        key \\ "",
+        callback \\ fn x -> x end
+      ) do
     key = "#{key}#{generate_scrubber_signature(scrubbers)}|#{activity.id}"
 
     Cachex.fetch!(:scrubber_cache, key, fn _key ->
       object = Pleroma.Object.normalize(activity)
-      ensure_scrubbed_html(content, scrubbers, object.data["fake"] || false)
+      ensure_scrubbed_html(content, scrubbers, object.data["fake"] || false, callback)
     end)
   end
 
@@ -42,24 +48,27 @@ defmodule Pleroma.HTML do
       content,
       HtmlSanitizeEx.Scrubber.StripTags,
       activity,
-      key
+      key,
+      &HtmlEntities.decode/1
     )
   end
 
   def ensure_scrubbed_html(
         content,
         scrubbers,
-        false = _fake
+        fake,
+        callback
       ) do
-    {:commit, filter_tags(content, scrubbers)}
-  end
+    content =
+      content
+      |> filter_tags(scrubbers)
+      |> callback.()
 
-  def ensure_scrubbed_html(
-        content,
-        scrubbers,
-        true = _fake
-      ) do
-    {:ignore, filter_tags(content, scrubbers)}
+    if fake do
+      {:ignore, content}
+    else
+      {:commit, content}
+    end
   end
 
   defp generate_scrubber_signature(scrubber) when is_atom(scrubber) do
@@ -142,6 +151,7 @@ defmodule Pleroma.HTML.Scrubber.TwitterText do
     Meta.allow_tag_with_these_attributes("img", [
       "width",
       "height",
+      "class",
       "title",
       "alt"
     ])
@@ -212,6 +222,7 @@ defmodule Pleroma.HTML.Scrubber.Default do
     Meta.allow_tag_with_these_attributes("img", [
       "width",
       "height",
+      "class",
       "title",
       "alt"
     ])

lib/pleroma/http/connection.ex

@@ -8,7 +8,7 @@ defmodule Pleroma.HTTP.Connection do
   """
 
   @hackney_options [
-    connect_timeout: 2_000,
+    connect_timeout: 10_000,
     recv_timeout: 20_000,
     follow_redirect: true,
     pool: :federation

lib/pleroma/http/request_builder.ex

@@ -45,8 +45,15 @@ defmodule Pleroma.HTTP.RequestBuilder do
   Add headers to the request
   """
   @spec headers(map(), list(tuple)) :: map()
-  def headers(request, h) do
-    Map.put_new(request, :headers, h)
+  def headers(request, header_list) do
+    header_list =
+      if Pleroma.Config.get([:http, :send_user_agent]) do
+        header_list ++ [{"User-Agent", Pleroma.Application.user_agent()}]
+      else
+        header_list
+      end
+
+    Map.put_new(request, :headers, header_list)
   end
 
   @doc """

lib/pleroma/notification.ex

@@ -33,6 +33,13 @@ defmodule Pleroma.Notification do
   def for_user_query(user) do
     Notification
     |> where(user_id: ^user.id)
+    |> where(
+      [n, a],
+      fragment(
+        "? not in (SELECT ap_id FROM users WHERE info->'deactivated' @> 'true')",
+        a.actor
+      )
+    )
     |> join(:inner, [n], activity in assoc(n, :activity))
     |> join(:left, [n, a], object in Object,
       on:

lib/pleroma/object/containment.ex

@@ -1,7 +1,5 @@
 defmodule Pleroma.Object.Containment do
   @moduledoc """
-  # Object Containment
-
   This module contains some useful functions for containing objects to specific
   origins and determining those origins.  They previously lived in the
   ActivityPub `Transmogrifier` module.

lib/pleroma/plugs/ensure_public_or_authenticated_plug.ex

@@ -0,0 +1,31 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Plugs.EnsurePublicOrAuthenticatedPlug do
+  import Plug.Conn
+  alias Pleroma.Config
+  alias Pleroma.User
+
+  def init(options) do
+    options
+  end
+
+  def call(conn, _) do
+    public? = Config.get!([:instance, :public])
+
+    case {public?, conn} do
+      {true, _} ->
+        conn
+
+      {false, %{assigns: %{user: %User{}}}} ->
+        conn
+
+      {false, _} ->
+        conn
+        |> put_resp_content_type("application/json")
+        |> send_resp(403, Jason.encode!(%{error: "This resource requires authentication."}))
+        |> halt
+    end
+  end
+end

lib/pleroma/plugs/http_security_plug.ex

@@ -20,8 +20,9 @@ defmodule Pleroma.Plugs.HTTPSecurityPlug do
 
   defp headers do
     referrer_policy = Config.get([:http_security, :referrer_policy])
+    report_uri = Config.get([:http_security, :report_uri])
 
-    [
+    headers = [
       {"x-xss-protection", "1; mode=block"},
       {"x-permitted-cross-domain-policies", "none"},
       {"x-frame-options", "DENY"},
@@ -30,12 +31,27 @@ defmodule Pleroma.Plugs.HTTPSecurityPlug do
       {"x-download-options", "noopen"},
       {"content-security-policy", csp_string() <> ";"}
     ]
+
+    if report_uri do
+      report_group = %{
+        "group" => "csp-endpoint",
+        "max-age" => 10_886_400,
+        "endpoints" => [
+          %{"url" => report_uri}
+        ]
+      }
+
+      headers ++ [{"reply-to", Jason.encode!(report_group)}]
+    else
+      headers
+    end
   end
 
   defp csp_string do
     scheme = Config.get([Pleroma.Web.Endpoint, :url])[:scheme]
     static_url = Pleroma.Web.Endpoint.static_url()
-    websocket_url = String.replace(static_url, "http", "ws")
+    websocket_url = Pleroma.Web.Endpoint.websocket_url()
+    report_uri = Config.get([:http_security, :report_uri])
 
     connect_src = "connect-src 'self' #{static_url} #{websocket_url}"
 
@@ -53,7 +69,7 @@ defmodule Pleroma.Plugs.HTTPSecurityPlug do
         "script-src 'self'"
       end
 
-    [
+    main_part = [
       "default-src 'none'",
       "base-uri 'self'",
       "frame-ancestors 'none'",
@@ -63,11 +79,14 @@ defmodule Pleroma.Plugs.HTTPSecurityPlug do
       "font-src 'self'",
       "manifest-src 'self'",
       connect_src,
-      script_src,
-      if scheme == "https" do
-        "upgrade-insecure-requests"
-      end
+      script_src
     ]
+
+    report = if report_uri, do: ["report-uri #{report_uri}; report-to csp-endpoint"], else: []
+
+    insecure = if scheme == "https", do: ["upgrade-insecure-requests"], else: []
+
+    (main_part ++ report ++ insecure)
     |> Enum.join("; ")
   end
 

lib/pleroma/plugs/http_signature.ex

@@ -4,7 +4,6 @@
 
 defmodule Pleroma.Web.Plugs.HTTPSignaturePlug do
   alias Pleroma.Web.ActivityPub.Utils
-  alias Pleroma.Web.HTTPSignatures
   import Plug.Conn
   require Logger
 

lib/pleroma/plugs/oauth_plug.ex

@@ -8,6 +8,7 @@ defmodule Pleroma.Plugs.OAuthPlug do
 
   alias Pleroma.Repo
   alias Pleroma.User
+  alias Pleroma.Web.OAuth.App
   alias Pleroma.Web.OAuth.Token
 
   @realm_reg Regex.compile!("Bearer\:?\s+(.*)$", "i")
@@ -16,14 +17,45 @@ defmodule Pleroma.Plugs.OAuthPlug do
 
   def call(%{assigns: %{user: %User{}}} = conn, _), do: conn
 
-  def call(conn, _) do
-    with {:ok, token_str} <- fetch_token_str(conn),
-         {:ok, user, token_record} <- fetch_user_and_token(token_str) do
+  def call(%{params: %{"access_token" => access_token}} = conn, _) do
+    with {:ok, user, token_record} <- fetch_user_and_token(access_token) do
       conn
       |> assign(:token, token_record)
       |> assign(:user, user)
     else
-      _ -> conn
+      _ ->
+        # token found, but maybe only with app
+        with {:ok, app, token_record} <- fetch_app_and_token(access_token) do
+          conn
+          |> assign(:token, token_record)
+          |> assign(:app, app)
+        else
+          _ -> conn
+        end
+    end
+  end
+
+  def call(conn, _) do
+    case fetch_token_str(conn) do
+      {:ok, token} ->
+        with {:ok, user, token_record} <- fetch_user_and_token(token) do
+          conn
+          |> assign(:token, token_record)
+          |> assign(:user, user)
+        else
+          _ ->
+            # token found, but maybe only with app
+            with {:ok, app, token_record} <- fetch_app_and_token(token) do
+              conn
+              |> assign(:token, token_record)
+              |> assign(:app, app)
+            else
+              _ -> conn
+            end
+        end
+
+      _ ->
+        conn
     end
   end
 
@@ -44,6 +76,16 @@ defmodule Pleroma.Plugs.OAuthPlug do
     end
   end
 
+  @spec fetch_app_and_token(String.t()) :: {:ok, App.t(), Token.t()} | nil
+  defp fetch_app_and_token(token) do
+    query =
+      from(t in Token, where: t.token == ^token, join: app in assoc(t, :app), preload: [app: app])
+
+    with %Token{app: app} = token_record <- Repo.one(query) do
+      {:ok, app, token_record}
+    end
+  end
+
   # Gets token from session by :oauth_token key
   #
   @spec fetch_token_from_session(Plug.Conn.t()) :: :no_token_found | {:ok, String.t()}

lib/pleroma/plugs/rate_limit_plug.ex

@@ -0,0 +1,36 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Plugs.RateLimitPlug do
+  import Phoenix.Controller, only: [json: 2]
+  import Plug.Conn
+
+  def init(opts), do: opts
+
+  def call(conn, opts) do
+    enabled? = Pleroma.Config.get([:app_account_creation, :enabled])
+
+    case check_rate(conn, Map.put(opts, :enabled, enabled?)) do
+      {:ok, _count} -> conn
+      {:error, _count} -> render_error(conn)
+      %Plug.Conn{} = conn -> conn
+    end
+  end
+
+  defp check_rate(conn, %{enabled: true} = opts) do
+    max_requests = opts[:max_requests]
+    bucket_name = conn.remote_ip |> Tuple.to_list() |> Enum.join(".")
+
+    ExRated.check_rate(bucket_name, opts[:interval] * 1000, max_requests)
+  end
+
+  defp check_rate(conn, _), do: conn
+
+  defp render_error(conn) do
+    conn
+    |> put_status(:forbidden)
+    |> json(%{error: "Rate limit exceeded."})
+    |> halt()
+  end
+end

lib/pleroma/repo.ex

@@ -19,4 +19,32 @@ defmodule Pleroma.Repo do
   def init(_, opts) do
     {:ok, Keyword.put(opts, :url, System.get_env("DATABASE_URL"))}
   end
+
+  @doc "find resource based on prepared query"
+  @spec find_resource(Ecto.Query.t()) :: {:ok, struct()} | {:error, :not_found}
+  def find_resource(%Ecto.Query{} = query) do
+    case __MODULE__.one(query) do
+      nil -> {:error, :not_found}
+      resource -> {:ok, resource}
+    end
+  end
+
+  def find_resource(_query), do: {:error, :not_found}
+
+  @doc """
+  Gets association from cache or loads if need
+
+  ## Examples
+
+    iex> Repo.get_assoc(token, :user)
+    %User{}
+
+  """
+  @spec get_assoc(struct(), atom()) :: {:ok, struct()} | {:error, :not_found}
+  def get_assoc(resource, association) do
+    case __MODULE__.preload(resource, association) do
+      %{^association => assoc} when not is_nil(assoc) -> {:ok, assoc}
+      _ -> {:error, :not_found}
+    end
+  end
 end

lib/pleroma/signature.ex

@@ -0,0 +1,41 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Signature do
+  @behaviour HTTPSignatures.Adapter
+
+  alias Pleroma.User
+  alias Pleroma.Web.ActivityPub.ActivityPub
+  alias Pleroma.Web.ActivityPub.Utils
+  alias Pleroma.Web.Salmon
+  alias Pleroma.Web.WebFinger
+
+  def fetch_public_key(conn) do
+    with actor_id <- Utils.get_ap_id(conn.params["actor"]),
+         {:ok, public_key} <- User.get_public_key_for_ap_id(actor_id) do
+      {:ok, public_key}
+    else
+      e ->
+        {:error, e}
+    end
+  end
+
+  def refetch_public_key(conn) do
+    with actor_id <- Utils.get_ap_id(conn.params["actor"]),
+         {:ok, _user} <- ActivityPub.make_user_from_ap_id(actor_id),
+         {:ok, public_key} <- User.get_public_key_for_ap_id(actor_id) do
+      {:ok, public_key}
+    else
+      e ->
+        {:error, e}
+    end
+  end
+
+  def sign(%User{} = user, headers) do
+    with {:ok, %{info: %{keys: keys}}} <- WebFinger.ensure_keys_present(user),
+         {:ok, private_key, _} <- Salmon.keys_from_pem(keys) do
+      HTTPSignatures.sign(private_key, user.ap_id <> "#main-key", headers)
+    end
+  end
+end

lib/pleroma/stats.ex

@@ -34,7 +34,7 @@ defmodule Pleroma.Stats do
   def update_stats do
     peers =
       from(
-        u in Pleroma.User,
+        u in User,
         select: fragment("distinct split_part(?, '@', 2)", u.nickname),
         where: u.local != ^true
       )
@@ -44,10 +44,13 @@ defmodule Pleroma.Stats do
     domain_count = Enum.count(peers)
 
     status_query =
-      from(u in User.local_user_query(), select: fragment("sum((?->>'note_count')::int)", u.info))
+      from(u in User.Query.build(%{local: true}),
+        select: fragment("sum((?->>'note_count')::int)", u.info)
+      )
 
     status_count = Repo.one(status_query)
-    user_count = Repo.aggregate(User.active_local_user_query(), :count, :id)
+
+    user_count = Repo.aggregate(User.Query.build(%{local: true, active: true}), :count, :id)
 
     Agent.update(__MODULE__, fn _ ->
       {peers, %{domain_count: domain_count, status_count: status_count, user_count: user_count}}

lib/pleroma/upload.ex

@@ -4,7 +4,7 @@
 
 defmodule Pleroma.Upload do
   @moduledoc """
-  # Upload
+  Manage user uploads
 
   Options:
   * `:type`: presets for activity type (defaults to Document) and size limits from app configuration

lib/pleroma/uploaders/swift/keystone.ex

@@ -14,7 +14,7 @@ defmodule Pleroma.Uploaders.Swift.Keystone do
 
   def process_response_body(body) do
     body
-    |> Poison.decode!()
+    |> Jason.decode!()
   end
 
   def get_token do
@@ -38,7 +38,7 @@ defmodule Pleroma.Uploaders.Swift.Keystone do
   end
 
   def make_auth_body(username, password, tenant) do
-    Poison.encode!(%{
+    Jason.encode!(%{
       :auth => %{
         :passwordCredentials => %{
           :username => username,

lib/pleroma/user.ex

@@ -10,8 +10,6 @@ defmodule Pleroma.User do
 
   alias Comeonin.Pbkdf2
   alias Pleroma.Activity
-  alias Pleroma.Bookmark
-  alias Pleroma.Formatter
   alias Pleroma.Notification
   alias Pleroma.Object
   alias Pleroma.Registration
@@ -55,10 +53,9 @@ defmodule Pleroma.User do
     field(:search_type, :integer, virtual: true)
     field(:tags, {:array, :string}, default: [])
     field(:last_refreshed_at, :naive_datetime_usec)
-    has_many(:bookmarks, Bookmark)
     has_many(:notifications, Notification)
     has_many(:registrations, Registration)
-    embeds_one(:info, Pleroma.User.Info)
+    embeds_one(:info, User.Info)
 
     timestamps()
   end
@@ -108,10 +105,8 @@ defmodule Pleroma.User do
   def ap_followers(%User{} = user), do: "#{ap_id(user)}/followers"
 
   def user_info(%User{} = user) do
-    oneself = if user.local, do: 1, else: 0
-
     %{
-      following_count: length(user.following) - oneself,
+      following_count: following_count(user),
       note_count: user.info.note_count,
       follower_count: user.info.follower_count,
       locked: user.info.locked,
@@ -120,6 +115,20 @@ defmodule Pleroma.User do
     }
   end
 
+  def restrict_deactivated(query) do
+    from(u in query,
+      where: not fragment("? \\? 'deactivated' AND ?->'deactivated' @> 'true'", u.info, u.info)
+    )
+  end
+
+  def following_count(%User{following: []}), do: 0
+
+  def following_count(%User{} = user) do
+    user
+    |> get_friends_query()
+    |> Repo.aggregate(:count, :id)
+  end
+
   def remote_user_creation(params) do
     params =
       params
@@ -157,7 +166,7 @@ defmodule Pleroma.User do
 
   def update_changeset(struct, params \\ %{}) do
     struct
-    |> cast(params, [:bio, :name, :avatar])
+    |> cast(params, [:bio, :name, :avatar, :following])
     |> unique_constraint(:nickname)
     |> validate_format(:nickname, local_nickname_regex())
     |> validate_length(:bio, max: 5000)
@@ -207,14 +216,15 @@ defmodule Pleroma.User do
   end
 
   def register_changeset(struct, params \\ %{}, opts \\ []) do
-    confirmation_status =
-      if opts[:confirmed] || !Pleroma.Config.get([:instance, :account_activation_required]) do
-        :confirmed
+    need_confirmation? =
+      if is_nil(opts[:need_confirmation]) do
+        Pleroma.Config.get([:instance, :account_activation_required])
       else
-        :unconfirmed
+        opts[:need_confirmation]
       end
 
-    info_change = User.Info.confirmation_changeset(%User.Info{}, confirmation_status)
+    info_change =
+      User.Info.confirmation_changeset(%User.Info{}, need_confirmation: need_confirmation?)
 
     changeset =
       struct
@@ -223,7 +233,7 @@ defmodule Pleroma.User do
       |> validate_confirmation(:password)
       |> unique_constraint(:email)
       |> unique_constraint(:nickname)
-      |> validate_exclusion(:nickname, Pleroma.Config.get([Pleroma.User, :restricted_nicknames]))
+      |> validate_exclusion(:nickname, Pleroma.Config.get([User, :restricted_nicknames]))
       |> validate_format(:nickname, local_nickname_regex())
       |> validate_format(:email, @email_regex)
       |> validate_length(:bio, max: 1000)
@@ -257,10 +267,7 @@ defmodule Pleroma.User do
     candidates = Pleroma.Config.get([:instance, :autofollowed_nicknames])
 
     autofollowed_users =
-      from(u in User,
-        where: u.local == true,
-        where: u.nickname in ^candidates
-      )
+      User.Query.build(%{nickname: candidates, local: true, deactivated: false})
       |> Repo.all()
 
     follow_all(user, autofollowed_users)
@@ -271,7 +278,7 @@ defmodule Pleroma.User do
     with {:ok, user} <- Repo.insert(changeset),
          {:ok, user} <- autofollow_users(user),
          {:ok, user} <- set_cache(user),
-         {:ok, _} <- Pleroma.User.WelcomeMessage.post_welcome_message_to_user(user),
+         {:ok, _} <- User.WelcomeMessage.post_welcome_message_to_user(user),
          {:ok, _} <- try_send_confirmation_email(user) do
       {:ok, user}
     end
@@ -418,24 +425,6 @@ defmodule Pleroma.User do
     Enum.member?(follower.following, followed.follower_address)
   end
 
-  def follow_import(%User{} = follower, followed_identifiers)
-      when is_list(followed_identifiers) do
-    Enum.map(
-      followed_identifiers,
-      fn followed_identifier ->
-        with %User{} = followed <- get_or_fetch(followed_identifier),
-             {:ok, follower} <- maybe_direct_follow(follower, followed),
-             {:ok, _} <- ActivityPub.follow(follower, followed) do
-          followed
-        else
-          err ->
-            Logger.debug("follow_import failed for #{followed_identifier} with: #{inspect(err)}")
-            err
-        end
-      end
-    )
-  end
-
   def locked?(%User{} = user) do
     user.info.locked || false
   end
@@ -507,7 +496,15 @@ defmodule Pleroma.User do
 
   def get_cached_by_nickname(nickname) do
     key = "nickname:#{nickname}"
-    Cachex.fetch!(:user_cache, key, fn _ -> get_or_fetch_by_nickname(nickname) end)
+
+    Cachex.fetch!(:user_cache, key, fn ->
+      user_result = get_or_fetch_by_nickname(nickname)
+
+      case user_result do
+        {:ok, user} -> {:commit, user}
+        {:error, _error} -> {:ignore, nil}
+      end
+    end)
   end
 
   def get_cached_by_nickname_or_id(nickname_or_id) do
@@ -543,47 +540,37 @@ defmodule Pleroma.User do
 
   def get_or_fetch_by_nickname(nickname) do
     with %User{} = user <- get_by_nickname(nickname) do
-      user
+      {:ok, user}
     else
       _e ->
         with [_nick, _domain] <- String.split(nickname, "@"),
              {:ok, user} <- fetch_by_nickname(nickname) do
           if Pleroma.Config.get([:fetch_initial_posts, :enabled]) do
-            # TODO turn into job
-            {:ok, _} = Task.start(__MODULE__, :fetch_initial_posts, [user])
+            fetch_initial_posts(user)
           end
 
-          user
+          {:ok, user}
         else
-          _e -> nil
+          _e -> {:error, "not found " <> nickname}
         end
     end
   end
 
   @doc "Fetch some posts when the user has just been federated with"
-  def fetch_initial_posts(user) do
-    pages = Pleroma.Config.get!([:fetch_initial_posts, :pages])
+  def fetch_initial_posts(user),
+    do: PleromaJobQueue.enqueue(:background, __MODULE__, [:fetch_initial_posts, user])
 
-    Enum.each(
-      # Insert all the posts in reverse order, so they're in the right order on the timeline
-      Enum.reverse(Utils.fetch_ordered_collection(user.info.source_data["outbox"], pages)),
-      &Pleroma.Web.Federator.incoming_ap_doc/1
-    )
-  end
-
-  def get_followers_query(%User{id: id, follower_address: follower_address}, nil) do
-    from(
-      u in User,
-      where: fragment("? <@ ?", ^[follower_address], u.following),
-      where: u.id != ^id
-    )
+  @spec get_followers_query(User.t(), pos_integer() | nil) :: Ecto.Query.t()
+  def get_followers_query(%User{} = user, nil) do
+    User.Query.build(%{followers: user, deactivated: false})
   end
 
   def get_followers_query(user, page) do
     from(u in get_followers_query(user, nil))
-    |> paginate(page, 20)
+    |> User.Query.paginate(page, 20)
   end
 
+  @spec get_followers_query(User.t()) :: Ecto.Query.t()
   def get_followers_query(user), do: get_followers_query(user, nil)
 
   def get_followers(user, page \\ nil) do
@@ -598,19 +585,17 @@ defmodule Pleroma.User do
     Repo.all(from(u in q, select: u.id))
   end
 
-  def get_friends_query(%User{id: id, following: following}, nil) do
-    from(
-      u in User,
-      where: u.follower_address in ^following,
-      where: u.id != ^id
-    )
+  @spec get_friends_query(User.t(), pos_integer() | nil) :: Ecto.Query.t()
+  def get_friends_query(%User{} = user, nil) do
+    User.Query.build(%{friends: user, deactivated: false})
   end
 
   def get_friends_query(user, page) do
     from(u in get_friends_query(user, nil))
-    |> paginate(page, 20)
+    |> User.Query.paginate(page, 20)
   end
 
+  @spec get_friends_query(User.t()) :: Ecto.Query.t()
   def get_friends_query(user), do: get_friends_query(user, nil)
 
   def get_friends(user, page \\ nil) do
@@ -625,33 +610,10 @@ defmodule Pleroma.User do
     Repo.all(from(u in q, select: u.id))
   end
 
-  def get_follow_requests_query(%User{} = user) do
-    from(
-      a in Activity,
-      where:
-        fragment(
-          "? ->> 'type' = 'Follow'",
-          a.data
-        ),
-      where:
-        fragment(
-          "? ->> 'state' = 'pending'",
-          a.data
-        ),
-      where:
-        fragment(
-          "coalesce((?)->'object'->>'id', (?)->>'object') = ?",
-          a.data,
-          a.data,
-          ^user.ap_id
-        )
-    )
-  end
-
+  @spec get_follow_requests(User.t()) :: {:ok, [User.t()]}
   def get_follow_requests(%User{} = user) do
     users =
-      user
-      |> User.get_follow_requests_query()
+      Activity.follow_requests_for_actor(user)
       |> join(:inner, [a], u in User, on: a.actor == u.ap_id)
       |> where([a, u], not fragment("? @> ?", u.following, ^[user.follower_address]))
       |> group_by([a, u], u.id)
@@ -715,18 +677,15 @@ defmodule Pleroma.User do
 
     info_cng = User.Info.set_note_count(user.info, note_count)
 
-    cng =
-      change(user)
-      |> put_embed(:info, info_cng)
-
-    update_and_set_cache(cng)
+    user
+    |> change()
+    |> put_embed(:info, info_cng)
+    |> update_and_set_cache()
   end
 
   def update_follower_count(%User{} = user) do
     follower_count_query =
-      User
-      |> where([u], ^user.follower_address in u.following)
-      |> where([u], u.id != ^user.id)
+      User.Query.build(%{followers: user, deactivated: false})
       |> select([u], %{count: count(u.id)})
 
     User
@@ -750,38 +709,31 @@ defmodule Pleroma.User do
     end
   end
 
-  def get_users_from_set_query(ap_ids, false) do
-    from(
-      u in User,
-      where: u.ap_id in ^ap_ids
-    )
-  end
-
-  def get_users_from_set_query(ap_ids, true) do
-    query = get_users_from_set_query(ap_ids, false)
-
-    from(
-      u in query,
-      where: u.local == true
-    )
+  def remove_duplicated_following(%User{following: following} = user) do
+    uniq_following = Enum.uniq(following)
+
+    if length(following) == length(uniq_following) do
+      {:ok, user}
+    else
+      user
+      |> update_changeset(%{following: uniq_following})
+      |> update_and_set_cache()
+    end
   end
 
+  @spec get_users_from_set([String.t()], boolean()) :: [User.t()]
   def get_users_from_set(ap_ids, local_only \\ true) do
-    get_users_from_set_query(ap_ids, local_only)
+    criteria = %{ap_id: ap_ids, deactivated: false}
+    criteria = if local_only, do: Map.put(criteria, :local, true), else: criteria
+
+    User.Query.build(criteria)
     |> Repo.all()
   end
 
+  @spec get_recipients_from_activity(Activity.t()) :: [User.t()]
   def get_recipients_from_activity(%Activity{recipients: to}) do
-    query =
-      from(
-        u in User,
-        where: u.ap_id in ^to,
-        or_where: fragment("? && ?", u.following, ^to)
-      )
-
-    query = from(u in query, where: u.local == true)
-
-    Repo.all(query)
+    User.Query.build(%{recipients_from_activity: to, local: true, deactivated: false})
+    |> Repo.all()
   end
 
   def search(query, resolve \\ false, for_user \\ nil) do
@@ -878,6 +830,7 @@ defmodule Pleroma.User do
           ^processed_query
         )
     )
+    |> restrict_deactivated()
   end
 
   defp trigram_search_subquery(term) do
@@ -896,23 +849,7 @@ defmodule Pleroma.User do
       },
       where: fragment("trim(? || ' ' || coalesce(?, '')) % ?", u.nickname, u.name, ^term)
     )
-  end
-
-  def blocks_import(%User{} = blocker, blocked_identifiers) when is_list(blocked_identifiers) do
-    Enum.map(
-      blocked_identifiers,
-      fn blocked_identifier ->
-        with %User{} = blocked <- get_or_fetch(blocked_identifier),
-             {:ok, blocker} <- block(blocker, blocked),
-             {:ok, _} <- ActivityPub.block(blocker, blocked) do
-          blocked
-        else
-          err ->
-            Logger.debug("blocks_import failed for #{blocked_identifier} with: #{inspect(err)}")
-            err
-        end
-      end
-    )
+    |> restrict_deactivated()
   end
 
   def mute(muter, %User{ap_id: ap_id}) do
@@ -1043,14 +980,23 @@ defmodule Pleroma.User do
     end
   end
 
-  def muted_users(user),
-    do: Repo.all(from(u in User, where: u.ap_id in ^user.info.mutes))
+  @spec muted_users(User.t()) :: [User.t()]
+  def muted_users(user) do
+    User.Query.build(%{ap_id: user.info.mutes, deactivated: false})
+    |> Repo.all()
+  end
 
-  def blocked_users(user),
-    do: Repo.all(from(u in User, where: u.ap_id in ^user.info.blocks))
+  @spec blocked_users(User.t()) :: [User.t()]
+  def blocked_users(user) do
+    User.Query.build(%{ap_id: user.info.blocks, deactivated: false})
+    |> Repo.all()
+  end
 
-  def subscribers(user),
-    do: Repo.all(from(u in User, where: u.ap_id in ^user.info.subscribers))
+  @spec subscribers(User.t()) :: [User.t()]
+  def subscribers(user) do
+    User.Query.build(%{ap_id: user.info.subscribers, deactivated: false})
+    |> Repo.all()
+  end
 
   def block_domain(user, domain) do
     info_cng =
@@ -1076,77 +1022,25 @@ defmodule Pleroma.User do
     update_and_set_cache(cng)
   end
 
-  def maybe_local_user_query(query, local) do
-    if local, do: local_user_query(query), else: query
-  end
-
-  def local_user_query(query \\ User) do
-    from(
-      u in query,
-      where: u.local == true,
-      where: not is_nil(u.nickname)
-    )
-  end
-
-  def maybe_external_user_query(query, external) do
-    if external, do: external_user_query(query), else: query
-  end
-
-  def external_user_query(query \\ User) do
-    from(
-      u in query,
-      where: u.local == false,
-      where: not is_nil(u.nickname)
-    )
-  end
-
-  def maybe_active_user_query(query, active) do
-    if active, do: active_user_query(query), else: query
-  end
-
-  def active_user_query(query \\ User) do
-    from(
-      u in query,
-      where: fragment("not (?->'deactivated' @> 'true')", u.info),
-      where: not is_nil(u.nickname)
-    )
-  end
-
-  def maybe_deactivated_user_query(query, deactivated) do
-    if deactivated, do: deactivated_user_query(query), else: query
-  end
-
-  def deactivated_user_query(query \\ User) do
-    from(
-      u in query,
-      where: fragment("(?->'deactivated' @> 'true')", u.info),
-      where: not is_nil(u.nickname)
-    )
-  end
-
-  def active_local_user_query do
-    from(
-      u in local_user_query(),
-      where: fragment("not (?->'deactivated' @> 'true')", u.info)
-    )
-  end
-
-  def moderator_user_query do
-    from(
-      u in User,
-      where: u.local == true,
-      where: fragment("?->'is_moderator' @> 'true'", u.info)
-    )
+  def deactivate_async(user, status \\ true) do
+    PleromaJobQueue.enqueue(:background, __MODULE__, [:deactivate_async, user, status])
   end
 
   def deactivate(%User{} = user, status \\ true) do
     info_cng = User.Info.set_activation_status(user.info, status)
 
-    cng =
-      change(user)
-      |> put_embed(:info, info_cng)
+    with {:ok, friends} <- User.get_friends(user),
+         {:ok, followers} <- User.get_followers(user),
+         {:ok, user} <-
+           user
+           |> change()
+           |> put_embed(:info, info_cng)
+           |> update_and_set_cache() do
+      Enum.each(followers, &invalidate_cache(&1))
+      Enum.each(friends, &update_follower_count(&1))
 
-    update_and_set_cache(cng)
+      {:ok, user}
+    end
   end
 
   def update_notification_settings(%User{} = user, settings \\ %{}) do
@@ -1157,7 +1051,12 @@ defmodule Pleroma.User do
     |> update_and_set_cache()
   end
 
-  def delete(%User{} = user) do
+  @spec delete(User.t()) :: :ok
+  def delete(%User{} = user),
+    do: PleromaJobQueue.enqueue(:background, __MODULE__, [:delete, user])
+
+  @spec perform(atom(), User.t()) :: {:ok, User.t()}
+  def perform(:delete, %User{} = user) do
     {:ok, user} = User.deactivate(user)
 
     # Remove all relationships
@@ -1172,23 +1071,92 @@ defmodule Pleroma.User do
     delete_user_activities(user)
   end
 
-  def delete_user_activities(%User{ap_id: ap_id} = user) do
-    Activity
-    |> where(actor: ^ap_id)
-    |> Activity.with_preloaded_object()
-    |> Repo.all()
-    |> Enum.each(fn
-      %{data: %{"type" => "Create"}} = activity ->
-        activity |> Object.normalize() |> ActivityPub.delete()
+  @spec perform(atom(), User.t()) :: {:ok, User.t()}
+  def perform(:fetch_initial_posts, %User{} = user) do
+    pages = Pleroma.Config.get!([:fetch_initial_posts, :pages])
 
-      # TODO: Do something with likes, follows, repeats.
-      _ ->
-        "Doing nothing"
-    end)
+    Enum.each(
+      # Insert all the posts in reverse order, so they're in the right order on the timeline
+      Enum.reverse(Utils.fetch_ordered_collection(user.info.source_data["outbox"], pages)),
+      &Pleroma.Web.Federator.incoming_ap_doc/1
+    )
 
     {:ok, user}
   end
 
+  def perform(:deactivate_async, user, status), do: deactivate(user, status)
+
+  @spec perform(atom(), User.t(), list()) :: list() | {:error, any()}
+  def perform(:blocks_import, %User{} = blocker, blocked_identifiers)
+      when is_list(blocked_identifiers) do
+    Enum.map(
+      blocked_identifiers,
+      fn blocked_identifier ->
+        with {:ok, %User{} = blocked} <- get_or_fetch(blocked_identifier),
+             {:ok, blocker} <- block(blocker, blocked),
+             {:ok, _} <- ActivityPub.block(blocker, blocked) do
+          blocked
+        else
+          err ->
+            Logger.debug("blocks_import failed for #{blocked_identifier} with: #{inspect(err)}")
+            err
+        end
+      end
+    )
+  end
+
+  @spec perform(atom(), User.t(), list()) :: list() | {:error, any()}
+  def perform(:follow_import, %User{} = follower, followed_identifiers)
+      when is_list(followed_identifiers) do
+    Enum.map(
+      followed_identifiers,
+      fn followed_identifier ->
+        with {:ok, %User{} = followed} <- get_or_fetch(followed_identifier),
+             {:ok, follower} <- maybe_direct_follow(follower, followed),
+             {:ok, _} <- ActivityPub.follow(follower, followed) do
+          followed
+        else
+          err ->
+            Logger.debug("follow_import failed for #{followed_identifier} with: #{inspect(err)}")
+            err
+        end
+      end
+    )
+  end
+
+  def blocks_import(%User{} = blocker, blocked_identifiers) when is_list(blocked_identifiers),
+    do:
+      PleromaJobQueue.enqueue(:background, __MODULE__, [
+        :blocks_import,
+        blocker,
+        blocked_identifiers
+      ])
+
+  def follow_import(%User{} = follower, followed_identifiers) when is_list(followed_identifiers),
+    do:
+      PleromaJobQueue.enqueue(:background, __MODULE__, [
+        :follow_import,
+        follower,
+        followed_identifiers
+      ])
+
+  def delete_user_activities(%User{ap_id: ap_id} = user) do
+    stream =
+      ap_id
+      |> Activity.query_by_actor()
+      |> Repo.stream()
+
+    Repo.transaction(fn -> Enum.each(stream, &delete_activity(&1)) end, timeout: :infinity)
+
+    {:ok, user}
+  end
+
+  defp delete_activity(%{data: %{"type" => "Create"}} = activity) do
+    Object.normalize(activity) |> ActivityPub.delete()
+  end
+
+  defp delete_activity(_activity), do: "Doing nothing"
+
   def html_filter_policy(%User{info: %{no_rich_text: true}}) do
     Pleroma.HTML.Scrubber.TwitterText
   end
@@ -1202,11 +1170,11 @@ defmodule Pleroma.User do
 
     case ap_try do
       {:ok, user} ->
-        user
+        {:ok, user}
 
       _ ->
         case OStatus.make_user(ap_id) do
-          {:ok, user} -> user
+          {:ok, user} -> {:ok, user}
           _ -> {:error, "Could not fetch by AP id"}
         end
     end
@@ -1216,20 +1184,20 @@ defmodule Pleroma.User do
     user = get_cached_by_ap_id(ap_id)
 
     if !is_nil(user) and !User.needs_update?(user) do
-      user
+      {:ok, user}
     else
       # Whether to fetch initial posts for the user (if it's a new user & the fetching is enabled)
       should_fetch_initial = is_nil(user) and Pleroma.Config.get([:fetch_initial_posts, :enabled])
 
-      user = fetch_by_ap_id(ap_id)
+      resp = fetch_by_ap_id(ap_id)
 
       if should_fetch_initial do
-        with %User{} = user do
-          {:ok, _} = Task.start(__MODULE__, :fetch_initial_posts, [user])
+        with {:ok, %User{} = user} <- resp do
+          fetch_initial_posts(user)
         end
       end
 
-      user
+      resp
     end
   end
 
@@ -1271,7 +1239,7 @@ defmodule Pleroma.User do
   end
 
   def get_public_key_for_ap_id(ap_id) do
-    with %User{} = user <- get_or_fetch_by_ap_id(ap_id),
+    with {:ok, %User{} = user} <- get_or_fetch_by_ap_id(ap_id),
          {:ok, public_key} <- public_key_from_info(user.info) do
       {:ok, public_key}
     else
@@ -1295,7 +1263,7 @@ defmodule Pleroma.User do
   def ap_enabled?(_), do: false
 
   @doc "Gets or fetch a user by uri or nickname."
-  @spec get_or_fetch(String.t()) :: User.t()
+  @spec get_or_fetch(String.t()) :: {:ok, User.t()} | {:error, String.t()}
   def get_or_fetch("http" <> _host = uri), do: get_or_fetch_by_ap_id(uri)
   def get_or_fetch(nickname), do: get_or_fetch_by_nickname(nickname)
 
@@ -1323,18 +1291,15 @@ defmodule Pleroma.User do
     end
   end
 
-  def parse_bio(bio, user \\ %User{info: %{source_data: %{}}})
-  def parse_bio(nil, _user), do: ""
-  def parse_bio(bio, _user) when bio == "", do: bio
+  def parse_bio(bio) when is_binary(bio) and bio != "" do
+    bio
+    |> CommonUtils.format_input("text/plain", mentions_format: :full)
+    |> elem(0)
+  end
 
-  def parse_bio(bio, user) do
-    emoji =
-      (user.info.source_data["tag"] || [])
-      |> Enum.filter(fn %{"type" => t} -> t == "Emoji" end)
-      |> Enum.map(fn %{"icon" => %{"url" => url}, "name" => name} ->
-        {String.trim(name, ":"), url}
-      end)
+  def parse_bio(_), do: ""
 
+  def parse_bio(bio, user) when is_binary(bio) and bio != "" do
     # TODO: get profile URLs other than user.ap_id
     profile_urls = [user.ap_id]
 
@@ -1344,9 +1309,10 @@ defmodule Pleroma.User do
       rel: &RelMe.maybe_put_rel_me(&1, profile_urls)
     )
     |> elem(0)
-    |> Formatter.emojify(emoji)
   end
 
+  def parse_bio(_, _), do: ""
+
   def tag(user_identifiers, tags) when is_list(user_identifiers) do
     Repo.transaction(fn ->
       for user_identifier <- user_identifiers, do: tag(user_identifier, tags)
@@ -1414,23 +1380,46 @@ defmodule Pleroma.User do
     }
   end
 
+  @spec all_superusers() :: [User.t()]
   def all_superusers do
-    from(
-      u in User,
-      where: u.local == true,
-      where: fragment("?->'is_admin' @> 'true' OR ?->'is_moderator' @> 'true'", u.info, u.info)
-    )
+    User.Query.build(%{super_users: true, local: true, deactivated: false})
     |> Repo.all()
   end
 
-  defp paginate(query, page, page_size) do
-    from(u in query,
-      limit: ^page_size,
-      offset: ^((page - 1) * page_size)
-    )
-  end
-
   def showing_reblogs?(%User{} = user, %User{} = target) do
     target.ap_id not in user.info.muted_reblogs
   end
+
+  @spec toggle_confirmation(User.t()) :: {:ok, User.t()} | {:error, Changeset.t()}
+  def toggle_confirmation(%User{} = user) do
+    need_confirmation? = !user.info.confirmation_pending
+
+    info_changeset =
+      User.Info.confirmation_changeset(user.info, need_confirmation: need_confirmation?)
+
+    user
+    |> change()
+    |> put_embed(:info, info_changeset)
+    |> update_and_set_cache()
+  end
+
+  def get_mascot(%{info: %{mascot: %{} = mascot}}) when not is_nil(mascot) do
+    mascot
+  end
+
+  def get_mascot(%{info: %{mascot: mascot}}) when is_nil(mascot) do
+    # use instance-default
+    config = Pleroma.Config.get([:assets, :mascots])
+    default_mascot = Pleroma.Config.get([:assets, :default_mascot])
+    mascot = Keyword.get(config, default_mascot)
+
+    %{
+      "id" => "default-mascot",
+      "url" => mascot[:url],
+      "preview_url" => mascot[:url],
+      "pleroma" => %{
+        "mime_type" => mascot[:mime_type]
+      }
+    }
+  end
 end

lib/pleroma/user/info.ex

@@ -8,6 +8,8 @@ defmodule Pleroma.User.Info do
 
   alias Pleroma.User.Info
 
+  @type t :: %__MODULE__{}
+
   embedded_schema do
     field(:banner, :map, default: %{})
     field(:background, :map, default: %{})
@@ -41,6 +43,8 @@ defmodule Pleroma.User.Info do
     field(:hide_favorites, :boolean, default: true)
     field(:pinned_activities, {:array, :string}, default: [])
     field(:flavour, :string, default: nil)
+    field(:mascot, :map, default: nil)
+    field(:emoji, {:array, :map}, default: [])
 
     field(:notification_settings, :map,
       default: %{"remote" => true, "local" => true, "followers" => true, "follows" => true}
@@ -209,21 +213,23 @@ defmodule Pleroma.User.Info do
     ])
   end
 
-  def confirmation_changeset(info, :confirmed) do
-    confirmation_changeset(info, %{
-      confirmation_pending: false,
-      confirmation_token: nil
-    })
-  end
+  @spec confirmation_changeset(Info.t(), keyword()) :: Changeset.t()
+  def confirmation_changeset(info, opts) do
+    need_confirmation? = Keyword.get(opts, :need_confirmation)
 
-  def confirmation_changeset(info, :unconfirmed) do
-    confirmation_changeset(info, %{
-      confirmation_pending: true,
-      confirmation_token: :crypto.strong_rand_bytes(32) |> Base.url_encode64()
-    })
-  end
+    params =
+      if need_confirmation? do
+        %{
+          confirmation_pending: true,
+          confirmation_token: :crypto.strong_rand_bytes(32) |> Base.url_encode64()
+        }
+      else
+        %{
+          confirmation_pending: false,
+          confirmation_token: nil
+        }
+      end
 
-  def confirmation_changeset(info, params) do
     cast(info, params, [:confirmation_pending, :confirmation_token])
   end
 
@@ -243,6 +249,14 @@ defmodule Pleroma.User.Info do
     |> validate_required([:flavour])
   end
 
+  def mascot_update(info, url) do
+    params = %{mascot: url}
+
+    info
+    |> cast(params, [:mascot])
+    |> validate_required([:mascot])
+  end
+
   def set_source_data(info, source_data) do
     params = %{source_data: source_data}
 

lib/pleroma/user/query.ex

@@ -0,0 +1,154 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.User.Query do
+  @moduledoc """
+  User query builder module. Builds query from new query or another user query.
+
+    ## Example:
+        query = Pleroma.User.Query(%{nickname: "nickname"})
+        another_query = Pleroma.User.Query.build(query, %{email: "email@example.com"})
+        Pleroma.Repo.all(query)
+        Pleroma.Repo.all(another_query)
+
+  Adding new rules:
+    - *ilike criteria*
+      - add field to @ilike_criteria list
+      - pass non empty string
+      - e.g. Pleroma.User.Query.build(%{nickname: "nickname"})
+    - *equal criteria*
+      - add field to @equal_criteria list
+      - pass non empty string
+      - e.g. Pleroma.User.Query.build(%{email: "email@example.com"})
+    - *contains criteria*
+      - add field to @containns_criteria list
+      - pass values list
+      - e.g. Pleroma.User.Query.build(%{ap_id: ["http://ap_id1", "http://ap_id2"]})
+  """
+  import Ecto.Query
+  import Pleroma.Web.AdminAPI.Search, only: [not_empty_string: 1]
+  alias Pleroma.User
+
+  @type criteria ::
+          %{
+            query: String.t(),
+            tags: [String.t()],
+            name: String.t(),
+            email: String.t(),
+            local: boolean(),
+            external: boolean(),
+            active: boolean(),
+            deactivated: boolean(),
+            is_admin: boolean(),
+            is_moderator: boolean(),
+            super_users: boolean(),
+            followers: User.t(),
+            friends: User.t(),
+            recipients_from_activity: [String.t()],
+            nickname: [String.t()],
+            ap_id: [String.t()]
+          }
+          | %{}
+
+  @ilike_criteria [:nickname, :name, :query]
+  @equal_criteria [:email]
+  @role_criteria [:is_admin, :is_moderator]
+  @contains_criteria [:ap_id, :nickname]
+
+  @spec build(criteria()) :: Query.t()
+  def build(query \\ base_query(), criteria) do
+    prepare_query(query, criteria)
+  end
+
+  @spec paginate(Ecto.Query.t(), pos_integer(), pos_integer()) :: Ecto.Query.t()
+  def paginate(query, page, page_size) do
+    from(u in query,
+      limit: ^page_size,
+      offset: ^((page - 1) * page_size)
+    )
+  end
+
+  defp base_query do
+    from(u in User)
+  end
+
+  defp prepare_query(query, criteria) do
+    Enum.reduce(criteria, query, &compose_query/2)
+  end
+
+  defp compose_query({key, value}, query)
+       when key in @ilike_criteria and not_empty_string(value) do
+    # hack for :query key
+    key = if key == :query, do: :nickname, else: key
+    where(query, [u], ilike(field(u, ^key), ^"%#{value}%"))
+  end
+
+  defp compose_query({key, value}, query)
+       when key in @equal_criteria and not_empty_string(value) do
+    where(query, [u], ^[{key, value}])
+  end
+
+  defp compose_query({key, values}, query) when key in @contains_criteria and is_list(values) do
+    where(query, [u], field(u, ^key) in ^values)
+  end
+
+  defp compose_query({:tags, tags}, query) when is_list(tags) and length(tags) > 0 do
+    Enum.reduce(tags, query, &prepare_tag_criteria/2)
+  end
+
+  defp compose_query({key, _}, query) when key in @role_criteria do
+    where(query, [u], fragment("(?->? @> 'true')", u.info, ^to_string(key)))
+  end
+
+  defp compose_query({:super_users, _}, query) do
+    where(
+      query,
+      [u],
+      fragment("?->'is_admin' @> 'true' OR ?->'is_moderator' @> 'true'", u.info, u.info)
+    )
+  end
+
+  defp compose_query({:local, _}, query), do: location_query(query, true)
+
+  defp compose_query({:external, _}, query), do: location_query(query, false)
+
+  defp compose_query({:active, _}, query) do
+    where(query, [u], fragment("not (?->'deactivated' @> 'true')", u.info))
+    |> where([u], not is_nil(u.nickname))
+  end
+
+  defp compose_query({:deactivated, false}, query) do
+    User.restrict_deactivated(query)
+  end
+
+  defp compose_query({:deactivated, true}, query) do
+    where(query, [u], fragment("?->'deactivated' @> 'true'", u.info))
+    |> where([u], not is_nil(u.nickname))
+  end
+
+  defp compose_query({:followers, %User{id: id, follower_address: follower_address}}, query) do
+    where(query, [u], fragment("? <@ ?", ^[follower_address], u.following))
+    |> where([u], u.id != ^id)
+  end
+
+  defp compose_query({:friends, %User{id: id, following: following}}, query) do
+    where(query, [u], u.follower_address in ^following)
+    |> where([u], u.id != ^id)
+  end
+
+  defp compose_query({:recipients_from_activity, to}, query) do
+    where(query, [u], u.ap_id in ^to or fragment("? && ?", u.following, ^to))
+  end
+
+  defp compose_query(_unsupported_param, query), do: query
+
+  defp prepare_tag_criteria(tag, query) do
+    or_where(query, [u], fragment("? = any(?)", ^tag, u.tags))
+  end
+
+  defp location_query(query, local) do
+    where(query, [u], u.local == ^local)
+    |> where([u], not is_nil(u.nickname))
+  end
+end

lib/pleroma/user_invite_token.ex

@@ -24,7 +24,7 @@ defmodule Pleroma.UserInviteToken do
     timestamps()
   end
 
-  @spec create_invite(map()) :: UserInviteToken.t()
+  @spec create_invite(map()) :: {:ok, UserInviteToken.t()}
   def create_invite(params \\ %{}) do
     %UserInviteToken{}
     |> cast(params, [:max_use, :expires_at])

lib/pleroma/web/activity_pub/activity_pub.ex

@@ -4,7 +4,7 @@
 
 defmodule Pleroma.Web.ActivityPub.ActivityPub do
   alias Pleroma.Activity
-  alias Pleroma.Instances
+  alias Pleroma.Conversation
   alias Pleroma.Notification
   alias Pleroma.Object
   alias Pleroma.Object.Fetcher
@@ -14,7 +14,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
   alias Pleroma.User
   alias Pleroma.Web.ActivityPub.MRF
   alias Pleroma.Web.ActivityPub.Transmogrifier
-  alias Pleroma.Web.Federator
   alias Pleroma.Web.WebFinger
 
   import Ecto.Query
@@ -23,8 +22,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
 
   require Logger
 
-  @httpoison Application.get_env(:pleroma, :httpoison)
-
   # For Announce activities, we filter the recipients based on following status for any actors
   # that match actual users.  See issue #164 for more information about why this is necessary.
   defp get_recipients(%{"type" => "Announce"} = data) do
@@ -136,12 +133,17 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
           activity
         end
 
-      Task.start(fn ->
-        Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity)
-      end)
+      PleromaJobQueue.enqueue(:background, Pleroma.Web.RichMedia.Helpers, [:fetch, activity])
 
       Notification.create_notifications(activity)
+
+      participations =
+        activity
+        |> Conversation.create_or_bump_for()
+        |> get_participations()
+
       stream_out(activity)
+      stream_out_participations(participations)
       {:ok, activity}
     else
       %Activity{} = activity ->
@@ -164,11 +166,23 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     end
   end
 
+  defp get_participations({:ok, %{participations: participations}}), do: participations
+  defp get_participations(_), do: []
+
+  def stream_out_participations(participations) do
+    participations =
+      participations
+      |> Repo.preload(:user)
+
+    Enum.each(participations, fn participation ->
+      Pleroma.Web.Streamer.stream("participation", participation)
+    end)
+  end
+
   def stream_out(activity) do
     public = "https://www.w3.org/ns/activitystreams#Public"
 
     if activity.data["type"] in ["Create", "Announce", "Delete"] do
-      object = Object.normalize(activity)
       Pleroma.Web.Streamer.stream("user", activity)
       Pleroma.Web.Streamer.stream("list", activity)
 
@@ -180,6 +194,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
         end
 
         if activity.data["type"] in ["Create"] do
+          object = Object.normalize(activity)
+
           object.data
           |> Map.get("tag", [])
           |> Enum.filter(fn tag -> is_bitstring(tag) end)
@@ -194,6 +210,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
           end
         end
       else
+        # TODO: Write test, replace with visibility test
         if !Enum.member?(activity.data["cc"] || [], public) &&
              !Enum.member?(
                activity.data["to"],
@@ -456,35 +473,44 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     end
   end
 
-  def fetch_activities_for_context(context, opts \\ %{}) do
+  defp fetch_activities_for_context_query(context, opts) do
     public = ["https://www.w3.org/ns/activitystreams#Public"]
 
     recipients =
       if opts["user"], do: [opts["user"].ap_id | opts["user"].following] ++ public, else: public
 
-    query = from(activity in Activity)
-
-    query =
-      query
-      |> restrict_blocked(opts)
-      |> restrict_recipients(recipients, opts["user"])
-
-    query =
-      from(
-        activity in query,
-        where:
-          fragment(
-            "?->>'type' = ? and ?->>'context' = ?",
-            activity.data,
-            "Create",
-            activity.data,
-            ^context
-          ),
-        order_by: [desc: :id]
+    from(activity in Activity)
+    |> restrict_blocked(opts)
+    |> restrict_recipients(recipients, opts["user"])
+    |> where(
+      [activity],
+      fragment(
+        "?->>'type' = ? and ?->>'context' = ?",
+        activity.data,
+        "Create",
+        activity.data,
+        ^context
       )
-      |> Activity.with_preloaded_object()
+    )
+    |> order_by([activity], desc: activity.id)
+  end
 
-    Repo.all(query)
+  @spec fetch_activities_for_context(String.t(), keyword() | map()) :: [Activity.t()]
+  def fetch_activities_for_context(context, opts \\ %{}) do
+    context
+    |> fetch_activities_for_context_query(opts)
+    |> Activity.with_preloaded_object()
+    |> Repo.all()
+  end
+
+  @spec fetch_latest_activity_id_for_context(String.t(), keyword() | map()) ::
+          Pleroma.FlakeId.t() | nil
+  def fetch_latest_activity_id_for_context(context, opts \\ %{}) do
+    context
+    |> fetch_activities_for_context_query(opts)
+    |> limit(1)
+    |> select([a], a.id)
+    |> Repo.one()
   end
 
   def fetch_public_activities(opts \\ %{}) do
@@ -514,8 +540,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
             )
         )
 
-      Ecto.Adapters.SQL.to_sql(:all, Repo, query)
-
       query
     else
       Logger.error("Could not restrict visibility to #{visibility}")
@@ -531,8 +555,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
           fragment("activity_visibility(?, ?, ?) = ?", a.actor, a.recipients, a.data, ^visibility)
       )
 
-    Ecto.Adapters.SQL.to_sql(:all, Repo, query)
-
     query
   end
 
@@ -543,6 +565,18 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
 
   defp restrict_visibility(query, _visibility), do: query
 
+  defp restrict_thread_visibility(query, %{"user" => %User{ap_id: ap_id}}) do
+    query =
+      from(
+        a in query,
+        where: fragment("thread_visibility(?, (?)->>'id') = true", ^ap_id, a.data)
+      )
+
+    query
+  end
+
+  defp restrict_thread_visibility(query, _), do: query
+
   def fetch_user_activities(user, reading_user, params \\ %{}) do
     params =
       params
@@ -669,6 +703,12 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
 
   defp restrict_type(query, _), do: query
 
+  defp restrict_state(query, %{"state" => state}) do
+    from(activity in query, where: fragment("?->>'state' = ?", activity.data, ^state))
+  end
+
+  defp restrict_state(query, _), do: query
+
   defp restrict_favorited_by(query, %{"favorited_by" => ap_id}) do
     from(
       activity in query,
@@ -724,8 +764,11 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     blocks = info.blocks || []
     domain_blocks = info.domain_blocks || []
 
+    query =
+      if has_named_binding?(query, :object), do: query, else: Activity.with_joined_object(query)
+
     from(
-      activity in query,
+      [activity, object: o] in query,
       where: fragment("not (? = ANY(?))", activity.actor, ^blocks),
       where: fragment("not (? && ?)", activity.recipients, ^blocks),
       where:
@@ -735,7 +778,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
           activity.data,
           ^blocks
         ),
-      where: fragment("not (split_part(?, '/', 3) = ANY(?))", activity.actor, ^domain_blocks)
+      where: fragment("not (split_part(?, '/', 3) = ANY(?))", activity.actor, ^domain_blocks),
+      where: fragment("not (split_part(?->>'actor', '/', 3) = ANY(?))", o.data, ^domain_blocks)
     )
   end
 
@@ -783,11 +827,40 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     |> Activity.with_preloaded_object()
   end
 
+  defp maybe_preload_bookmarks(query, %{"skip_preload" => true}), do: query
+
+  defp maybe_preload_bookmarks(query, opts) do
+    query
+    |> Activity.with_preloaded_bookmark(opts["user"])
+  end
+
+  defp maybe_set_thread_muted_field(query, %{"skip_preload" => true}), do: query
+
+  defp maybe_set_thread_muted_field(query, opts) do
+    query
+    |> Activity.with_set_thread_muted_field(opts["user"])
+  end
+
+  defp maybe_order(query, %{order: :desc}) do
+    query
+    |> order_by(desc: :id)
+  end
+
+  defp maybe_order(query, %{order: :asc}) do
+    query
+    |> order_by(asc: :id)
+  end
+
+  defp maybe_order(query, _), do: query
+
   def fetch_activities_query(recipients, opts \\ %{}) do
     base_query = from(activity in Activity)
 
     base_query
     |> maybe_preload_objects(opts)
+    |> maybe_preload_bookmarks(opts)
+    |> maybe_set_thread_muted_field(opts)
+    |> maybe_order(opts)
     |> restrict_recipients(recipients, opts["user"])
     |> restrict_tag(opts)
     |> restrict_tag_reject(opts)
@@ -796,15 +869,18 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     |> restrict_local(opts)
     |> restrict_actor(opts)
     |> restrict_type(opts)
+    |> restrict_state(opts)
     |> restrict_favorited_by(opts)
     |> restrict_blocked(opts)
     |> restrict_muted(opts)
     |> restrict_media(opts)
     |> restrict_visibility(opts)
+    |> restrict_thread_visibility(opts)
     |> restrict_replies(opts)
     |> restrict_reblogs(opts)
     |> restrict_pinned(opts)
     |> restrict_muted_reblogs(opts)
+    |> Activity.restrict_deactivated_users()
   end
 
   def fetch_activities(recipients, opts \\ %{}) do
@@ -908,89 +984,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     end
   end
 
-  def should_federate?(inbox, public) do
-    if public do
-      true
-    else
-      inbox_info = URI.parse(inbox)
-      !Enum.member?(Pleroma.Config.get([:instance, :quarantined_instances], []), inbox_info.host)
-    end
-  end
-
-  def publish(actor, activity) do
-    remote_followers =
-      if actor.follower_address in activity.recipients do
-        {:ok, followers} = User.get_followers(actor)
-        followers |> Enum.filter(&(!&1.local))
-      else
-        []
-      end
-
-    public = is_public?(activity)
-
-    {:ok, data} = Transmogrifier.prepare_outgoing(activity.data)
-    json = Jason.encode!(data)
-
-    (Pleroma.Web.Salmon.remote_users(activity) ++ remote_followers)
-    |> Enum.filter(fn user -> User.ap_enabled?(user) end)
-    |> Enum.map(fn %{info: %{source_data: data}} ->
-      (is_map(data["endpoints"]) && Map.get(data["endpoints"], "sharedInbox")) || data["inbox"]
-    end)
-    |> Enum.uniq()
-    |> Enum.filter(fn inbox -> should_federate?(inbox, public) end)
-    |> Instances.filter_reachable()
-    |> Enum.each(fn {inbox, unreachable_since} ->
-      Federator.publish_single_ap(%{
-        inbox: inbox,
-        json: json,
-        actor: actor,
-        id: activity.data["id"],
-        unreachable_since: unreachable_since
-      })
-    end)
-  end
-
-  def publish_one(%{inbox: inbox, json: json, actor: actor, id: id} = params) do
-    Logger.info("Federating #{id} to #{inbox}")
-    host = URI.parse(inbox).host
-
-    digest = "SHA-256=" <> (:crypto.hash(:sha256, json) |> Base.encode64())
-
-    date =
-      NaiveDateTime.utc_now()
-      |> Timex.format!("{WDshort}, {0D} {Mshort} {YYYY} {h24}:{m}:{s} GMT")
-
-    signature =
-      Pleroma.Web.HTTPSignatures.sign(actor, %{
-        host: host,
-        "content-length": byte_size(json),
-        digest: digest,
-        date: date
-      })
-
-    with {:ok, %{status: code}} when code in 200..299 <-
-           result =
-             @httpoison.post(
-               inbox,
-               json,
-               [
-                 {"Content-Type", "application/activity+json"},
-                 {"Date", date},
-                 {"signature", signature},
-                 {"digest", digest}
-               ]
-             ) do
-      if !Map.has_key?(params, :unreachable_since) || params[:unreachable_since],
-        do: Instances.set_reachable(inbox)
-
-      result
-    else
-      {_post_result, response} ->
-        unless params[:unreachable_since], do: Instances.set_unreachable(inbox)
-        {:error, response}
-    end
-  end
-
   # filter out broken threads
   def contain_broken_threads(%Activity{} = activity, %User{} = user) do
     entire_thread_visible_for_user?(activity, user)
@@ -1001,11 +994,10 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     contain_broken_threads(activity, user)
   end
 
-  # do post-processing on a timeline
-  def contain_timeline(timeline, user) do
-    timeline
-    |> Enum.filter(fn activity ->
-      contain_activity(activity, user)
-    end)
+  def fetch_direct_messages_query do
+    Activity
+    |> restrict_type(%{"type" => "Create"})
+    |> restrict_visibility(%{visibility: "direct"})
+    |> order_by([activity], asc: activity.id)
   end
 end

lib/pleroma/web/activity_pub/activity_pub_controller.ex

@@ -155,7 +155,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
 
   def inbox(%{assigns: %{valid_signature: true}} = conn, %{"nickname" => nickname} = params) do
     with %User{} = recipient <- User.get_cached_by_nickname(nickname),
-         %User{} = actor <- User.get_or_fetch_by_ap_id(params["actor"]),
+         {:ok, %User{} = actor} <- User.get_or_fetch_by_ap_id(params["actor"]),
          true <- Utils.recipient_in_message(recipient, actor, params),
          params <- Utils.maybe_splice_recipient(recipient.ap_id, params) do
       Federator.incoming_ap_doc(params)

lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex

@@ -5,6 +5,8 @@
 defmodule Pleroma.Web.ActivityPub.MRF.AntiFollowbotPolicy do
   alias Pleroma.User
 
+  @moduledoc "Prevent followbots from following with a bit of heuristic"
+
   @behaviour Pleroma.Web.ActivityPub.MRF
 
   # XXX: this should become User.normalize_by_ap_id() or similar, really.

lib/pleroma/web/activity_pub/mrf/drop_policy.ex

@@ -4,6 +4,7 @@
 
 defmodule Pleroma.Web.ActivityPub.MRF.DropPolicy do
   require Logger
+  @moduledoc "Drop and log everything received"
   @behaviour Pleroma.Web.ActivityPub.MRF
 
   @impl true

lib/pleroma/web/activity_pub/mrf/ensure_re_prepended.ex

@@ -5,6 +5,7 @@
 defmodule Pleroma.Web.ActivityPub.MRF.EnsureRePrepended do
   alias Pleroma.Object
 
+  @moduledoc "Ensure a re: is prepended on replies to a post with a Subject"
   @behaviour Pleroma.Web.ActivityPub.MRF
 
   @reply_prefix Regex.compile!("^re:[[:space:]]*", [:caseless])

lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex

@@ -4,6 +4,8 @@
 
 defmodule Pleroma.Web.ActivityPub.MRF.HellthreadPolicy do
   alias Pleroma.User
+  @moduledoc "Block messages with too much mentions (configurable)"
+
   @behaviour Pleroma.Web.ActivityPub.MRF
 
   defp delist_message(message, threshold) when threshold > 0 do

lib/pleroma/web/activity_pub/mrf/keyword_policy.ex

@@ -3,6 +3,8 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.ActivityPub.MRF.KeywordPolicy do
+  @moduledoc "Reject or Word-Replace messages with a keyword or regex"
+
   @behaviour Pleroma.Web.ActivityPub.MRF
   defp string_matches?(string, _) when not is_binary(string) do
     false

lib/pleroma/web/activity_pub/mrf/no_placeholder_text_policy.ex

@@ -3,6 +3,7 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.ActivityPub.MRF.NoPlaceholderTextPolicy do
+  @moduledoc "Ensure no content placeholder is present (such as the dot from mastodon)"
   @behaviour Pleroma.Web.ActivityPub.MRF
 
   @impl true

lib/pleroma/web/activity_pub/mrf/noop_policy.ex

@@ -3,6 +3,7 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.ActivityPub.MRF.NoOpPolicy do
+  @moduledoc "Does nothing (lets the messages go through unmodified)"
   @behaviour Pleroma.Web.ActivityPub.MRF
 
   @impl true

lib/pleroma/web/activity_pub/mrf/normalize_markup.ex

@@ -3,6 +3,7 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.ActivityPub.MRF.NormalizeMarkup do
+  @moduledoc "Scrub configured hypertext markup"
   alias Pleroma.HTML
 
   @behaviour Pleroma.Web.ActivityPub.MRF

lib/pleroma/web/activity_pub/mrf/reject_non_public.ex

@@ -4,6 +4,7 @@
 
 defmodule Pleroma.Web.ActivityPub.MRF.RejectNonPublic do
   alias Pleroma.User
+  @moduledoc "Rejects non-public (followers-only, direct) activities"
   @behaviour Pleroma.Web.ActivityPub.MRF
 
   @impl true

lib/pleroma/web/activity_pub/mrf/simple_policy.ex

@@ -4,6 +4,7 @@
 
 defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do
   alias Pleroma.User
+  @moduledoc "Filter activities depending on their origin instance"
   @behaviour Pleroma.Web.ActivityPub.MRF
 
   defp check_accept(%{host: actor_host} = _actor_info, object) do
@@ -47,14 +48,13 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do
          %{host: actor_host} = _actor_info,
          %{
            "type" => "Create",
-           "object" => %{"attachment" => child_attachment} = child_object
+           "object" => child_object
          } = object
-       )
-       when length(child_attachment) > 0 do
+       ) do
     object =
       if Enum.member?(Pleroma.Config.get([:mrf_simple, :media_nsfw]), actor_host) do
         tags = (child_object["tag"] || []) ++ ["nsfw"]
-        child_object = Map.put(child_object, "tags", tags)
+        child_object = Map.put(child_object, "tag", tags)
         child_object = Map.put(child_object, "sensitive", true)
         Map.put(object, "object", child_object)
       else
@@ -94,6 +94,16 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do
     {:ok, object}
   end
 
+  defp check_report_removal(%{host: actor_host} = _actor_info, %{"type" => "Flag"} = object) do
+    if actor_host in Pleroma.Config.get([:mrf_simple, :report_removal]) do
+      {:reject, nil}
+    else
+      {:ok, object}
+    end
+  end
+
+  defp check_report_removal(_actor_info, object), do: {:ok, object}
+
   @impl true
   def filter(object) do
     actor_info = URI.parse(object["actor"])
@@ -102,7 +112,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do
          {:ok, object} <- check_reject(actor_info, object),
          {:ok, object} <- check_media_removal(actor_info, object),
          {:ok, object} <- check_media_nsfw(actor_info, object),
-         {:ok, object} <- check_ftl_removal(actor_info, object) do
+         {:ok, object} <- check_ftl_removal(actor_info, object),
+         {:ok, object} <- check_report_removal(actor_info, object) do
       {:ok, object}
     else
       _e -> {:reject, nil}

lib/pleroma/web/activity_pub/mrf/tag_policy.ex

@@ -5,6 +5,19 @@
 defmodule Pleroma.Web.ActivityPub.MRF.TagPolicy do
   alias Pleroma.User
   @behaviour Pleroma.Web.ActivityPub.MRF
+  @moduledoc """
+     Apply policies based on user tags
+
+     This policy applies policies on a user activities depending on their tags
+     on your instance.
+
+     - `mrf_tag:media-force-nsfw`: Mark as sensitive on presence of attachments
+     - `mrf_tag:media-strip`: Remove attachments
+     - `mrf_tag:force-unlisted`: Mark as unlisted (removes from the federated timeline)
+     - `mrf_tag:sandbox`: Remove from public (local and federated) timelines
+     - `mrf_tag:disable-remote-subscription`: Reject non-local follow requests
+     - `mrf_tag:disable-any-subscription`: Reject any follow requests
+  """
 
   defp get_tags(%User{tags: tags}) when is_list(tags), do: tags
   defp get_tags(_), do: []
@@ -18,7 +31,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.TagPolicy do
 
     object =
       object
-      |> Map.put("tags", tags)
+      |> Map.put("tag", tags)
       |> Map.put("sensitive", true)
 
     message = Map.put(message, "object", object)

lib/pleroma/web/activity_pub/mrf/user_allowlist.ex

@@ -5,6 +5,7 @@
 defmodule Pleroma.Web.ActivityPub.MRF.UserAllowListPolicy do
   alias Pleroma.Config
 
+  @moduledoc "Accept-list of users from specified instances"
   @behaviour Pleroma.Web.ActivityPub.MRF
 
   defp filter_by_list(object, []), do: {:ok, object}

lib/pleroma/web/activity_pub/publisher.ex

@@ -0,0 +1,152 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ActivityPub.Publisher do
+  alias Pleroma.Activity
+  alias Pleroma.Config
+  alias Pleroma.Instances
+  alias Pleroma.User
+  alias Pleroma.Web.ActivityPub.Relay
+  alias Pleroma.Web.ActivityPub.Transmogrifier
+
+  import Pleroma.Web.ActivityPub.Visibility
+
+  @behaviour Pleroma.Web.Federator.Publisher
+
+  require Logger
+
+  @httpoison Application.get_env(:pleroma, :httpoison)
+
+  @moduledoc """
+  ActivityPub outgoing federation module.
+  """
+
+  @doc """
+  Determine if an activity can be represented by running it through Transmogrifier.
+  """
+  def is_representable?(%Activity{} = activity) do
+    with {:ok, _data} <- Transmogrifier.prepare_outgoing(activity.data) do
+      true
+    else
+      _e ->
+        false
+    end
+  end
+
+  @doc """
+  Publish a single message to a peer.  Takes a struct with the following
+  parameters set:
+
+  * `inbox`: the inbox to publish to
+  * `json`: the JSON message body representing the ActivityPub message
+  * `actor`: the actor which is signing the message
+  * `id`: the ActivityStreams URI of the message
+  """
+  def publish_one(%{inbox: inbox, json: json, actor: %User{} = actor, id: id} = params) do
+    Logger.info("Federating #{id} to #{inbox}")
+    host = URI.parse(inbox).host
+
+    digest = "SHA-256=" <> (:crypto.hash(:sha256, json) |> Base.encode64())
+
+    date =
+      NaiveDateTime.utc_now()
+      |> Timex.format!("{WDshort}, {0D} {Mshort} {YYYY} {h24}:{m}:{s} GMT")
+
+    signature =
+      Pleroma.Signature.sign(actor, %{
+        host: host,
+        "content-length": byte_size(json),
+        digest: digest,
+        date: date
+      })
+
+    with {:ok, %{status: code}} when code in 200..299 <-
+           result =
+             @httpoison.post(
+               inbox,
+               json,
+               [
+                 {"Content-Type", "application/activity+json"},
+                 {"Date", date},
+                 {"signature", signature},
+                 {"digest", digest}
+               ]
+             ) do
+      if !Map.has_key?(params, :unreachable_since) || params[:unreachable_since],
+        do: Instances.set_reachable(inbox)
+
+      result
+    else
+      {_post_result, response} ->
+        unless params[:unreachable_since], do: Instances.set_unreachable(inbox)
+        {:error, response}
+    end
+  end
+
+  defp should_federate?(inbox, public) do
+    if public do
+      true
+    else
+      inbox_info = URI.parse(inbox)
+      !Enum.member?(Pleroma.Config.get([:instance, :quarantined_instances], []), inbox_info.host)
+    end
+  end
+
+  @doc """
+  Publishes an activity to all relevant peers.
+  """
+  def publish(%User{} = actor, %Activity{} = activity) do
+    remote_followers =
+      if actor.follower_address in activity.recipients do
+        {:ok, followers} = User.get_followers(actor)
+        followers |> Enum.filter(&(!&1.local))
+      else
+        []
+      end
+
+    public = is_public?(activity)
+
+    if public && Config.get([:instance, :allow_relay]) do
+      Logger.info(fn -> "Relaying #{activity.data["id"]} out" end)
+      Relay.publish(activity)
+    end
+
+    {:ok, data} = Transmogrifier.prepare_outgoing(activity.data)
+    json = Jason.encode!(data)
+
+    (Pleroma.Web.Salmon.remote_users(activity) ++ remote_followers)
+    |> Enum.filter(fn user -> User.ap_enabled?(user) end)
+    |> Enum.map(fn %{info: %{source_data: data}} ->
+      (is_map(data["endpoints"]) && Map.get(data["endpoints"], "sharedInbox")) || data["inbox"]
+    end)
+    |> Enum.uniq()
+    |> Enum.filter(fn inbox -> should_federate?(inbox, public) end)
+    |> Instances.filter_reachable()
+    |> Enum.each(fn {inbox, unreachable_since} ->
+      Pleroma.Web.Federator.Publisher.enqueue_one(
+        __MODULE__,
+        %{
+          inbox: inbox,
+          json: json,
+          actor: actor,
+          id: activity.data["id"],
+          unreachable_since: unreachable_since
+        }
+      )
+    end)
+  end
+
+  def gather_webfinger_links(%User{} = user) do
+    [
+      %{"rel" => "self", "type" => "application/activity+json", "href" => user.ap_id},
+      %{
+        "rel" => "self",
+        "type" => "application/ld+json; profile=\"https://www.w3.org/ns/activitystreams\"",
+        "href" => user.ap_id
+      }
+    ]
+  end
+
+  def gather_nodeinfo_protocol_names, do: ["activitypub"]
+end

lib/pleroma/web/activity_pub/relay.ex

@@ -15,7 +15,7 @@ defmodule Pleroma.Web.ActivityPub.Relay do
 
   def follow(target_instance) do
     with %User{} = local_user <- get_actor(),
-         %User{} = target_user <- User.get_or_fetch_by_ap_id(target_instance),
+         {:ok, %User{} = target_user} <- User.get_or_fetch_by_ap_id(target_instance),
          {:ok, activity} <- ActivityPub.follow(local_user, target_user) do
       Logger.info("relay: followed instance: #{target_instance}; id=#{activity.data["id"]}")
       {:ok, activity}
@@ -28,7 +28,7 @@ defmodule Pleroma.Web.ActivityPub.Relay do
 
   def unfollow(target_instance) do
     with %User{} = local_user <- get_actor(),
-         %User{} = target_user <- User.get_or_fetch_by_ap_id(target_instance),
+         {:ok, %User{} = target_user} <- User.get_or_fetch_by_ap_id(target_instance),
          {:ok, activity} <- ActivityPub.unfollow(local_user, target_user) do
       Logger.info("relay: unfollowed instance: #{target_instance}: id=#{activity.data["id"]}")
       {:ok, activity}

lib/pleroma/web/activity_pub/transmogrifier.ex

@@ -11,7 +11,6 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
   alias Pleroma.Object.Containment
   alias Pleroma.Repo
   alias Pleroma.User
-  alias Pleroma.User
   alias Pleroma.Web.ActivityPub.ActivityPub
   alias Pleroma.Web.ActivityPub.Utils
   alias Pleroma.Web.ActivityPub.Visibility
@@ -126,7 +125,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
   def fix_implicit_addressing(object, _), do: object
 
   def fix_addressing(object) do
-    %User{} = user = User.get_or_fetch_by_ap_id(object["actor"])
+    {:ok, %User{} = user} = User.get_or_fetch_by_ap_id(object["actor"])
     followers_collection = User.ap_followers(user)
 
     object
@@ -407,7 +406,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
       |> fix_addressing
 
     with nil <- Activity.get_create_by_object_ap_id(object["id"]),
-         %User{} = user <- User.get_or_fetch_by_ap_id(data["actor"]) do
+         {:ok, %User{} = user} <- User.get_or_fetch_by_ap_id(data["actor"]) do
       object = fix_object(data["object"])
 
       params = %{
@@ -436,7 +435,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
         %{"type" => "Follow", "object" => followed, "actor" => follower, "id" => id} = data
       ) do
     with %User{local: true} = followed <- User.get_cached_by_ap_id(followed),
-         %User{} = follower <- User.get_or_fetch_by_ap_id(follower),
+         {:ok, %User{} = follower} <- User.get_or_fetch_by_ap_id(follower),
          {:ok, activity} <- ActivityPub.follow(follower, followed, id, false) do
       with deny_follow_blocked <- Pleroma.Config.get([:user, :deny_follow_blocked]),
            {:user_blocked, false} <-
@@ -485,7 +484,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
         %{"type" => "Accept", "object" => follow_object, "actor" => _actor, "id" => _id} = data
       ) do
     with actor <- Containment.get_actor(data),
-         %User{} = followed <- User.get_or_fetch_by_ap_id(actor),
+         {:ok, %User{} = followed} <- User.get_or_fetch_by_ap_id(actor),
          {:ok, follow_activity} <- get_follow_activity(follow_object, followed),
          {:ok, follow_activity} <- Utils.update_follow_state(follow_activity, "accept"),
          %User{local: true} = follower <- User.get_cached_by_ap_id(follow_activity.data["actor"]),
@@ -511,7 +510,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
         %{"type" => "Reject", "object" => follow_object, "actor" => _actor, "id" => _id} = data
       ) do
     with actor <- Containment.get_actor(data),
-         %User{} = followed <- User.get_or_fetch_by_ap_id(actor),
+         {:ok, %User{} = followed} <- User.get_or_fetch_by_ap_id(actor),
          {:ok, follow_activity} <- get_follow_activity(follow_object, followed),
          {:ok, follow_activity} <- Utils.update_follow_state(follow_activity, "reject"),
          %User{local: true} = follower <- User.get_cached_by_ap_id(follow_activity.data["actor"]),
@@ -535,7 +534,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
         %{"type" => "Like", "object" => object_id, "actor" => _actor, "id" => id} = data
       ) do
     with actor <- Containment.get_actor(data),
-         %User{} = actor <- User.get_or_fetch_by_ap_id(actor),
+         {:ok, %User{} = actor} <- User.get_or_fetch_by_ap_id(actor),
          {:ok, object} <- get_obj_helper(object_id),
          {:ok, activity, _object} <- ActivityPub.like(actor, object, id, false) do
       {:ok, activity}
@@ -548,7 +547,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
         %{"type" => "Announce", "object" => object_id, "actor" => _actor, "id" => id} = data
       ) do
     with actor <- Containment.get_actor(data),
-         %User{} = actor <- User.get_or_fetch_by_ap_id(actor),
+         {:ok, %User{} = actor} <- User.get_or_fetch_by_ap_id(actor),
          {:ok, object} <- get_obj_helper(object_id),
          public <- Visibility.is_public?(data),
          {:ok, activity, _object} <- ActivityPub.announce(actor, object, id, false, public) do
@@ -603,7 +602,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
     object_id = Utils.get_ap_id(object_id)
 
     with actor <- Containment.get_actor(data),
-         %User{} = actor <- User.get_or_fetch_by_ap_id(actor),
+         {:ok, %User{} = actor} <- User.get_or_fetch_by_ap_id(actor),
          {:ok, object} <- get_obj_helper(object_id),
          :ok <- Containment.contain_origin(actor.ap_id, object.data),
          {:ok, activity} <- ActivityPub.delete(object, false) do
@@ -622,7 +621,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
         } = data
       ) do
     with actor <- Containment.get_actor(data),
-         %User{} = actor <- User.get_or_fetch_by_ap_id(actor),
+         {:ok, %User{} = actor} <- User.get_or_fetch_by_ap_id(actor),
          {:ok, object} <- get_obj_helper(object_id),
          {:ok, activity, _} <- ActivityPub.unannounce(actor, object, id, false) do
       {:ok, activity}
@@ -640,7 +639,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
         } = _data
       ) do
     with %User{local: true} = followed <- User.get_cached_by_ap_id(followed),
-         %User{} = follower <- User.get_or_fetch_by_ap_id(follower),
+         {:ok, %User{} = follower} <- User.get_or_fetch_by_ap_id(follower),
          {:ok, activity} <- ActivityPub.unfollow(follower, followed, id, false) do
       User.unfollow(follower, followed)
       {:ok, activity}
@@ -659,7 +658,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
       ) do
     with true <- Pleroma.Config.get([:activitypub, :accept_blocks]),
          %User{local: true} = blocked <- User.get_cached_by_ap_id(blocked),
-         %User{} = blocker <- User.get_or_fetch_by_ap_id(blocker),
+         {:ok, %User{} = blocker} <- User.get_or_fetch_by_ap_id(blocker),
          {:ok, activity} <- ActivityPub.unblock(blocker, blocked, id, false) do
       User.unblock(blocker, blocked)
       {:ok, activity}
@@ -673,7 +672,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
       ) do
     with true <- Pleroma.Config.get([:activitypub, :accept_blocks]),
          %User{local: true} = blocked = User.get_cached_by_ap_id(blocked),
-         %User{} = blocker = User.get_or_fetch_by_ap_id(blocker),
+         {:ok, %User{} = blocker} = User.get_or_fetch_by_ap_id(blocker),
          {:ok, activity} <- ActivityPub.block(blocker, blocked, id, false) do
       User.unfollow(blocker, blocked)
       User.block(blocker, blocked)
@@ -692,7 +691,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
         } = data
       ) do
     with actor <- Containment.get_actor(data),
-         %User{} = actor <- User.get_or_fetch_by_ap_id(actor),
+         {:ok, %User{} = actor} <- User.get_or_fetch_by_ap_id(actor),
          {:ok, object} <- get_obj_helper(object_id),
          {:ok, activity, _, _} <- ActivityPub.unlike(actor, object, id, false) do
       {:ok, activity}
@@ -856,10 +855,16 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
     |> Map.put("tag", tags ++ mentions)
   end
 
+  def add_emoji_tags(%User{info: %{"emoji" => _emoji} = user_info} = object) do
+    user_info = add_emoji_tags(user_info)
+
+    object
+    |> Map.put(:info, user_info)
+  end
+
   # TODO: we should probably send mtime instead of unix epoch time for updated
-  def add_emoji_tags(object) do
+  def add_emoji_tags(%{"emoji" => emoji} = object) do
     tags = object["tag"] || []
-    emoji = object["emoji"] || []
 
     out =
       emoji
@@ -877,6 +882,10 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
     |> Map.put("tag", tags ++ out)
   end
 
+  def add_emoji_tags(object) do
+    object
+  end
+
   def set_conversation(object) do
     Map.put(object, "conversation", object["context"])
   end

lib/pleroma/web/activity_pub/utils.ex

@@ -20,6 +20,8 @@ defmodule Pleroma.Web.ActivityPub.Utils do
   require Logger
 
   @supported_object_types ["Article", "Note", "Video", "Page"]
+  @supported_report_states ~w(open closed resolved)
+  @valid_visibilities ~w(public unlisted private direct)
 
   # Some implementations send the actor URI as the actor field, others send the entire actor object,
   # so figure out what the actor's URI is based on what we have.
@@ -670,7 +672,8 @@ defmodule Pleroma.Web.ActivityPub.Utils do
       "actor" => params.actor.ap_id,
       "content" => params.content,
       "object" => object,
-      "context" => params.context
+      "context" => params.context,
+      "state" => "open"
     }
     |> Map.merge(additional)
   end
@@ -682,7 +685,7 @@ defmodule Pleroma.Web.ActivityPub.Utils do
   """
   def fetch_ordered_collection(from, pages_left, acc \\ []) do
     with {:ok, response} <- Tesla.get(from),
-         {:ok, collection} <- Poison.decode(response.body) do
+         {:ok, collection} <- Jason.decode(response.body) do
       case collection["type"] do
         "OrderedCollection" ->
           # If we've encountered the OrderedCollection and not the page,
@@ -713,4 +716,77 @@ defmodule Pleroma.Web.ActivityPub.Utils do
       end
     end
   end
+
+  #### Report-related helpers
+
+  def update_report_state(%Activity{} = activity, state) when state in @supported_report_states do
+    with new_data <- Map.put(activity.data, "state", state),
+         changeset <- Changeset.change(activity, data: new_data),
+         {:ok, activity} <- Repo.update(changeset) do
+      {:ok, activity}
+    end
+  end
+
+  def update_report_state(_, _), do: {:error, "Unsupported state"}
+
+  def update_activity_visibility(activity, visibility) when visibility in @valid_visibilities do
+    [to, cc, recipients] =
+      activity
+      |> get_updated_targets(visibility)
+      |> Enum.map(&Enum.uniq/1)
+
+    object_data =
+      activity.object.data
+      |> Map.put("to", to)
+      |> Map.put("cc", cc)
+
+    {:ok, object} =
+      activity.object
+      |> Object.change(%{data: object_data})
+      |> Object.update_and_set_cache()
+
+    activity_data =
+      activity.data
+      |> Map.put("to", to)
+      |> Map.put("cc", cc)
+
+    activity
+    |> Map.put(:object, object)
+    |> Activity.change(%{data: activity_data, recipients: recipients})
+    |> Repo.update()
+  end
+
+  def update_activity_visibility(_, _), do: {:error, "Unsupported visibility"}
+
+  defp get_updated_targets(
+         %Activity{data: %{"to" => to} = data, recipients: recipients},
+         visibility
+       ) do
+    cc = Map.get(data, "cc", [])
+    follower_address = User.get_cached_by_ap_id(data["actor"]).follower_address
+    public = "https://www.w3.org/ns/activitystreams#Public"
+
+    case visibility do
+      "public" ->
+        to = [public | List.delete(to, follower_address)]
+        cc = [follower_address | List.delete(cc, public)]
+        recipients = [public | recipients]
+        [to, cc, recipients]
+
+      "private" ->
+        to = [follower_address | List.delete(to, public)]
+        cc = List.delete(cc, public)
+        recipients = List.delete(recipients, public)
+        [to, cc, recipients]
+
+      "unlisted" ->
+        to = [follower_address | List.delete(to, public)]
+        cc = [public | List.delete(cc, follower_address)]
+        recipients = recipients ++ [follower_address, public]
+        [to, cc, recipients]
+
+      _ ->
+        [to, cc, recipients]
+    end
+  end
 end

lib/pleroma/web/activity_pub/views/user_view.ex

@@ -69,6 +69,11 @@ defmodule Pleroma.Web.ActivityPub.UserView do
 
     endpoints = render("endpoints.json", %{user: user})
 
+    user_tags =
+      user
+      |> Transmogrifier.add_emoji_tags()
+      |> Map.get("tag", [])
+
     %{
       "id" => user.ap_id,
       "type" => "Person",
@@ -87,7 +92,7 @@ defmodule Pleroma.Web.ActivityPub.UserView do
         "publicKeyPem" => public_key
       },
       "endpoints" => endpoints,
-      "tag" => user.info.source_data["tag"] || []
+      "tag" => (user.info.source_data["tag"] || []) ++ user_tags
     }
     |> Map.merge(maybe_make_image(&User.avatar_url/2, "icon", user))
     |> Map.merge(maybe_make_image(&User.banner_url/2, "image", user))

lib/pleroma/web/activity_pub/visibility.ex

@@ -1,6 +1,7 @@
 defmodule Pleroma.Web.ActivityPub.Visibility do
   alias Pleroma.Activity
   alias Pleroma.Object
+  alias Pleroma.Repo
   alias Pleroma.User
 
   def is_public?(%Object{data: %{"type" => "Tombstone"}}), do: false
@@ -13,11 +14,12 @@ defmodule Pleroma.Web.ActivityPub.Visibility do
   end
 
   def is_private?(activity) do
-    unless is_public?(activity) do
-      follower_address = User.get_cached_by_ap_id(activity.data["actor"]).follower_address
-      Enum.any?(activity.data["to"], &(&1 == follower_address))
+    with false <- is_public?(activity),
+         %User{follower_address: follower_address} <-
+           User.get_cached_by_ap_id(activity.data["actor"]) do
+      follower_address in activity.data["to"]
     else
-      false
+      _ -> false
     end
   end
 
@@ -38,24 +40,37 @@ defmodule Pleroma.Web.ActivityPub.Visibility do
     visible_for_user?(activity, nil) || Enum.any?(x, &(&1 in y))
   end
 
-  # guard
-  def entire_thread_visible_for_user?(nil, _user), do: false
+  def entire_thread_visible_for_user?(%Activity{} = activity, %User{} = user) do
+    {:ok, %{rows: [[result]]}} =
+      Ecto.Adapters.SQL.query(Repo, "SELECT thread_visibility($1, $2)", [
+        user.ap_id,
+        activity.data["id"]
+      ])
 
-  # XXX: Probably even more inefficient than the previous implementation intended to be a placeholder untill https://git.pleroma.social/pleroma/pleroma/merge_requests/971 is in develop
-  # credo:disable-for-previous-line Credo.Check.Readability.MaxLineLength
+    result
+  end
 
-  def entire_thread_visible_for_user?(
-        %Activity{} = tail,
-        # %Activity{data: %{"object" => %{"inReplyTo" => parent_id}}} = tail,
-        user
-      ) do
-    case Object.normalize(tail) do
-      %{data: %{"inReplyTo" => parent_id}} when is_binary(parent_id) ->
-        parent = Activity.get_in_reply_to_activity(tail)
-        visible_for_user?(tail, user) && entire_thread_visible_for_user?(parent, user)
+  def get_visibility(object) do
+    public = "https://www.w3.org/ns/activitystreams#Public"
+    to = object.data["to"] || []
+    cc = object.data["cc"] || []
 
-      _ ->
-        visible_for_user?(tail, user)
+    cond do
+      public in to ->
+        "public"
+
+      public in cc ->
+        "unlisted"
+
+      # this should use the sql for the object's activity
+      Enum.any?(to, &String.contains?(&1, "/followers")) ->
+        "private"
+
+      length(cc) > 0 ->
+        "private"
+
+      true ->
+        "direct"
     end
   end
 end

lib/pleroma/web/admin_api/admin_api_controller.ex

@@ -4,11 +4,16 @@
 
 defmodule Pleroma.Web.AdminAPI.AdminAPIController do
   use Pleroma.Web, :controller
+  alias Pleroma.Activity
   alias Pleroma.User
   alias Pleroma.UserInviteToken
+  alias Pleroma.Web.ActivityPub.ActivityPub
   alias Pleroma.Web.ActivityPub.Relay
   alias Pleroma.Web.AdminAPI.AccountView
+  alias Pleroma.Web.AdminAPI.ReportView
   alias Pleroma.Web.AdminAPI.Search
+  alias Pleroma.Web.CommonAPI
+  alias Pleroma.Web.MastodonAPI.StatusView
 
   import Pleroma.Web.ControllerHelper, only: [json_response: 3]
 
@@ -59,7 +64,7 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
       bio: "."
     }
 
-    changeset = User.register_changeset(%User{}, user_data, confirmed: true)
+    changeset = User.register_changeset(%User{}, user_data, need_confirmation: false)
     {:ok, user} = User.register(changeset)
 
     conn
@@ -101,7 +106,10 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
     search_params = %{
       query: params["query"],
       page: page,
-      page_size: page_size
+      page_size: page_size,
+      tags: params["tags"],
+      name: params["name"],
+      email: params["email"]
     }
 
     with {:ok, users, count} <- Search.user(Map.merge(search_params, filters)),
@@ -116,11 +124,11 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
            )
   end
 
-  @filters ~w(local external active deactivated)
-
-  defp maybe_parse_filters(filters) when is_nil(filters) or filters == "", do: %{}
+  @filters ~w(local external active deactivated is_admin is_moderator)
 
   @spec maybe_parse_filters(String.t()) :: %{required(String.t()) => true} | %{}
+  defp maybe_parse_filters(filters) when is_nil(filters) or filters == "", do: %{}
+
   defp maybe_parse_filters(filters) do
     filters
     |> String.split(",")
@@ -284,12 +292,88 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
     |> json(token.token)
   end
 
+  def list_reports(conn, params) do
+    params =
+      params
+      |> Map.put("type", "Flag")
+      |> Map.put("skip_preload", true)
+
+    reports =
+      []
+      |> ActivityPub.fetch_activities(params)
+      |> Enum.reverse()
+
+    conn
+    |> put_view(ReportView)
+    |> render("index.json", %{reports: reports})
+  end
+
+  def report_show(conn, %{"id" => id}) do
+    with %Activity{} = report <- Activity.get_by_id(id) do
+      conn
+      |> put_view(ReportView)
+      |> render("show.json", %{report: report})
+    else
+      _ -> {:error, :not_found}
+    end
+  end
+
+  def report_update_state(conn, %{"id" => id, "state" => state}) do
+    with {:ok, report} <- CommonAPI.update_report_state(id, state) do
+      conn
+      |> put_view(ReportView)
+      |> render("show.json", %{report: report})
+    end
+  end
+
+  def report_respond(%{assigns: %{user: user}} = conn, %{"id" => id} = params) do
+    with false <- is_nil(params["status"]),
+         %Activity{} <- Activity.get_by_id(id) do
+      params =
+        params
+        |> Map.put("in_reply_to_status_id", id)
+        |> Map.put("visibility", "direct")
+
+      {:ok, activity} = CommonAPI.post(user, params)
+
+      conn
+      |> put_view(StatusView)
+      |> render("status.json", %{activity: activity})
+    else
+      true ->
+        {:param_cast, nil}
+
+      nil ->
+        {:error, :not_found}
+    end
+  end
+
+  def status_update(conn, %{"id" => id} = params) do
+    with {:ok, activity} <- CommonAPI.update_activity_scope(id, params) do
+      conn
+      |> put_view(StatusView)
+      |> render("status.json", %{activity: activity})
+    end
+  end
+
+  def status_delete(%{assigns: %{user: user}} = conn, %{"id" => id}) do
+    with {:ok, %Activity{}} <- CommonAPI.delete(id, user) do
+      json(conn, %{})
+    end
+  end
+
   def errors(conn, {:error, :not_found}) do
     conn
     |> put_status(404)
     |> json("Not found")
   end
 
+  def errors(conn, {:error, reason}) do
+    conn
+    |> put_status(400)
+    |> json(reason)
+  end
+
   def errors(conn, {:param_cast, _}) do
     conn
     |> put_status(400)

lib/pleroma/web/admin_api/search.ex

@@ -10,45 +10,23 @@ defmodule Pleroma.Web.AdminAPI.Search do
 
   @page_size 50
 
-  def user(%{query: term} = params) when is_nil(term) or term == "" do
-    query = maybe_filtered_query(params)
+  defmacro not_empty_string(string) do
+    quote do
+      is_binary(unquote(string)) and unquote(string) != ""
+    end
+  end
+
+  @spec user(map()) :: {:ok, [User.t()], pos_integer()}
+  def user(params \\ %{}) do
+    query = User.Query.build(params) |> order_by([u], u.nickname)
 
     paginated_query =
-      maybe_filtered_query(params)
-      |> paginate(params[:page] || 1, params[:page_size] || @page_size)
+      User.Query.paginate(query, params[:page] || 1, params[:page_size] || @page_size)
 
-    count = query |> Repo.aggregate(:count, :id)
+    count = Repo.aggregate(query, :count, :id)
 
     results = Repo.all(paginated_query)
 
     {:ok, results, count}
   end
-
-  def user(%{query: term} = params) when is_binary(term) do
-    search_query = from(u in maybe_filtered_query(params), where: ilike(u.nickname, ^"%#{term}%"))
-
-    count = search_query |> Repo.aggregate(:count, :id)
-
-    results =
-      search_query
-      |> paginate(params[:page] || 1, params[:page_size] || @page_size)
-      |> Repo.all()
-
-    {:ok, results, count}
-  end
-
-  defp maybe_filtered_query(params) do
-    from(u in User, order_by: u.nickname)
-    |> User.maybe_local_user_query(params[:local])
-    |> User.maybe_external_user_query(params[:external])
-    |> User.maybe_active_user_query(params[:active])
-    |> User.maybe_deactivated_user_query(params[:deactivated])
-  end
-
-  defp paginate(query, page, page_size) do
-    from(u in query,
-      limit: ^page_size,
-      offset: ^((page - 1) * page_size)
-    )
-  end
 end

lib/pleroma/web/admin_api/views/report_view.ex

@@ -0,0 +1,41 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.AdminAPI.ReportView do
+  use Pleroma.Web, :view
+  alias Pleroma.Activity
+  alias Pleroma.User
+  alias Pleroma.Web.CommonAPI.Utils
+  alias Pleroma.Web.MastodonAPI.AccountView
+  alias Pleroma.Web.MastodonAPI.StatusView
+
+  def render("index.json", %{reports: reports}) do
+    %{
+      reports: render_many(reports, __MODULE__, "show.json", as: :report)
+    }
+  end
+
+  def render("show.json", %{report: report}) do
+    user = User.get_cached_by_ap_id(report.data["actor"])
+    created_at = Utils.to_masto_date(report.data["published"])
+
+    [account_ap_id | status_ap_ids] = report.data["object"]
+    account = User.get_cached_by_ap_id(account_ap_id)
+
+    statuses =
+      Enum.map(status_ap_ids, fn ap_id ->
+        Activity.get_by_ap_id_with_object(ap_id)
+      end)
+
+    %{
+      id: report.id,
+      account: AccountView.render("account.json", %{user: account}),
+      actor: AccountView.render("account.json", %{user: user}),
+      content: report.data["content"],
+      created_at: created_at,
+      statuses: StatusView.render("index.json", %{activities: statuses, as: :activity}),
+      state: report.data["state"]
+    }
+  end
+end

lib/pleroma/web/auth/authenticator.ex

@@ -42,4 +42,30 @@ defmodule Pleroma.Web.Auth.Authenticator do
     implementation().oauth_consumer_template() ||
       Pleroma.Config.get([:auth, :oauth_consumer_template], "consumer.html")
   end
+
+  @doc "Gets user by nickname or email for auth."
+  @spec fetch_user(String.t()) :: User.t() | nil
+  def fetch_user(name) do
+    User.get_by_nickname_or_email(name)
+  end
+
+  # Gets name and password from conn
+  #
+  @spec fetch_credentials(Plug.Conn.t() | map()) ::
+          {:ok, {name :: any, password :: any}} | {:error, :invalid_credentials}
+  def fetch_credentials(%Plug.Conn{params: params} = _),
+    do: fetch_credentials(params)
+
+  def fetch_credentials(params) do
+    case params do
+      %{"authorization" => %{"name" => name, "password" => password}} ->
+        {:ok, {name, password}}
+
+      %{"grant_type" => "password", "username" => name, "password" => password} ->
+        {:ok, {name, password}}
+
+      _ ->
+        {:error, :invalid_credentials}
+    end
+  end
 end

lib/pleroma/web/auth/ldap_authenticator.ex

@@ -7,6 +7,9 @@ defmodule Pleroma.Web.Auth.LDAPAuthenticator do
 
   require Logger
 
+  import Pleroma.Web.Auth.Authenticator,
+    only: [fetch_credentials: 1, fetch_user: 1]
+
   @behaviour Pleroma.Web.Auth.Authenticator
   @base Pleroma.Web.Auth.PleromaAuthenticator
 
@@ -20,30 +23,20 @@ defmodule Pleroma.Web.Auth.LDAPAuthenticator do
   defdelegate oauth_consumer_template, to: @base
 
   def get_user(%Plug.Conn{} = conn) do
-    if Pleroma.Config.get([:ldap, :enabled]) do
-      {name, password} =
-        case conn.params do
-          %{"authorization" => %{"name" => name, "password" => password}} ->
-            {name, password}
-
-          %{"grant_type" => "password", "username" => name, "password" => password} ->
-            {name, password}
-        end
-
-      case ldap_user(name, password) do
-        %User{} = user ->
-          {:ok, user}
-
-        {:error, {:ldap_connection_error, _}} ->
-          # When LDAP is unavailable, try default authenticator
-          @base.get_user(conn)
-
-        error ->
-          error
-      end
+    with {:ldap, true} <- {:ldap, Pleroma.Config.get([:ldap, :enabled])},
+         {:ok, {name, password}} <- fetch_credentials(conn),
+         %User{} = user <- ldap_user(name, password) do
+      {:ok, user}
     else
-      # Fall back to default authenticator
-      @base.get_user(conn)
+      {:error, {:ldap_connection_error, _}} ->
+        # When LDAP is unavailable, try default authenticator
+        @base.get_user(conn)
+
+      {:ldap, _} ->
+        @base.get_user(conn)
+
+      error ->
+        error
     end
   end
 
@@ -94,7 +87,7 @@ defmodule Pleroma.Web.Auth.LDAPAuthenticator do
 
     case :eldap.simple_bind(connection, "#{uid}=#{name},#{base}", password) do
       :ok ->
-        case User.get_by_nickname_or_email(name) do
+        case fetch_user(name) do
           %User{} = user ->
             user
 

lib/pleroma/web/auth/pleroma_authenticator.ex

@@ -8,19 +8,14 @@ defmodule Pleroma.Web.Auth.PleromaAuthenticator do
   alias Pleroma.Repo
   alias Pleroma.User
 
+  import Pleroma.Web.Auth.Authenticator,
+    only: [fetch_credentials: 1, fetch_user: 1]
+
   @behaviour Pleroma.Web.Auth.Authenticator
 
   def get_user(%Plug.Conn{} = conn) do
-    {name, password} =
-      case conn.params do
-        %{"authorization" => %{"name" => name, "password" => password}} ->
-          {name, password}
-
-        %{"grant_type" => "password", "username" => name, "password" => password} ->
-          {name, password}
-      end
-
-    with {_, %User{} = user} <- {:user, User.get_by_nickname_or_email(name)},
+    with {:ok, {name, password}} <- fetch_credentials(conn),
+         {_, %User{} = user} <- {:user, fetch_user(name)},
          {_, true} <- {:checkpw, Pbkdf2.checkpw(password, user.password_hash)} do
       {:ok, user}
     else
@@ -79,7 +74,7 @@ defmodule Pleroma.Web.Auth.PleromaAuthenticator do
                password_confirmation: random_password
              },
              external: true,
-             confirmed: true
+             need_confirmation: false
            )
            |> Repo.insert(),
          {:ok, _} <-

lib/pleroma/web/common_api/common_api.ex

@@ -71,6 +71,9 @@ defmodule Pleroma.Web.CommonAPI do
          {:ok, _} <- unpin(activity_id, user),
          {:ok, delete} <- ActivityPub.delete(object) do
       {:ok, delete}
+    else
+      _ ->
+        {:error, "Could not delete"}
     end
   end
 
@@ -116,32 +119,34 @@ defmodule Pleroma.Web.CommonAPI do
     end
   end
 
-  def get_visibility(%{"visibility" => visibility})
+  def get_visibility(%{"visibility" => visibility}, in_reply_to)
       when visibility in ~w{public unlisted private direct},
-      do: visibility
+      do: {visibility, get_replied_to_visibility(in_reply_to)}
 
-  def get_visibility(%{"in_reply_to_status_id" => status_id}) when not is_nil(status_id) do
-    case get_replied_to_activity(status_id) do
-      nil ->
-        "public"
+  def get_visibility(_, in_reply_to) when not is_nil(in_reply_to) do
+    visibility = get_replied_to_visibility(in_reply_to)
+    {visibility, visibility}
+  end
 
-      in_reply_to ->
-        # XXX: these heuristics should be moved out of MastodonAPI.
-        with %Object{} = object <- Object.normalize(in_reply_to) do
-          Pleroma.Web.MastodonAPI.StatusView.get_visibility(object)
-        end
+  def get_visibility(_, in_reply_to), do: {"public", get_replied_to_visibility(in_reply_to)}
+
+  def get_replied_to_visibility(nil), do: nil
+
+  def get_replied_to_visibility(activity) do
+    with %Object{} = object <- Object.normalize(activity) do
+      Pleroma.Web.ActivityPub.Visibility.get_visibility(object)
     end
   end
 
-  def get_visibility(_), do: "public"
-
   def post(user, %{"status" => status} = data) do
-    visibility = get_visibility(data)
     limit = Pleroma.Config.get([:instance, :limit])
 
     with status <- String.trim(status),
          attachments <- attachments_from_ids(data),
          in_reply_to <- get_replied_to_activity(data["in_reply_to_status_id"]),
+         {visibility, in_reply_to_visibility} <- get_visibility(data, in_reply_to),
+         {_, false} <-
+           {:private_to_public, in_reply_to_visibility == "direct" && visibility != "direct"},
          {content_html, mentions, tags} <-
            make_content_html(
              status,
@@ -151,8 +156,9 @@ defmodule Pleroma.Web.CommonAPI do
            ),
          {to, cc} <- to_for_user_and_mentions(user, mentions, in_reply_to, visibility),
          context <- make_context(in_reply_to),
-         cw <- data["spoiler_text"],
-         full_payload <- String.trim(status <> (data["spoiler_text"] || "")),
+         cw <- data["spoiler_text"] || "",
+         sensitive <- data["sensitive"] || Enum.member?(tags, {"#nsfw", "nsfw"}),
+         full_payload <- String.trim(status <> cw),
          length when length in 1..limit <- String.length(full_payload),
          object <-
            make_note_data(
@@ -164,16 +170,14 @@ defmodule Pleroma.Web.CommonAPI do
              in_reply_to,
              tags,
              cw,
-             cc
+             cc,
+             sensitive
            ),
          object <-
            Map.put(
              object,
              "emoji",
-             (Formatter.get_emoji(status) ++ Formatter.get_emoji(data["spoiler_text"]))
-             |> Enum.reduce(%{}, fn {name, file, _}, acc ->
-               Map.put(acc, name, "#{Pleroma.Web.Endpoint.static_url()}#{file}")
-             end)
+             Formatter.get_emoji_map(full_payload)
            ) do
       res =
         ActivityPub.create(
@@ -188,6 +192,8 @@ defmodule Pleroma.Web.CommonAPI do
         )
 
       res
+    else
+      e -> {:error, e}
     end
   end
 
@@ -196,7 +202,7 @@ defmodule Pleroma.Web.CommonAPI do
     user =
       with emoji <- emoji_from_profile(user),
            source_data <- (user.info.source_data || %{}) |> Map.put("tag", emoji),
-           info_cng <- Pleroma.User.Info.set_source_data(user.info, source_data),
+           info_cng <- User.Info.set_source_data(user.info, source_data),
            change <- Ecto.Changeset.change(user) |> Ecto.Changeset.put_embed(:info, info_cng),
            {:ok, user} <- User.update_and_set_cache(change) do
         user
@@ -229,7 +235,7 @@ defmodule Pleroma.Web.CommonAPI do
          } = activity <- get_by_id_or_ap_id(id_or_ap_id),
          true <- Enum.member?(object_to, "https://www.w3.org/ns/activitystreams#Public"),
          %{valid?: true} = info_changeset <-
-           Pleroma.User.Info.add_pinnned_activity(user.info, activity),
+           User.Info.add_pinnned_activity(user.info, activity),
          changeset <-
            Ecto.Changeset.change(user) |> Ecto.Changeset.put_embed(:info, info_changeset),
          {:ok, _user} <- User.update_and_set_cache(changeset) do
@@ -246,7 +252,7 @@ defmodule Pleroma.Web.CommonAPI do
   def unpin(id_or_ap_id, user) do
     with %Activity{} = activity <- get_by_id_or_ap_id(id_or_ap_id),
          %{valid?: true} = info_changeset <-
-           Pleroma.User.Info.remove_pinnned_activity(user.info, activity),
+           User.Info.remove_pinnned_activity(user.info, activity),
          changeset <-
            Ecto.Changeset.change(user) |> Ecto.Changeset.put_embed(:info, info_changeset),
          {:ok, _user} <- User.update_and_set_cache(changeset) do
@@ -314,6 +320,60 @@ defmodule Pleroma.Web.CommonAPI do
     end
   end
 
+  def update_report_state(activity_id, state) do
+    with %Activity{} = activity <- Activity.get_by_id(activity_id),
+         {:ok, activity} <- Utils.update_report_state(activity, state) do
+      {:ok, activity}
+    else
+      nil ->
+        {:error, :not_found}
+
+      {:error, reason} ->
+        {:error, reason}
+
+      _ ->
+        {:error, "Could not update state"}
+    end
+  end
+
+  def update_activity_scope(activity_id, opts \\ %{}) do
+    with %Activity{} = activity <- Activity.get_by_id_with_object(activity_id),
+         {:ok, activity} <- toggle_sensitive(activity, opts),
+         {:ok, activity} <- set_visibility(activity, opts) do
+      {:ok, activity}
+    else
+      nil ->
+        {:error, :not_found}
+
+      {:error, reason} ->
+        {:error, reason}
+    end
+  end
+
+  defp toggle_sensitive(activity, %{"sensitive" => sensitive}) when sensitive in ~w(true false) do
+    toggle_sensitive(activity, %{"sensitive" => String.to_existing_atom(sensitive)})
+  end
+
+  defp toggle_sensitive(%Activity{object: object} = activity, %{"sensitive" => sensitive})
+       when is_boolean(sensitive) do
+    new_data = Map.put(object.data, "sensitive", sensitive)
+
+    {:ok, object} =
+      object
+      |> Object.change(%{data: new_data})
+      |> Object.update_and_set_cache()
+
+    {:ok, Map.put(activity, :object, object)}
+  end
+
+  defp toggle_sensitive(activity, _), do: {:ok, activity}
+
+  defp set_visibility(activity, %{"visibility" => visibility}) do
+    Utils.update_activity_visibility(activity, visibility)
+  end
+
+  defp set_visibility(activity, _), do: {:ok, activity}
+
   def hide_reblogs(user, muted) do
     ap_id = muted.ap_id
 

lib/pleroma/web/common_api/utils.ex

@@ -223,7 +223,8 @@ defmodule Pleroma.Web.CommonAPI.Utils do
         in_reply_to,
         tags,
         cw \\ nil,
-        cc \\ []
+        cc \\ [],
+        sensitive \\ false
       ) do
     object = %{
       "type" => "Note",
@@ -231,19 +232,18 @@ defmodule Pleroma.Web.CommonAPI.Utils do
       "cc" => cc,
       "content" => content_html,
       "summary" => cw,
+      "sensitive" => !Enum.member?(["false", "False", "0", false], sensitive),
       "context" => context,
       "attachment" => attachments,
       "actor" => actor,
       "tag" => tags |> Enum.map(fn {_, tag} -> tag end) |> Enum.uniq()
     }
 
-    if in_reply_to do
-      in_reply_to_object = Object.normalize(in_reply_to)
-
-      object
-      |> Map.put("inReplyTo", in_reply_to_object.data["id"])
+    with false <- is_nil(in_reply_to),
+         %Object{} = in_reply_to_object <- Object.normalize(in_reply_to) do
+      Map.put(object, "inReplyTo", in_reply_to_object.data["id"])
     else
-      object
+      _ -> object
     end
   end
 

lib/pleroma/web/controller_helper.ex

@@ -10,12 +10,6 @@ defmodule Pleroma.Web.ControllerHelper do
   def truthy_param?(blank_value) when blank_value in [nil, ""], do: nil
   def truthy_param?(value), do: value not in @falsy_param_values
 
-  def oauth_scopes(params, default) do
-    # Note: `scopes` is used by Mastodon — supporting it but sticking to
-    # OAuth's standard `scope` wherever we control it
-    Pleroma.Web.OAuth.parse_scopes(params["scope"] || params["scopes"], default)
-  end
-
   def json_response(conn, status, json) do
     conn
     |> put_status(status)

lib/pleroma/web/endpoint.ex

@@ -29,6 +29,13 @@ defmodule Pleroma.Web.Endpoint do
     # credo:disable-for-previous-line Credo.Check.Readability.MaxLineLength
   )
 
+  plug(Plug.Static.IndexHtml, at: "/pleroma/admin/")
+
+  plug(Plug.Static,
+    at: "/pleroma/admin/",
+    from: {:pleroma, "priv/static/adminfe/"}
+  )
+
   # Code reloading can be explicitly enabled under the
   # :code_reloader configuration of your endpoint.
   if code_reloading? do

lib/pleroma/web/federator/federator.ex

@@ -7,13 +7,10 @@ defmodule Pleroma.Web.Federator do
   alias Pleroma.Object.Containment
   alias Pleroma.User
   alias Pleroma.Web.ActivityPub.ActivityPub
-  alias Pleroma.Web.ActivityPub.Relay
   alias Pleroma.Web.ActivityPub.Transmogrifier
   alias Pleroma.Web.ActivityPub.Utils
-  alias Pleroma.Web.ActivityPub.Visibility
+  alias Pleroma.Web.Federator.Publisher
   alias Pleroma.Web.Federator.RetryQueue
-  alias Pleroma.Web.OStatus
-  alias Pleroma.Web.Salmon
   alias Pleroma.Web.WebFinger
   alias Pleroma.Web.Websub
 
@@ -42,14 +39,6 @@ defmodule Pleroma.Web.Federator do
     PleromaJobQueue.enqueue(:federator_outgoing, __MODULE__, [:publish, activity], priority)
   end
 
-  def publish_single_ap(params) do
-    PleromaJobQueue.enqueue(:federator_outgoing, __MODULE__, [:publish_single_ap, params])
-  end
-
-  def publish_single_websub(websub) do
-    PleromaJobQueue.enqueue(:federator_outgoing, __MODULE__, [:publish_single_websub, websub])
-  end
-
   def verify_websub(websub) do
     PleromaJobQueue.enqueue(:federator_outgoing, __MODULE__, [:verify_websub, websub])
   end
@@ -62,10 +51,6 @@ defmodule Pleroma.Web.Federator do
     PleromaJobQueue.enqueue(:federator_outgoing, __MODULE__, [:refresh_subscriptions])
   end
 
-  def publish_single_salmon(params) do
-    PleromaJobQueue.enqueue(:federator_outgoing, __MODULE__, [:publish_single_salmon, params])
-  end
-
   # Job Worker Callbacks
 
   def perform(:refresh_subscriptions) do
@@ -95,23 +80,7 @@ defmodule Pleroma.Web.Federator do
     with actor when not is_nil(actor) <- User.get_cached_by_ap_id(activity.data["actor"]) do
       {:ok, actor} = WebFinger.ensure_keys_present(actor)
 
-      if Visibility.is_public?(activity) do
-        if OStatus.is_representable?(activity) do
-          Logger.info(fn -> "Sending #{activity.data["id"]} out via WebSub" end)
-          Websub.publish(Pleroma.Web.OStatus.feed_path(actor), actor, activity)
-
-          Logger.info(fn -> "Sending #{activity.data["id"]} out via Salmon" end)
-          Pleroma.Web.Salmon.publish(actor, activity)
-        end
-
-        if Keyword.get(Application.get_env(:pleroma, :instance), :allow_relay) do
-          Logger.info(fn -> "Relaying #{activity.data["id"]} out" end)
-          Relay.publish(activity)
-        end
-      end
-
-      Logger.info(fn -> "Sending #{activity.data["id"]} out via AP" end)
-      Pleroma.Web.ActivityPub.ActivityPub.publish(actor, activity)
+      Publisher.publish(actor, activity)
     end
   end
 
@@ -148,25 +117,11 @@ defmodule Pleroma.Web.Federator do
       _e ->
         # Just drop those for now
         Logger.info("Unhandled activity")
-        Logger.info(Poison.encode!(params, pretty: 2))
+        Logger.info(Jason.encode!(params, pretty: true))
         :error
     end
   end
 
-  def perform(:publish_single_salmon, params) do
-    Salmon.send_to_user(params)
-  end
-
-  def perform(:publish_single_ap, params) do
-    case ActivityPub.publish_one(params) do
-      {:ok, _} ->
-        :ok
-
-      {:error, _} ->
-        RetryQueue.enqueue(params, ActivityPub)
-    end
-  end
-
   def perform(
         :publish_single_websub,
         %{xml: _xml, topic: _topic, callback: _callback, secret: _secret} = params

lib/pleroma/web/federator/publisher.ex

@@ -0,0 +1,95 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.Federator.Publisher do
+  alias Pleroma.Activity
+  alias Pleroma.Config
+  alias Pleroma.User
+  alias Pleroma.Web.Federator.RetryQueue
+
+  require Logger
+
+  @moduledoc """
+  Defines the contract used by federation implementations to publish messages to
+  their peers.
+  """
+
+  @doc """
+  Determine whether an activity can be relayed using the federation module.
+  """
+  @callback is_representable?(Pleroma.Activity.t()) :: boolean()
+
+  @doc """
+  Relays an activity to a specified peer, determined by the parameters.  The
+  parameters used are controlled by the federation module.
+  """
+  @callback publish_one(Map.t()) :: {:ok, Map.t()} | {:error, any()}
+
+  @doc """
+  Enqueue publishing a single activity.
+  """
+  @spec enqueue_one(module(), Map.t()) :: :ok
+  def enqueue_one(module, %{} = params),
+    do: PleromaJobQueue.enqueue(:federator_outgoing, __MODULE__, [:publish_one, module, params])
+
+  @spec perform(atom(), module(), any()) :: {:ok, any()} | {:error, any()}
+  def perform(:publish_one, module, params) do
+    case apply(module, :publish_one, [params]) do
+      {:ok, _} ->
+        :ok
+
+      {:error, _e} ->
+        RetryQueue.enqueue(params, module)
+    end
+  end
+
+  def perform(type, _, _) do
+    Logger.debug("Unknown task: #{type}")
+    {:error, "Don't know what to do with this"}
+  end
+
+  @doc """
+  Relays an activity to all specified peers.
+  """
+  @callback publish(User.t(), Activity.t()) :: :ok | {:error, any()}
+
+  @spec publish(User.t(), Activity.t()) :: :ok
+  def publish(%User{} = user, %Activity{} = activity) do
+    Config.get([:instance, :federation_publisher_modules])
+    |> Enum.each(fn module ->
+      if module.is_representable?(activity) do
+        Logger.info("Publishing #{activity.data["id"]} using #{inspect(module)}")
+        module.publish(user, activity)
+      end
+    end)
+
+    :ok
+  end
+
+  @doc """
+  Gathers links used by an outgoing federation module for WebFinger output.
+  """
+  @callback gather_webfinger_links(User.t()) :: list()
+
+  @spec gather_webfinger_links(User.t()) :: list()
+  def gather_webfinger_links(%User{} = user) do
+    Config.get([:instance, :federation_publisher_modules])
+    |> Enum.reduce([], fn module, links ->
+      links ++ module.gather_webfinger_links(user)
+    end)
+  end
+
+  @doc """
+  Gathers nodeinfo protocol names supported by the federation module.
+  """
+  @callback gather_nodeinfo_protocol_names() :: list()
+
+  @spec gather_nodeinfo_protocol_names() :: list()
+  def gather_nodeinfo_protocol_names do
+    Config.get([:instance, :federation_publisher_modules])
+    |> Enum.reduce([], fn module, links ->
+      links ++ module.gather_nodeinfo_protocol_names()
+    end)
+  end
+end

lib/pleroma/web/http_signatures/http_signatures.ex

@@ -1,91 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-# https://tools.ietf.org/html/draft-cavage-http-signatures-08
-defmodule Pleroma.Web.HTTPSignatures do
-  alias Pleroma.User
-  alias Pleroma.Web.ActivityPub.ActivityPub
-  alias Pleroma.Web.ActivityPub.Utils
-
-  require Logger
-
-  def split_signature(sig) do
-    default = %{"headers" => "date"}
-
-    sig =
-      sig
-      |> String.trim()
-      |> String.split(",")
-      |> Enum.reduce(default, fn part, acc ->
-        [key | rest] = String.split(part, "=")
-        value = Enum.join(rest, "=")
-        Map.put(acc, key, String.trim(value, "\""))
-      end)
-
-    Map.put(sig, "headers", String.split(sig["headers"], ~r/\s/))
-  end
-
-  def validate(headers, signature, public_key) do
-    sigstring = build_signing_string(headers, signature["headers"])
-    Logger.debug("Signature: #{signature["signature"]}")
-    Logger.debug("Sigstring: #{sigstring}")
-    {:ok, sig} = Base.decode64(signature["signature"])
-    :public_key.verify(sigstring, :sha256, sig, public_key)
-  end
-
-  def validate_conn(conn) do
-    # TODO: How to get the right key and see if it is actually valid for that request.
-    # For now, fetch the key for the actor.
-    with actor_id <- Utils.get_ap_id(conn.params["actor"]),
-         {:ok, public_key} <- User.get_public_key_for_ap_id(actor_id) do
-      if validate_conn(conn, public_key) do
-        true
-      else
-        Logger.debug("Could not validate, re-fetching user and trying one more time")
-        # Fetch user anew and try one more time
-        with actor_id <- Utils.get_ap_id(conn.params["actor"]),
-             {:ok, _user} <- ActivityPub.make_user_from_ap_id(actor_id),
-             {:ok, public_key} <- User.get_public_key_for_ap_id(actor_id) do
-          validate_conn(conn, public_key)
-        end
-      end
-    else
-      _e ->
-        Logger.debug("Could not public key!")
-        false
-    end
-  end
-
-  def validate_conn(conn, public_key) do
-    headers = Enum.into(conn.req_headers, %{})
-    signature = split_signature(headers["signature"])
-    validate(headers, signature, public_key)
-  end
-
-  def build_signing_string(headers, used_headers) do
-    used_headers
-    |> Enum.map(fn header -> "#{header}: #{headers[header]}" end)
-    |> Enum.join("\n")
-  end
-
-  def sign(user, headers) do
-    with {:ok, %{info: %{keys: keys}}} <- Pleroma.Web.WebFinger.ensure_keys_present(user),
-         {:ok, private_key, _} = Pleroma.Web.Salmon.keys_from_pem(keys) do
-      sigstring = build_signing_string(headers, Map.keys(headers))
-
-      signature =
-        :public_key.sign(sigstring, :sha256, private_key)
-        |> Base.encode64()
-
-      [
-        keyId: user.ap_id <> "#main-key",
-        algorithm: "rsa-sha256",
-        headers: Map.keys(headers) |> Enum.join(" "),
-        signature: signature
-      ]
-      |> Enum.map(fn {k, v} -> "#{k}=\"#{v}\"" end)
-      |> Enum.join(",")
-    end
-  end
-end

lib/pleroma/web/mastodon_api/mastodon_api_controller.ex

@@ -8,7 +8,9 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
   alias Pleroma.Activity
   alias Pleroma.Bookmark
   alias Pleroma.Config
+  alias Pleroma.Conversation.Participation
   alias Pleroma.Filter
+  alias Pleroma.Formatter
   alias Pleroma.Notification
   alias Pleroma.Object
   alias Pleroma.Object.Fetcher
@@ -23,6 +25,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
   alias Pleroma.Web.CommonAPI
   alias Pleroma.Web.MastodonAPI.AccountView
   alias Pleroma.Web.MastodonAPI.AppView
+  alias Pleroma.Web.MastodonAPI.ConversationView
   alias Pleroma.Web.MastodonAPI.FilterView
   alias Pleroma.Web.MastodonAPI.ListView
   alias Pleroma.Web.MastodonAPI.MastodonAPI
@@ -34,20 +37,31 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
   alias Pleroma.Web.MediaProxy
   alias Pleroma.Web.OAuth.App
   alias Pleroma.Web.OAuth.Authorization
+  alias Pleroma.Web.OAuth.Scopes
   alias Pleroma.Web.OAuth.Token
+  alias Pleroma.Web.TwitterAPI.TwitterAPI
 
   alias Pleroma.Web.ControllerHelper
   import Ecto.Query
 
   require Logger
 
+  plug(
+    Pleroma.Plugs.RateLimitPlug,
+    %{
+      max_requests: Config.get([:app_account_creation, :max_requests]),
+      interval: Config.get([:app_account_creation, :interval])
+    }
+    when action in [:account_register]
+  )
+
   @httpoison Application.get_env(:pleroma, :httpoison)
   @local_mastodon_name "Mastodon-Local"
 
   action_fallback(:errors)
 
   def create_app(conn, params) do
-    scopes = ControllerHelper.oauth_scopes(params, ["read"])
+    scopes = Scopes.fetch_scopes(params, ["read"])
 
     app_attrs =
       params
@@ -86,7 +100,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
     user_params =
       %{}
       |> add_if_present(params, "display_name", :name)
-      |> add_if_present(params, "note", :bio, fn value -> {:ok, User.parse_bio(value)} end)
+      |> add_if_present(params, "note", :bio, fn value -> {:ok, User.parse_bio(value, user)} end)
       |> add_if_present(params, "avatar", :avatar, fn value ->
         with %Plug.Upload{} <- value,
              {:ok, object} <- ActivityPub.upload(value, type: :avatar) do
@@ -96,6 +110,12 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
         end
       end)
 
+    emojis_text = (user_params["display_name"] || "") <> (user_params["note"] || "")
+
+    user_info_emojis =
+      ((user.info.emoji || []) ++ Formatter.get_emoji_map(emojis_text))
+      |> Enum.dedup()
+
     info_params =
       [:no_rich_text, :locked, :hide_followers, :hide_follows, :hide_favorites, :show_role]
       |> Enum.reduce(%{}, fn key, acc ->
@@ -103,6 +123,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
           {:ok, ControllerHelper.truthy_param?(value)}
         end)
       end)
+      |> add_if_present(params, "default_scope", :default_scope)
       |> add_if_present(params, "header", :banner, fn value ->
         with %Plug.Upload{} <- value,
              {:ok, object} <- ActivityPub.upload(value, type: :banner) do
@@ -111,6 +132,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
           _ -> :error
         end
       end)
+      |> Map.put(:emoji, user_info_emojis)
 
     info_cng = User.Info.profile_update(user.info, info_params)
 
@@ -156,7 +178,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
     end
   end
 
-  @mastodon_api_level "2.5.0"
+  @mastodon_api_level "2.7.2"
 
   def masto_instance(conn, _params) do
     instance = Config.get(:instance)
@@ -281,11 +303,8 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
     activities =
       [user.ap_id | user.following]
       |> ActivityPub.fetch_activities(params)
-      |> ActivityPub.contain_timeline(user)
       |> Enum.reverse()
 
-    user = Repo.preload(user, bookmarks: :activity)
-
     conn
     |> add_link_headers(:home_timeline, activities)
     |> put_view(StatusView)
@@ -304,8 +323,6 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
       |> ActivityPub.fetch_public_activities()
       |> Enum.reverse()
 
-    user = Repo.preload(user, bookmarks: :activity)
-
     conn
     |> add_link_headers(:public_timeline, activities, false, %{"local" => local_only})
     |> put_view(StatusView)
@@ -313,8 +330,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
   end
 
   def user_statuses(%{assigns: %{user: reading_user}} = conn, params) do
-    with %User{} = user <- User.get_cached_by_id(params["id"]),
-         reading_user <- Repo.preload(reading_user, :bookmarks) do
+    with %User{} = user <- User.get_cached_by_id(params["id"]) do
       activities = ActivityPub.fetch_user_activities(user, reading_user, params)
 
       conn
@@ -341,8 +357,6 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
       |> ActivityPub.fetch_activities_query(params)
       |> Pagination.fetch_paginated(params)
 
-    user = Repo.preload(user, bookmarks: :activity)
-
     conn
     |> add_link_headers(:dm_timeline, activities)
     |> put_view(StatusView)
@@ -352,8 +366,6 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
   def get_status(%{assigns: %{user: user}} = conn, %{"id" => id}) do
     with %Activity{} = activity <- Activity.get_by_id_with_object(id),
          true <- Visibility.visible_for_user?(activity, user) do
-      user = Repo.preload(user, bookmarks: :activity)
-
       conn
       |> put_view(StatusView)
       |> try_render("status.json", %{activity: activity, for: user})
@@ -503,8 +515,6 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
   def reblog_status(%{assigns: %{user: user}} = conn, %{"id" => ap_id_or_id}) do
     with {:ok, announce, _activity} <- CommonAPI.repeat(ap_id_or_id, user),
          %Activity{} = announce <- Activity.normalize(announce.data) do
-      user = Repo.preload(user, bookmarks: :activity)
-
       conn
       |> put_view(StatusView)
       |> try_render("status.json", %{activity: announce, for: user, as: :activity})
@@ -514,8 +524,6 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
   def unreblog_status(%{assigns: %{user: user}} = conn, %{"id" => ap_id_or_id}) do
     with {:ok, _unannounce, %{data: %{"id" => id}}} <- CommonAPI.unrepeat(ap_id_or_id, user),
          %Activity{} = activity <- Activity.get_create_by_object_ap_id_with_object(id) do
-      user = Repo.preload(user, bookmarks: :activity)
-
       conn
       |> put_view(StatusView)
       |> try_render("status.json", %{activity: activity, for: user, as: :activity})
@@ -566,8 +574,6 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
          %User{} = user <- User.get_cached_by_nickname(user.nickname),
          true <- Visibility.visible_for_user?(activity, user),
          {:ok, _bookmark} <- Bookmark.create(user.id, activity.id) do
-      user = Repo.preload(user, bookmarks: :activity)
-
       conn
       |> put_view(StatusView)
       |> try_render("status.json", %{activity: activity, for: user, as: :activity})
@@ -579,8 +585,6 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
          %User{} = user <- User.get_cached_by_nickname(user.nickname),
          true <- Visibility.visible_for_user?(activity, user),
          {:ok, _bookmark} <- Bookmark.destroy(user.id, activity.id) do
-      user = Repo.preload(user, bookmarks: :activity)
-
       conn
       |> put_view(StatusView)
       |> try_render("status.json", %{activity: activity, for: user, as: :activity})
@@ -703,7 +707,42 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
     end
   end
 
-  def favourited_by(conn, %{"id" => id}) do
+  def set_mascot(%{assigns: %{user: user}} = conn, %{"file" => file}) do
+    with {:ok, object} <- ActivityPub.upload(file, actor: User.ap_id(user)),
+         %{} = attachment_data <- Map.put(object.data, "id", object.id),
+         %{type: type} = rendered <-
+           StatusView.render("attachment.json", %{attachment: attachment_data}) do
+      # Reject if not an image
+      if type == "image" do
+        # Sure!
+        # Save to the user's info
+        info_changeset = User.Info.mascot_update(user.info, rendered)
+
+        user_changeset =
+          user
+          |> Ecto.Changeset.change()
+          |> Ecto.Changeset.put_embed(:info, info_changeset)
+
+        {:ok, _user} = User.update_and_set_cache(user_changeset)
+
+        conn
+        |> json(rendered)
+      else
+        conn
+        |> put_resp_content_type("application/json")
+        |> send_resp(415, Jason.encode!(%{"error" => "mascots can only be images"}))
+      end
+    end
+  end
+
+  def get_mascot(%{assigns: %{user: user}} = conn, _params) do
+    mascot = User.get_mascot(user)
+
+    conn
+    |> json(mascot)
+  end
+
+  def favourited_by(%{assigns: %{user: user}} = conn, %{"id" => id}) do
     with %Activity{data: %{"object" => object}} <- Repo.get(Activity, id),
          %Object{data: %{"likes" => likes}} <- Object.normalize(object) do
       q = from(u in User, where: u.ap_id in ^likes)
@@ -711,13 +750,13 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
 
       conn
       |> put_view(AccountView)
-      |> render(AccountView, "accounts.json", %{users: users, as: :user})
+      |> render(AccountView, "accounts.json", %{for: user, users: users, as: :user})
     else
       _ -> json(conn, [])
     end
   end
 
-  def reblogged_by(conn, %{"id" => id}) do
+  def reblogged_by(%{assigns: %{user: user}} = conn, %{"id" => id}) do
     with %Activity{data: %{"object" => object}} <- Repo.get(Activity, id),
          %Object{data: %{"announcements" => announces}} <- Object.normalize(object) do
       q = from(u in User, where: u.ap_id in ^announces)
@@ -725,7 +764,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
 
       conn
       |> put_view(AccountView)
-      |> render("accounts.json", %{users: users, as: :user})
+      |> render("accounts.json", %{for: user, users: users, as: :user})
     else
       _ -> json(conn, [])
     end
@@ -782,7 +821,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
       conn
       |> add_link_headers(:followers, followers, user)
       |> put_view(AccountView)
-      |> render("accounts.json", %{users: followers, as: :user})
+      |> render("accounts.json", %{for: for_user, users: followers, as: :user})
     end
   end
 
@@ -799,7 +838,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
       conn
       |> add_link_headers(:following, followers, user)
       |> put_view(AccountView)
-      |> render("accounts.json", %{users: followers, as: :user})
+      |> render("accounts.json", %{for: for_user, users: followers, as: :user})
     end
   end
 
@@ -807,7 +846,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
     with {:ok, follow_requests} <- User.get_follow_requests(followed) do
       conn
       |> put_view(AccountView)
-      |> render("accounts.json", %{users: follow_requests, as: :user})
+      |> render("accounts.json", %{for: followed, users: follow_requests, as: :user})
     end
   end
 
@@ -1005,6 +1044,30 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
     end
   end
 
+  def status_search_query_with_gin(q, query) do
+    from([a, o] in q,
+      where:
+        fragment(
+          "to_tsvector('english', ?->>'content') @@ plainto_tsquery('english', ?)",
+          o.data,
+          ^query
+        ),
+      order_by: [desc: :id]
+    )
+  end
+
+  def status_search_query_with_rum(q, query) do
+    from([a, o] in q,
+      where:
+        fragment(
+          "? @@ plainto_tsquery('english', ?)",
+          o.fts_content,
+          ^query
+        ),
+      order_by: [fragment("? <=> now()::date", o.inserted_at)]
+    )
+  end
+
   def status_search(user, query) do
     fetched =
       if Regex.match?(~r/https?:/, query) do
@@ -1018,20 +1081,19 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
       end || []
 
     q =
-      from(
-        [a, o] in Activity.with_preloaded_object(Activity),
+      from([a, o] in Activity.with_preloaded_object(Activity),
         where: fragment("?->>'type' = 'Create'", a.data),
         where: "https://www.w3.org/ns/activitystreams#Public" in a.recipients,
-        where:
-          fragment(
-            "to_tsvector('english', ?->>'content') @@ plainto_tsquery('english', ?)",
-            o.data,
-            ^query
-          ),
-        limit: 20,
-        order_by: [desc: :id]
+        limit: 20
       )
 
+    q =
+      if Pleroma.Config.get([:database, :rum_enabled]) do
+        status_search_query_with_rum(q, query)
+      else
+        status_search_query_with_gin(q, query)
+      end
+
     Repo.all(q) ++ fetched
   end
 
@@ -1101,8 +1163,6 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
       ActivityPub.fetch_activities([], params)
       |> Enum.reverse()
 
-    user = Repo.preload(user, bookmarks: :activity)
-
     conn
     |> add_link_headers(:favourites, activities)
     |> put_view(StatusView)
@@ -1148,7 +1208,6 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
 
   def bookmarks(%{assigns: %{user: user}} = conn, params) do
     user = User.get_cached_by_id(user.id)
-    user = Repo.preload(user, bookmarks: :activity)
 
     bookmarks =
       Bookmark.for_user_query(user.id)
@@ -1156,7 +1215,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
 
     activities =
       bookmarks
-      |> Enum.map(fn b -> b.activity end)
+      |> Enum.map(fn b -> Map.put(b.activity, :bookmark, Map.delete(b, :activity)) end)
 
     conn
     |> add_link_headers(:bookmarks, bookmarks)
@@ -1221,7 +1280,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
     accounts
     |> Enum.each(fn account_id ->
       with %Pleroma.List{} = list <- Pleroma.List.get(id, user),
-           %User{} = followed <- Pleroma.User.get_cached_by_id(account_id) do
+           %User{} = followed <- User.get_cached_by_id(account_id) do
         Pleroma.List.unfollow(list, followed)
       end
     end)
@@ -1234,7 +1293,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
          {:ok, users} = Pleroma.List.get_following(list) do
       conn
       |> put_view(AccountView)
-      |> render("accounts.json", %{users: users, as: :user})
+      |> render("accounts.json", %{for: user, users: users, as: :user})
     end
   end
 
@@ -1265,8 +1324,6 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
         |> ActivityPub.fetch_activities_bounded(following, params)
         |> Enum.reverse()
 
-      user = Repo.preload(user, bookmarks: :activity)
-
       conn
       |> put_view(StatusView)
       |> render("index.json", %{activities: activities, for: user, as: :activity})
@@ -1294,8 +1351,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
       initial_state =
         %{
           meta: %{
-            streaming_api_base_url:
-              String.replace(Pleroma.Web.Endpoint.static_url(), "http", "ws"),
+            streaming_api_base_url: Pleroma.Web.Endpoint.websocket_url(),
             access_token: token,
             locale: "en",
             domain: Pleroma.Web.Endpoint.host(),
@@ -1308,7 +1364,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
             display_sensitive_media: false,
             reduce_motion: false,
             max_toot_chars: limit,
-            mascot: "/images/pleroma-fox-tan-smol.png"
+            mascot: User.get_mascot(user)["url"]
           },
           rights: %{
             delete_others_notice: present?(user.info.is_moderator),
@@ -1547,7 +1603,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
       user_id: user.id,
       phrase: phrase,
       context: context,
-      hide: Map.get(params, "irreversible", nil),
+      hide: Map.get(params, "irreversible", false),
       whole_word: Map.get(params, "boolean", true)
       # expires_at
     }
@@ -1652,7 +1708,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
               x,
               "id",
               case User.get_or_fetch(x["acct"]) do
-                %{id: id} -> id
+                {:ok, %User{id: id}} -> id
                 _ -> 0
               end
             )
@@ -1704,6 +1760,78 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
     end
   end
 
+  def account_register(
+        %{assigns: %{app: app}} = conn,
+        %{"username" => nickname, "email" => _, "password" => _, "agreement" => true} = params
+      ) do
+    params =
+      params
+      |> Map.take([
+        "email",
+        "captcha_solution",
+        "captcha_token",
+        "captcha_answer_data",
+        "token",
+        "password"
+      ])
+      |> Map.put("nickname", nickname)
+      |> Map.put("fullname", params["fullname"] || nickname)
+      |> Map.put("bio", params["bio"] || "")
+      |> Map.put("confirm", params["password"])
+
+    with {:ok, user} <- TwitterAPI.register_user(params, need_confirmation: true),
+         {:ok, token} <- Token.create_token(app, user, %{scopes: app.scopes}) do
+      json(conn, %{
+        token_type: "Bearer",
+        access_token: token.token,
+        scope: app.scopes,
+        created_at: Token.Utils.format_created_at(token)
+      })
+    else
+      {:error, errors} ->
+        conn
+        |> put_status(400)
+        |> json(Jason.encode!(errors))
+    end
+  end
+
+  def account_register(%{assigns: %{app: _app}} = conn, _params) do
+    conn
+    |> put_status(400)
+    |> json(%{error: "Missing parameters"})
+  end
+
+  def account_register(conn, _) do
+    conn
+    |> put_status(403)
+    |> json(%{error: "Invalid credentials"})
+  end
+
+  def conversations(%{assigns: %{user: user}} = conn, params) do
+    participations = Participation.for_user_with_last_activity_id(user, params)
+
+    conversations =
+      Enum.map(participations, fn participation ->
+        ConversationView.render("participation.json", %{participation: participation, user: user})
+      end)
+
+    conn
+    |> add_link_headers(:conversations, participations)
+    |> json(conversations)
+  end
+
+  def conversation_read(%{assigns: %{user: user}} = conn, %{"id" => participation_id}) do
+    with %Participation{} = participation <-
+           Repo.get_by(Participation, id: participation_id, user_id: user.id),
+         {:ok, participation} <- Participation.mark_as_read(participation) do
+      participation_view =
+        ConversationView.render("participation.json", %{participation: participation, user: user})
+
+      conn
+      |> json(participation_view)
+    end
+  end
+
   def try_render(conn, target, params)
       when is_binary(target) do
     res = render(conn, target, params)

lib/pleroma/web/mastodon_api/views/account_view.ex

@@ -40,7 +40,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do
     follow_activity = Pleroma.Web.ActivityPub.Utils.fetch_latest_follow(user, target)
 
     requested =
-      if follow_activity do
+      if follow_activity && !User.following?(target, user) do
         follow_activity.data["state"] == "pending"
       else
         false

lib/pleroma/web/mastodon_api/views/conversation_view.ex

@@ -0,0 +1,38 @@
+defmodule Pleroma.Web.MastodonAPI.ConversationView do
+  use Pleroma.Web, :view
+
+  alias Pleroma.Activity
+  alias Pleroma.Repo
+  alias Pleroma.Web.ActivityPub.ActivityPub
+  alias Pleroma.Web.MastodonAPI.AccountView
+  alias Pleroma.Web.MastodonAPI.StatusView
+
+  def render("participation.json", %{participation: participation, user: user}) do
+    participation = Repo.preload(participation, conversation: :users)
+
+    last_activity_id =
+      with nil <- participation.last_activity_id do
+        ActivityPub.fetch_latest_activity_id_for_context(participation.conversation.ap_id, %{
+          "user" => user,
+          "blocking_user" => user
+        })
+      end
+
+    activity = Activity.get_by_id_with_object(last_activity_id)
+
+    last_status = StatusView.render("status.json", %{activity: activity, for: user})
+
+    accounts =
+      AccountView.render("accounts.json", %{
+        users: participation.conversation.users,
+        as: :user
+      })
+
+    %{
+      id: participation.id |> to_string(),
+      accounts: accounts,
+      unread: !participation.read,
+      last_status: last_status
+    }
+  end
+end

lib/pleroma/web/mastodon_api/views/status_view.ex

@@ -16,6 +16,8 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
   alias Pleroma.Web.MastodonAPI.StatusView
   alias Pleroma.Web.MediaProxy
 
+  import Pleroma.Web.ActivityPub.Visibility, only: [get_visibility: 1]
+
   # TODO: Add cached version.
   defp get_replied_to_activities(activities) do
     activities
@@ -75,18 +77,22 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
 
   def render(
         "status.json",
-        %{activity: %{data: %{"type" => "Announce", "object" => object}} = activity} = opts
+        %{activity: %{data: %{"type" => "Announce", "object" => _object}} = activity} = opts
       ) do
     user = get_user(activity.data["actor"])
     created_at = Utils.to_masto_date(activity.data["published"])
+    activity_object = Object.normalize(activity)
+
+    reblogged_activity =
+      Activity.create_by_object_ap_id(activity_object.data["id"])
+      |> Activity.with_preloaded_bookmark(opts[:for])
+      |> Repo.one()
 
-    reblogged_activity = Activity.get_create_by_object_ap_id(object)
     reblogged = render("status.json", Map.put(opts, :activity, reblogged_activity))
 
-    activity_object = Object.normalize(activity)
     favorited = opts[:for] && opts[:for].ap_id in (activity_object.data["likes"] || [])
 
-    bookmarked = opts[:for] && CommonAPI.bookmarked?(opts[:for], reblogged_activity)
+    bookmarked = Activity.get_bookmark(reblogged_activity, opts[:for]) != nil
 
     mentions =
       activity.recipients
@@ -96,8 +102,8 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
 
     %{
       id: to_string(activity.id),
-      uri: object,
-      url: object,
+      uri: activity_object.data["id"],
+      url: activity_object.data["id"],
       account: AccountView.render("account.json", %{user: user}),
       in_reply_to_id: nil,
       in_reply_to_account_id: nil,
@@ -149,7 +155,13 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
 
     favorited = opts[:for] && opts[:for].ap_id in (object.data["likes"] || [])
 
-    bookmarked = opts[:for] && CommonAPI.bookmarked?(opts[:for], activity)
+    bookmarked = Activity.get_bookmark(activity, opts[:for]) != nil
+
+    thread_muted? =
+      case activity.thread_muted? do
+        thread_muted? when is_boolean(thread_muted?) -> thread_muted?
+        nil -> CommonAPI.thread_muted?(user, activity)
+      end
 
     attachment_data = object.data["attachment"] || []
     attachments = render_many(attachment_data, StatusView, "attachment.json", as: :attachment)
@@ -222,7 +234,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
       reblogged: reblogged?(activity, opts[:for]),
       favourited: present?(favorited),
       bookmarked: present?(bookmarked),
-      muted: CommonAPI.thread_muted?(user, activity) || User.mutes?(opts[:for], user),
+      muted: thread_muted? || User.mutes?(opts[:for], user),
       pinned: pinned?(activity, user),
       sensitive: sensitive,
       spoiler_text: summary_html,
@@ -336,30 +348,6 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
     end
   end
 
-  def get_visibility(object) do
-    public = "https://www.w3.org/ns/activitystreams#Public"
-    to = object.data["to"] || []
-    cc = object.data["cc"] || []
-
-    cond do
-      public in to ->
-        "public"
-
-      public in cc ->
-        "unlisted"
-
-      # this should use the sql for the object's activity
-      Enum.any?(to, &String.contains?(&1, "/followers")) ->
-        "private"
-
-      length(cc) > 0 ->
-        "private"
-
-      true ->
-        "direct"
-    end
-  end
-
   def render_content(%{data: %{"type" => "Video"}} = object) do
     with name when not is_nil(name) and name != "" <- object.data["name"] do
       "<p><a href=\"#{object.data["id"]}\">#{name}</a></p>#{object.data["content"]}"

lib/pleroma/web/nodeinfo/nodeinfo_controller.ex

@@ -10,6 +10,7 @@ defmodule Pleroma.Web.Nodeinfo.NodeinfoController do
   alias Pleroma.User
   alias Pleroma.Web
   alias Pleroma.Web.ActivityPub.MRF
+  alias Pleroma.Web.Federator.Publisher
 
   plug(Pleroma.Web.FederatingPlug)
 
@@ -137,7 +138,7 @@ defmodule Pleroma.Web.Nodeinfo.NodeinfoController do
         name: Pleroma.Application.name() |> String.downcase(),
         version: Pleroma.Application.version()
       },
-      protocols: ["ostatus", "activitypub"],
+      protocols: Publisher.gather_nodeinfo_protocol_names(),
       services: %{
         inbound: [],
         outbound: []

lib/pleroma/web/oauth.ex

@@ -3,18 +3,4 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.OAuth do
-  def parse_scopes(scopes, _default) when is_list(scopes) do
-    Enum.filter(scopes, &(&1 not in [nil, ""]))
-  end
-
-  def parse_scopes(scopes, default) when is_binary(scopes) do
-    scopes
-    |> String.trim()
-    |> String.split(~r/[\s,]+/)
-    |> parse_scopes(default)
-  end
-
-  def parse_scopes(_, default) do
-    default
-  end
 end

lib/pleroma/web/oauth/app.ex

@@ -6,6 +6,8 @@ defmodule Pleroma.Web.OAuth.App do
   use Ecto.Schema
   import Ecto.Changeset
 
+  @type t :: %__MODULE__{}
+
   schema "apps" do
     field(:client_name, :string)
     field(:redirect_uris, :string)

lib/pleroma/web/oauth/authorization.ex

@@ -13,39 +13,58 @@ defmodule Pleroma.Web.OAuth.Authorization do
   import Ecto.Changeset
   import Ecto.Query
 
+  @type t :: %__MODULE__{}
+
   schema "oauth_authorizations" do
     field(:token, :string)
     field(:scopes, {:array, :string}, default: [])
     field(:valid_until, :naive_datetime_usec)
     field(:used, :boolean, default: false)
-    belongs_to(:user, Pleroma.User, type: Pleroma.FlakeId)
+    belongs_to(:user, User, type: Pleroma.FlakeId)
     belongs_to(:app, App)
 
     timestamps()
   end
 
+  @spec create_authorization(App.t(), User.t() | %{}, [String.t()] | nil) ::
+          {:ok, Authorization.t()} | {:error, Changeset.t()}
   def create_authorization(%App{} = app, %User{} = user, scopes \\ nil) do
-    scopes = scopes || app.scopes
-    token = :crypto.strong_rand_bytes(32) |> Base.url_encode64(padding: false)
-
-    authorization = %Authorization{
-      token: token,
-      used: false,
+    %{
+      scopes: scopes || app.scopes,
       user_id: user.id,
-      app_id: app.id,
-      scopes: scopes,
-      valid_until: NaiveDateTime.add(NaiveDateTime.utc_now(), 60 * 10)
+      app_id: app.id
     }
-
-    Repo.insert(authorization)
+    |> create_changeset()
+    |> Repo.insert()
   end
 
+  @spec create_changeset(map()) :: Changeset.t()
+  def create_changeset(attrs \\ %{}) do
+    %Authorization{}
+    |> cast(attrs, [:user_id, :app_id, :scopes, :valid_until])
+    |> validate_required([:app_id, :scopes])
+    |> add_token()
+    |> add_lifetime()
+  end
+
+  defp add_token(changeset) do
+    token = :crypto.strong_rand_bytes(32) |> Base.url_encode64(padding: false)
+    put_change(changeset, :token, token)
+  end
+
+  defp add_lifetime(changeset) do
+    put_change(changeset, :valid_until, NaiveDateTime.add(NaiveDateTime.utc_now(), 60 * 10))
+  end
+
+  @spec use_changeset(Authtorizatiton.t(), map()) :: Changeset.t()
   def use_changeset(%Authorization{} = auth, params) do
     auth
     |> cast(params, [:used])
     |> validate_required([:used])
   end
 
+  @spec use_token(Authorization.t()) ::
+          {:ok, Authorization.t()} | {:error, Changeset.t()} | {:error, String.t()}
   def use_token(%Authorization{used: false, valid_until: valid_until} = auth) do
     if NaiveDateTime.diff(NaiveDateTime.utc_now(), valid_until) < 0 do
       Repo.update(use_changeset(auth, %{used: true}))
@@ -56,6 +75,7 @@ defmodule Pleroma.Web.OAuth.Authorization do
 
   def use_token(%Authorization{used: true}), do: {:error, "already used"}
 
+  @spec delete_user_authorizations(User.t()) :: {integer(), any()}
   def delete_user_authorizations(%User{id: user_id}) do
     from(
       a in Pleroma.Web.OAuth.Authorization,
@@ -63,4 +83,11 @@ defmodule Pleroma.Web.OAuth.Authorization do
     )
     |> Repo.delete_all()
   end
+
+  @doc "gets auth for app by token"
+  @spec get_by_token(App.t(), String.t()) :: {:ok, t()} | {:error, :not_found}
+  def get_by_token(%App{id: app_id} = _app, token) do
+    from(t in __MODULE__, where: t.app_id == ^app_id and t.token == ^token)
+    |> Repo.find_resource()
+  end
 end

lib/pleroma/web/oauth/oauth_controller.ex

@@ -13,8 +13,9 @@ defmodule Pleroma.Web.OAuth.OAuthController do
   alias Pleroma.Web.OAuth.App
   alias Pleroma.Web.OAuth.Authorization
   alias Pleroma.Web.OAuth.Token
-
-  import Pleroma.Web.ControllerHelper, only: [oauth_scopes: 2]
+  alias Pleroma.Web.OAuth.Token.Strategy.RefreshToken
+  alias Pleroma.Web.OAuth.Token.Strategy.Revoke, as: RevokeToken
+  alias Pleroma.Web.OAuth.Scopes
 
   if Pleroma.Config.oauth_consumer_enabled?(), do: plug(Ueberauth)
 
@@ -53,7 +54,7 @@ defmodule Pleroma.Web.OAuth.OAuthController do
   defp do_authorize(conn, params) do
     app = Repo.get_by(App, client_id: params["client_id"])
     available_scopes = (app && app.scopes) || []
-    scopes = oauth_scopes(params, nil) || available_scopes
+    scopes = Scopes.fetch_scopes(params, available_scopes)
 
     # Note: `params` might differ from `conn.params`; use `@params` not `@conn.params` in template
     render(conn, Authenticator.auth_template(), %{
@@ -109,7 +110,7 @@ defmodule Pleroma.Web.OAuth.OAuthController do
 
   defp handle_create_authorization_error(
          conn,
-         {scopes_issue, _},
+         {:error, scopes_issue},
          %{"authorization" => _} = params
        )
        when scopes_issue in [:unsupported_scopes, :missing_scopes] do
@@ -138,25 +139,33 @@ defmodule Pleroma.Web.OAuth.OAuthController do
     Authenticator.handle_error(conn, error)
   end
 
-  def token_exchange(conn, %{"grant_type" => "authorization_code"} = params) do
-    with %App{} = app <- get_app_from_request(conn, params),
-         fixed_token = fix_padding(params["code"]),
-         %Authorization{} = auth <-
-           Repo.get_by(Authorization, token: fixed_token, app_id: app.id),
-         %User{} = user <- User.get_cached_by_id(auth.user_id),
-         {:ok, token} <- Token.exchange_token(app, auth),
-         {:ok, inserted_at} <- DateTime.from_naive(token.inserted_at, "Etc/UTC") do
-      response = %{
-        token_type: "Bearer",
-        access_token: token.token,
-        refresh_token: token.refresh_token,
-        created_at: DateTime.to_unix(inserted_at),
-        expires_in: 60 * 10,
-        scope: Enum.join(token.scopes, " "),
-        me: user.ap_id
-      }
+  @doc "Renew access_token with refresh_token"
+  def token_exchange(
+        conn,
+        %{"grant_type" => "refresh_token", "refresh_token" => token} = _params
+      ) do
+    with {:ok, app} <- Token.Utils.fetch_app(conn),
+         {:ok, %{user: user} = token} <- Token.get_by_refresh_token(app, token),
+         {:ok, token} <- RefreshToken.grant(token) do
+      response_attrs = %{created_at: Token.Utils.format_created_at(token)}
 
-      json(conn, response)
+      json(conn, Token.Response.build(user, token, response_attrs))
+    else
+      _error ->
+        put_status(conn, 400)
+        |> json(%{error: "Invalid credentials"})
+    end
+  end
+
+  def token_exchange(conn, %{"grant_type" => "authorization_code"} = params) do
+    with {:ok, app} <- Token.Utils.fetch_app(conn),
+         fixed_token = Token.Utils.fix_padding(params["code"]),
+         {:ok, auth} <- Authorization.get_by_token(app, fixed_token),
+         %User{} = user <- User.get_cached_by_id(auth.user_id),
+         {:ok, token} <- Token.exchange_token(app, auth) do
+      response_attrs = %{created_at: Token.Utils.format_created_at(token)}
+
+      json(conn, Token.Response.build(user, token, response_attrs))
     else
       _error ->
         put_status(conn, 400)
@@ -168,25 +177,14 @@ defmodule Pleroma.Web.OAuth.OAuthController do
         conn,
         %{"grant_type" => "password"} = params
       ) do
-    with {_, {:ok, %User{} = user}} <- {:get_user, Authenticator.get_user(conn)},
-         %App{} = app <- get_app_from_request(conn, params),
+    with {:ok, %User{} = user} <- Authenticator.get_user(conn),
+         {:ok, app} <- Token.Utils.fetch_app(conn),
          {:auth_active, true} <- {:auth_active, User.auth_active?(user)},
          {:user_active, true} <- {:user_active, !user.info.deactivated},
-         scopes <- oauth_scopes(params, app.scopes),
-         [] <- scopes -- app.scopes,
-         true <- Enum.any?(scopes),
+         {:ok, scopes} <- validate_scopes(app, params),
          {:ok, auth} <- Authorization.create_authorization(app, user, scopes),
          {:ok, token} <- Token.exchange_token(app, auth) do
-      response = %{
-        token_type: "Bearer",
-        access_token: token.token,
-        refresh_token: token.refresh_token,
-        expires_in: 60 * 10,
-        scope: Enum.join(token.scopes, " "),
-        me: user.ap_id
-      }
-
-      json(conn, response)
+      json(conn, Token.Response.build(user, token))
     else
       {:auth_active, false} ->
         # Per https://github.com/tootsuite/mastodon/blob/
@@ -218,10 +216,24 @@ defmodule Pleroma.Web.OAuth.OAuthController do
     token_exchange(conn, params)
   end
 
-  def token_revoke(conn, %{"token" => token} = params) do
-    with %App{} = app <- get_app_from_request(conn, params),
-         %Token{} = token <- Repo.get_by(Token, token: token, app_id: app.id),
-         {:ok, %Token{}} <- Repo.delete(token) do
+  def token_exchange(conn, %{"grant_type" => "client_credentials"} = _params) do
+    with {:ok, app} <- Token.Utils.fetch_app(conn),
+         {:ok, auth} <- Authorization.create_authorization(app, %User{}),
+         {:ok, token} <- Token.exchange_token(app, auth) do
+      json(conn, Token.Response.build_for_client_credentials(token))
+    else
+      _error ->
+        put_status(conn, 400)
+        |> json(%{error: "Invalid credentials"})
+    end
+  end
+
+  # Bad request
+  def token_exchange(conn, params), do: bad_request(conn, params)
+
+  def token_revoke(conn, %{"token" => _token} = params) do
+    with {:ok, app} <- Token.Utils.fetch_app(conn),
+         {:ok, _token} <- RevokeToken.revoke(app, params) do
       json(conn, %{})
     else
       _error ->
@@ -230,17 +242,27 @@ defmodule Pleroma.Web.OAuth.OAuthController do
     end
   end
 
+  def token_revoke(conn, params), do: bad_request(conn, params)
+
+  # Response for bad request
+  defp bad_request(conn, _) do
+    conn
+    |> put_status(500)
+    |> json(%{error: "Bad request"})
+  end
+
   @doc "Prepares OAuth request to provider for Ueberauth"
   def prepare_request(conn, %{"provider" => provider, "authorization" => auth_attrs}) do
     scope =
-      oauth_scopes(auth_attrs, [])
-      |> Enum.join(" ")
+      auth_attrs
+      |> Scopes.fetch_scopes([])
+      |> Scopes.to_string()
 
     state =
       auth_attrs
       |> Map.delete("scopes")
       |> Map.put("scope", scope)
-      |> Poison.encode!()
+      |> Jason.encode!()
 
     params =
       auth_attrs
@@ -278,25 +300,22 @@ defmodule Pleroma.Web.OAuth.OAuthController do
     params = callback_params(params)
 
     with {:ok, registration} <- Authenticator.get_registration(conn) do
-      user = Repo.preload(registration, :user).user
       auth_attrs = Map.take(params, ~w(client_id redirect_uri scope scopes state))
 
-      if user do
-        create_authorization(
-          conn,
-          %{"authorization" => auth_attrs},
-          user: user
-        )
-      else
-        registration_params =
-          Map.merge(auth_attrs, %{
-            "nickname" => Registration.nickname(registration),
-            "email" => Registration.email(registration)
-          })
+      case Repo.get_assoc(registration, :user) do
+        {:ok, user} ->
+          create_authorization(conn, %{"authorization" => auth_attrs}, user: user)
 
-        conn
-        |> put_session(:registration_id, registration.id)
-        |> registration_details(%{"authorization" => registration_params})
+        _ ->
+          registration_params =
+            Map.merge(auth_attrs, %{
+              "nickname" => Registration.nickname(registration),
+              "email" => Registration.email(registration)
+            })
+
+          conn
+          |> put_session(:registration_id, registration.id)
+          |> registration_details(%{"authorization" => registration_params})
       end
     else
       _ ->
@@ -307,7 +326,7 @@ defmodule Pleroma.Web.OAuth.OAuthController do
   end
 
   defp callback_params(%{"state" => state} = params) do
-    Map.merge(params, Poison.decode!(state))
+    Map.merge(params, Jason.decode!(state))
   end
 
   def registration_details(conn, %{"authorization" => auth_attrs}) do
@@ -315,7 +334,7 @@ defmodule Pleroma.Web.OAuth.OAuthController do
       client_id: auth_attrs["client_id"],
       redirect_uri: auth_attrs["redirect_uri"],
       state: auth_attrs["state"],
-      scopes: oauth_scopes(auth_attrs, []),
+      scopes: Scopes.fetch_scopes(auth_attrs, []),
       nickname: auth_attrs["nickname"],
       email: auth_attrs["email"]
     })
@@ -390,48 +409,12 @@ defmodule Pleroma.Web.OAuth.OAuthController do
            {:get_user, (user && {:ok, user}) || Authenticator.get_user(conn)},
          %App{} = app <- Repo.get_by(App, client_id: client_id),
          true <- redirect_uri in String.split(app.redirect_uris),
-         scopes <- oauth_scopes(auth_attrs, []),
-         {:unsupported_scopes, []} <- {:unsupported_scopes, scopes -- app.scopes},
-         # Note: `scope` param is intentionally not optional in this context
-         {:missing_scopes, false} <- {:missing_scopes, scopes == []},
+         {:ok, scopes} <- validate_scopes(app, auth_attrs),
          {:auth_active, true} <- {:auth_active, User.auth_active?(user)} do
       Authorization.create_authorization(app, user, scopes)
     end
   end
 
-  # XXX - for whatever reason our token arrives urlencoded, but Plug.Conn should be
-  # decoding it.  Investigate sometime.
-  defp fix_padding(token) do
-    token
-    |> URI.decode()
-    |> Base.url_decode64!(padding: false)
-    |> Base.url_encode64(padding: false)
-  end
-
-  defp get_app_from_request(conn, params) do
-    # Per RFC 6749, HTTP Basic is preferred to body params
-    {client_id, client_secret} =
-      with ["Basic " <> encoded] <- get_req_header(conn, "authorization"),
-           {:ok, decoded} <- Base.decode64(encoded),
-           [id, secret] <-
-             String.split(decoded, ":")
-             |> Enum.map(fn s -> URI.decode_www_form(s) end) do
-        {id, secret}
-      else
-        _ -> {params["client_id"], params["client_secret"]}
-      end
-
-    if client_id && client_secret do
-      Repo.get_by(
-        App,
-        client_id: client_id,
-        client_secret: client_secret
-      )
-    else
-      nil
-    end
-  end
-
   # Special case: Local MastodonFE
   defp redirect_uri(conn, "."), do: mastodon_api_url(conn, :login)
 
@@ -441,4 +424,12 @@ defmodule Pleroma.Web.OAuth.OAuthController do
 
   defp put_session_registration_id(conn, registration_id),
     do: put_session(conn, :registration_id, registration_id)
+
+  @spec validate_scopes(App.t(), map()) ::
+          {:ok, list()} | {:error, :missing_scopes | :unsupported_scopes}
+  defp validate_scopes(app, params) do
+    params
+    |> Scopes.fetch_scopes(app.scopes)
+    |> Scopes.validates(app.scopes)
+  end
 end

lib/pleroma/web/oauth/scopes.ex

@@ -0,0 +1,67 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.OAuth.Scopes do
+  @moduledoc """
+  Functions for dealing with scopes.
+  """
+
+  @doc """
+  Fetch scopes from requiest params.
+
+  Note: `scopes` is used by Mastodon — supporting it but sticking to
+  OAuth's standard `scope` wherever we control it
+  """
+  @spec fetch_scopes(map(), list()) :: list()
+  def fetch_scopes(params, default) do
+    parse_scopes(params["scope"] || params["scopes"], default)
+  end
+
+  def parse_scopes(scopes, _default) when is_list(scopes) do
+    Enum.filter(scopes, &(&1 not in [nil, ""]))
+  end
+
+  def parse_scopes(scopes, default) when is_binary(scopes) do
+    scopes
+    |> to_list
+    |> parse_scopes(default)
+  end
+
+  def parse_scopes(_, default) do
+    default
+  end
+
+  @doc """
+  Convert scopes string to list
+  """
+  @spec to_list(binary()) :: [binary()]
+  def to_list(nil), do: []
+
+  def to_list(str) do
+    str
+    |> String.trim()
+    |> String.split(~r/[\s,]+/)
+  end
+
+  @doc """
+  Convert scopes list to string
+  """
+  @spec to_string(list()) :: binary()
+  def to_string(scopes), do: Enum.join(scopes, " ")
+
+  @doc """
+  Validates scopes.
+  """
+  @spec validates(list() | nil, list()) ::
+          {:ok, list()} | {:error, :missing_scopes | :unsupported_scopes}
+  def validates([], _app_scopes), do: {:error, :missing_scopes}
+  def validates(nil, _app_scopes), do: {:error, :missing_scopes}
+
+  def validates(scopes, app_scopes) do
+    case scopes -- app_scopes do
+      [] -> {:ok, scopes}
+      _ -> {:error, :unsupported_scopes}
+    end
+  end
+end

lib/pleroma/web/oauth/token.ex

@@ -6,6 +6,7 @@ defmodule Pleroma.Web.OAuth.Token do
   use Ecto.Schema
 
   import Ecto.Query
+  import Ecto.Changeset
 
   alias Pleroma.Repo
   alias Pleroma.User
@@ -13,39 +14,86 @@ defmodule Pleroma.Web.OAuth.Token do
   alias Pleroma.Web.OAuth.Authorization
   alias Pleroma.Web.OAuth.Token
 
+  @expires_in Pleroma.Config.get([:oauth2, :token_expires_in], 600)
+  @type t :: %__MODULE__{}
+
   schema "oauth_tokens" do
     field(:token, :string)
     field(:refresh_token, :string)
     field(:scopes, {:array, :string}, default: [])
     field(:valid_until, :naive_datetime_usec)
-    belongs_to(:user, Pleroma.User, type: Pleroma.FlakeId)
+    belongs_to(:user, User, type: Pleroma.FlakeId)
     belongs_to(:app, App)
 
     timestamps()
   end
 
+  @doc "Gets token for app by access token"
+  @spec get_by_token(App.t(), String.t()) :: {:ok, t()} | {:error, :not_found}
+  def get_by_token(%App{id: app_id} = _app, token) do
+    from(t in __MODULE__, where: t.app_id == ^app_id and t.token == ^token)
+    |> Repo.find_resource()
+  end
+
+  @doc "Gets token for app by refresh token"
+  @spec get_by_refresh_token(App.t(), String.t()) :: {:ok, t()} | {:error, :not_found}
+  def get_by_refresh_token(%App{id: app_id} = _app, token) do
+    from(t in __MODULE__,
+      where: t.app_id == ^app_id and t.refresh_token == ^token,
+      preload: [:user]
+    )
+    |> Repo.find_resource()
+  end
+
+  @spec exchange_token(App.t(), Authorization.t()) ::
+          {:ok, Token.t()} | {:error, Changeset.t()}
   def exchange_token(app, auth) do
     with {:ok, auth} <- Authorization.use_token(auth),
          true <- auth.app_id == app.id do
-      create_token(app, User.get_cached_by_id(auth.user_id), auth.scopes)
+      user = if auth.user_id, do: User.get_cached_by_id(auth.user_id), else: %User{}
+
+      create_token(
+        app,
+        user,
+        %{scopes: auth.scopes}
+      )
     end
   end
 
-  def create_token(%App{} = app, %User{} = user, scopes \\ nil) do
-    scopes = scopes || app.scopes
-    token = :crypto.strong_rand_bytes(32) |> Base.url_encode64(padding: false)
-    refresh_token = :crypto.strong_rand_bytes(32) |> Base.url_encode64(padding: false)
+  defp put_token(changeset) do
+    changeset
+    |> change(%{token: Token.Utils.generate_token()})
+    |> validate_required([:token])
+    |> unique_constraint(:token)
+  end
 
-    token = %Token{
-      token: token,
-      refresh_token: refresh_token,
-      scopes: scopes,
-      user_id: user.id,
-      app_id: app.id,
-      valid_until: NaiveDateTime.add(NaiveDateTime.utc_now(), 60 * 10)
-    }
+  defp put_refresh_token(changeset, attrs) do
+    refresh_token = Map.get(attrs, :refresh_token, Token.Utils.generate_token())
 
-    Repo.insert(token)
+    changeset
+    |> change(%{refresh_token: refresh_token})
+    |> validate_required([:refresh_token])
+    |> unique_constraint(:refresh_token)
+  end
+
+  defp put_valid_until(changeset, attrs) do
+    expires_in =
+      Map.get(attrs, :valid_until, NaiveDateTime.add(NaiveDateTime.utc_now(), @expires_in))
+
+    changeset
+    |> change(%{valid_until: expires_in})
+    |> validate_required([:valid_until])
+  end
+
+  @spec create_token(App.t(), User.t(), map()) :: {:ok, Token} | {:error, Changeset.t()}
+  def create_token(%App{} = app, %User{} = user, attrs \\ %{}) do
+    %__MODULE__{user_id: user.id, app_id: app.id}
+    |> cast(%{scopes: attrs[:scopes] || app.scopes}, [:scopes])
+    |> validate_required([:scopes, :app_id])
+    |> put_valid_until(attrs)
+    |> put_token()
+    |> put_refresh_token(attrs)
+    |> Repo.insert()
   end
 
   def delete_user_tokens(%User{id: user_id}) do
@@ -73,4 +121,10 @@ defmodule Pleroma.Web.OAuth.Token do
     |> Repo.all()
     |> Repo.preload(:app)
   end
+
+  def is_expired?(%__MODULE__{valid_until: valid_until}) do
+    NaiveDateTime.diff(NaiveDateTime.utc_now(), valid_until) > 0
+  end
+
+  def is_expired?(_), do: false
 end

lib/pleroma/web/oauth/token/response.ex

@@ -0,0 +1,32 @@
+defmodule Pleroma.Web.OAuth.Token.Response do
+  @moduledoc false
+
+  alias Pleroma.User
+  alias Pleroma.Web.OAuth.Token.Utils
+
+  @expires_in Pleroma.Config.get([:oauth2, :token_expires_in], 600)
+
+  @doc false
+  def build(%User{} = user, token, opts \\ %{}) do
+    %{
+      token_type: "Bearer",
+      access_token: token.token,
+      refresh_token: token.refresh_token,
+      expires_in: @expires_in,
+      scope: Enum.join(token.scopes, " "),
+      me: user.ap_id
+    }
+    |> Map.merge(opts)
+  end
+
+  def build_for_client_credentials(token) do
+    %{
+      token_type: "Bearer",
+      access_token: token.token,
+      refresh_token: token.refresh_token,
+      created_at: Utils.format_created_at(token),
+      expires_in: @expires_in,
+      scope: Enum.join(token.scopes, " ")
+    }
+  end
+end

lib/pleroma/web/oauth/token/strategy/refresh_token.ex

@@ -0,0 +1,54 @@
+defmodule Pleroma.Web.OAuth.Token.Strategy.RefreshToken do
+  @moduledoc """
+  Functions for dealing with refresh token strategy.
+  """
+
+  alias Pleroma.Config
+  alias Pleroma.Repo
+  alias Pleroma.Web.OAuth.Token
+  alias Pleroma.Web.OAuth.Token.Strategy.Revoke
+
+  @doc """
+  Will grant access token by refresh token.
+  """
+  @spec grant(Token.t()) :: {:ok, Token.t()} | {:error, any()}
+  def grant(token) do
+    access_token = Repo.preload(token, [:user, :app])
+
+    result =
+      Repo.transaction(fn ->
+        token_params = %{
+          app: access_token.app,
+          user: access_token.user,
+          scopes: access_token.scopes
+        }
+
+        access_token
+        |> revoke_access_token()
+        |> create_access_token(token_params)
+      end)
+
+    case result do
+      {:ok, {:error, reason}} -> {:error, reason}
+      {:ok, {:ok, token}} -> {:ok, token}
+      {:error, reason} -> {:error, reason}
+    end
+  end
+
+  defp revoke_access_token(token) do
+    Revoke.revoke(token)
+  end
+
+  defp create_access_token({:error, error}, _), do: {:error, error}
+
+  defp create_access_token({:ok, token}, %{app: app, user: user} = token_params) do
+    Token.create_token(app, user, add_refresh_token(token_params, token.refresh_token))
+  end
+
+  defp add_refresh_token(params, token) do
+    case Config.get([:oauth2, :issue_new_refresh_token], false) do
+      true -> Map.put(params, :refresh_token, token)
+      false -> params
+    end
+  end
+end

lib/pleroma/web/oauth/token/strategy/revoke.ex

@@ -0,0 +1,22 @@
+defmodule Pleroma.Web.OAuth.Token.Strategy.Revoke do
+  @moduledoc """
+  Functions for dealing with revocation.
+  """
+
+  alias Pleroma.Repo
+  alias Pleroma.Web.OAuth.App
+  alias Pleroma.Web.OAuth.Token
+
+  @doc "Finds and revokes access token for app and by token"
+  @spec revoke(App.t(), map()) :: {:ok, Token.t()} | {:error, :not_found | Ecto.Changeset.t()}
+  def revoke(%App{} = app, %{"token" => token} = _attrs) do
+    with {:ok, token} <- Token.get_by_token(app, token),
+         do: revoke(token)
+  end
+
+  @doc "Revokes access token"
+  @spec revoke(Token.t()) :: {:ok, Token.t()} | {:error, Ecto.Changeset.t()}
+  def revoke(%Token{} = token) do
+    Repo.delete(token)
+  end
+end

lib/pleroma/web/oauth/token/utils.ex

@@ -0,0 +1,68 @@
+defmodule Pleroma.Web.OAuth.Token.Utils do
+  @moduledoc """
+  Auxiliary functions for dealing with tokens.
+  """
+
+  alias Pleroma.Repo
+  alias Pleroma.Web.OAuth.App
+
+  @doc "Fetch app by client credentials from request"
+  @spec fetch_app(Plug.Conn.t()) :: {:ok, App.t()} | {:error, :not_found}
+  def fetch_app(conn) do
+    res =
+      conn
+      |> fetch_client_credentials()
+      |> fetch_client
+
+    case res do
+      %App{} = app -> {:ok, app}
+      _ -> {:error, :not_found}
+    end
+  end
+
+  defp fetch_client({id, secret}) when is_binary(id) and is_binary(secret) do
+    Repo.get_by(App, client_id: id, client_secret: secret)
+  end
+
+  defp fetch_client({_id, _secret}), do: nil
+
+  defp fetch_client_credentials(conn) do
+    # Per RFC 6749, HTTP Basic is preferred to body params
+    with ["Basic " <> encoded] <- Plug.Conn.get_req_header(conn, "authorization"),
+         {:ok, decoded} <- Base.decode64(encoded),
+         [id, secret] <-
+           Enum.map(
+             String.split(decoded, ":"),
+             fn s -> URI.decode_www_form(s) end
+           ) do
+      {id, secret}
+    else
+      _ -> {conn.params["client_id"], conn.params["client_secret"]}
+    end
+  end
+
+  @doc "convert token inserted_at to unix timestamp"
+  def format_created_at(%{inserted_at: inserted_at} = _token) do
+    inserted_at
+    |> DateTime.from_naive!("Etc/UTC")
+    |> DateTime.to_unix()
+  end
+
+  @doc false
+  @spec generate_token(keyword()) :: binary()
+  def generate_token(opts \\ []) do
+    opts
+    |> Keyword.get(:size, 32)
+    |> :crypto.strong_rand_bytes()
+    |> Base.url_encode64(padding: false)
+  end
+
+  # XXX - for whatever reason our token arrives urlencoded, but Plug.Conn should be
+  # decoding it.  Investigate sometime.
+  def fix_padding(token) do
+    token
+    |> URI.decode()
+    |> Base.url_decode64!(padding: false)
+    |> Base.url_encode64(padding: false)
+  end
+end