Merge remote-tracking branch 'origin/develop' into shigusegubu

* origin/develop: (734 commits)
  update fast sanitize dependency
  streamer: use direct object for filter checks when there is no valid child object in an activity
  Bump fast_sanitize version
  Fix moderation log crash
  Undo dialyzer fix
  Fix count
  AdminAPI: Omit relay user from users list
  Fix "the call ... will never return" warning
  Bump fast_sanitize dependency so this builds on more than just Linux
  Remove following column from the migrations
  Update test for get_recipients_from_activity
  Fix bookmark migration using a query with a schema
  Add a migration to fix blocked follows
  Fix duplicate recipients
  Rip out the rest of htmlsanitizeex
  Remove commented-out code
  Fix XSS emoji test
  Bump fast_sanitize dependency
  Fix two migrations sharing the same module name
  Migrate missing follow requests
  ...

.formatter.exs

@@ -1,3 +1,3 @@
 [
-  inputs: ["mix.exs", "{config,lib,test}/**/*.{ex,exs}"]
+  inputs: ["mix.exs", "{config,lib,test}/**/*.{ex,exs}", "priv/repo/migrations/*.exs"]
 ]

.gitignore

@@ -38,7 +38,12 @@ erl_crash.dump
 
 # Prevent committing docs files
 /priv/static/doc/*
+docs/generated_config.md
 
 # Code test coverage
 /cover
 /Elixir.*.coverdata
+
+.idea
+pleroma.iml
+

.gitlab-ci.yml

@@ -15,6 +15,7 @@ cache:
 stages:
   - build
   - test
+  - benchmark
   - deploy
   - release
 
@@ -44,6 +45,19 @@ docs-build:
     paths:
       - priv/static/doc
 
+benchmark:
+  stage: benchmark
+  variables:
+    MIX_ENV: benchmark
+  services:
+  - name: lainsoykaf/postgres-with-rum
+    alias: postgres
+    command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"]
+  script:
+    - mix deps.get
+    - mix ecto.create
+    - mix ecto.migrate
+    - mix pleroma.load_testing
 
 unit-testing:
   stage: test
@@ -85,19 +99,14 @@ analysis:
 
 docs-deploy:
   stage: deploy
-  image: alpine:3.9
+  image: alpine:latest
   only:
-  - master@pleroma/pleroma
+  - stable@pleroma/pleroma
   - develop@pleroma/pleroma
   before_script:
-    - apk update && apk add openssh-client rsync
+  - apk add curl
   script:
-    - mkdir -p ~/.ssh
+  - curl -X POST -F"token=$DOCS_PIPELINE_TRIGGER" -F'ref=master' -F"variables[BRANCH]=$CI_COMMIT_REF_NAME" https://git.pleroma.social/api/v4/projects/673/trigger/pipeline
-    - echo "${SSH_HOST_KEY}" > ~/.ssh/known_hosts
-    - eval $(ssh-agent -s)
-    - echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add -
-    - rsync -hrvz --delete -e "ssh -p ${SSH_PORT}" priv/static/doc/ "${SSH_USER_HOST_LOCATION}/${CI_COMMIT_REF_NAME}"
-
 review_app:
   image: alpine:3.9
   stage: deploy
@@ -149,8 +158,10 @@ amd64:
   # TODO: Replace with upstream image when 1.9.0 comes out
   image: rinpatch/elixir:1.9.0-rc.0
   only: &release-only
-  - master@pleroma/pleroma
+  - stable@pleroma/pleroma
   - develop@pleroma/pleroma
+  - /^maint/.*$/@pleroma/pleroma
+  - /^release/.*$/@pleroma/pleroma
   artifacts: &release-artifacts
     name: "pleroma-$CI_COMMIT_REF_NAME-$CI_COMMIT_SHORT_SHA-$CI_JOB_NAME"
     paths:

CHANGELOG.md

@@ -4,62 +4,169 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
 ## [Unreleased]
+### Removed
+- **Breaking**: Removed 1.0+ deprecated configurations `Pleroma.Upload, :strip_exif` and `:instance, :dedupe_media`
+- **Breaking**: OStatus protocol support
+
+### Changed
+- **Breaking:** Elixir >=1.8 is now required (was >= 1.7)
+- Replaced [pleroma_job_queue](https://git.pleroma.social/pleroma/pleroma_job_queue) and `Pleroma.Web.Federator.RetryQueue` with [Oban](https://github.com/sorentwo/oban) (see [`docs/config.md`](docs/config.md) on migrating customized worker / retry settings)
+- Introduced [quantum](https://github.com/quantum-elixir/quantum-core) job scheduler
+- Enabled `:instance, extended_nickname_format` in the default config
+- Add `rel="ugc"` to all links in statuses, to prevent SEO spam
+- Extract RSS functionality from OStatus
+- MRF (Simple Policy): Also use `:accept`/`:reject` on the actors rather than only their activities
+<details>
+  <summary>API Changes</summary>
+
+- **Breaking:** Admin API: Return link alongside with token on password reset
+- **Breaking:** `/api/pleroma/admin/users/invite_token` now uses `POST`, changed accepted params and returns full invite in json instead of only token string.
+- Admin API: Return `total` when querying for reports
+- Mastodon API: Return `pleroma.direct_conversation_id` when creating a direct message (`POST /api/v1/statuses`)
+- Admin API: Return link alongside with token on password reset
+- Mastodon API: Add `pleroma.direct_conversation_id` to the status endpoint (`GET /api/v1/statuses/:id`)
+- Mastodon API: `pleroma.thread_muted` to the Status entity
+- Mastodon API: Mark the direct conversation as read for the author when they send a new direct message
+</details>
+
+### Added
+- Refreshing poll results for remote polls
+- Authentication: Added rate limit for password-authorized actions / login existence checks
+- Mix task to re-count statuses for all users (`mix pleroma.count_statuses`)
+- Support for `X-Forwarded-For` and similar HTTP headers which used by reverse proxies to pass a real user IP address to the backend. Must not be enabled unless your instance is behind at least one reverse proxy (such as Nginx, Apache HTTPD or Varnish Cache).
+<details>
+  <summary>API Changes</summary>
+
+- Job queue stats to the healthcheck page
+- Admin API: Add ability to require password reset
+- Mastodon API: Account entities now include `follow_requests_count` (planned Mastodon 3.x addition)
+- Pleroma API: `GET /api/v1/pleroma/accounts/:id/scrobbles` to get a list of recently scrobbled items
+- Pleroma API: `POST /api/v1/pleroma/scrobble` to scrobble a media item
+- Mastodon API: Add `upload_limit`, `avatar_upload_limit`, `background_upload_limit`, and `banner_upload_limit` to `/api/v1/instance`
+- Mastodon API: Add `pleroma.unread_conversation_count` to the Account entity
+- OAuth: support for hierarchical permissions / [Mastodon 2.4.3 OAuth permissions](https://docs.joinmastodon.org/api/permissions/)
+- Metadata Link: Atom syndication Feed
+- Mastodon API: Add `exclude_visibilities` parameter to the timeline and notification endpoints
+- Admin API: `/users/:nickname/toggle_activation` endpoint is now deprecated in favor of: `/users/activate`, `/users/deactivate`, both accept `nicknames` array
+- Admin API: `POST/DELETE /api/pleroma/admin/users/:nickname/permission_group/:permission_group` are deprecated in favor of: `POST/DELETE /api/pleroma/admin/users/permission_group/:permission_group` (both accept `nicknames` array), `DELETE /api/pleroma/admin/users` (`nickname` query param or `nickname` sent in JSON body) is deprecated in favor of: `DELETE /api/pleroma/admin/users` (`nicknames` query array param or `nicknames` sent in JSON body).
+- Admin API: Add `GET /api/pleroma/admin/relay` endpoint - lists all followed relays
+- Pleroma API: `POST /api/v1/pleroma/conversations/read` to mark all conversations as read
+- Mastodon API: Add `/api/v1/markers` for managing timeline read markers
+
+### Changed
+- **Breaking:** Elixir >=1.8 is now required (was >= 1.7)
+- **Breaking:** Admin API: Return link alongside with token on password reset
+- Replaced [pleroma_job_queue](https://git.pleroma.social/pleroma/pleroma_job_queue) and `Pleroma.Web.Federator.RetryQueue` with [Oban](https://github.com/sorentwo/oban) (see [`docs/config.md`](docs/config.md) on migrating customized worker / retry settings)
+- Introduced [quantum](https://github.com/quantum-elixir/quantum-core) job scheduler
+- Admin API: Return `total` when querying for reports
+- Mastodon API: Return `pleroma.direct_conversation_id` when creating a direct message (`POST /api/v1/statuses`)
+- Admin API: Return link alongside with token on password reset
+- MRF (Simple Policy): Also use `:accept`/`:reject` on the actors rather than only their activities
+- OStatus: Extract RSS functionality
+- Mastodon API: Add `pleroma.direct_conversation_id` to the status endpoint (`GET /api/v1/statuses/:id`)
+- Mastodon API: Mark the direct conversation as read for the author when they send a new direct message
+</details>
+- Deprecated `User.Info` embedded schema (fields moved to `User`)
+
+### Fixed
+- Report emails now include functional links to profiles of remote user accounts
+<details>
+  <summary>API Changes</summary>
+
+- Mastodon API: Fix private and direct statuses not being filtered out from the public timeline for an authenticated user (`GET /api/v1/timelines/public`)
+- Mastodon API: Inability to get some local users by nickname in `/api/v1/accounts/:id_or_nickname`
+</details>
+
+## [1.1.2] - 2019-10-18
+### Fixed
+- `pleroma_ctl` trying to connect to a running instance when generating the config, which of course doesn't exist.
+
+## [1.1.1] - 2019-10-18
+### Fixed
+- One of the migrations between 1.0.0 and 1.1.0 wiping user info of the relay user because of unexpected behavior of postgresql's `jsonb_set`, resulting in inability to post in the default configuration. If you were affected, please run the following query in postgres console, the relay user will be recreated automatically:
+```
+delete from users where ap_id = 'https://your.instance.hostname/relay';
+```
+- Bad user search matches
+
+## [1.1.0] - 2019-10-14
+**Breaking:** The stable branch has been changed from `master` to `stable`. If you want to keep using 1.0, the `release/1.0` branch will receive security updates for 6 months after 1.1 release.
+
+**OTP Note:** `pleroma_ctl` in 1.0 defaults to `master` and doesn't support specifying arbitrary branches, making `./pleroma_ctl update` fail. To fix this, fetch a version of `pleroma_ctl` from 1.1 using the command below and proceed with the update normally:
+```
+curl -Lo ./bin/pleroma_ctl 'https://git.pleroma.social/pleroma/pleroma/raw/develop/rel/files/bin/pleroma_ctl'
+```
 ### Security
-- OStatus: eliminate the possibility of a protocol downgrade attack.
+- Mastodon API: respect post privacy in `/api/v1/statuses/:id/{favourited,reblogged}_by`
-- OStatus: prevent following locked accounts, bypassing the approval process.
+
+### Removed
+- **Breaking:** GNU Social API with Qvitter extensions support
+- Emoji: Remove longfox emojis.
+- Remove `Reply-To` header from report emails for admins.
+- ActivityPub: The `/objects/:uuid/likes` endpoint.
 
 ### Changed
 - **Breaking:** Configuration: A setting to explicitly disable the mailer was added, defaulting to true, if you are using a mailer add `config :pleroma, Pleroma.Emails.Mailer, enabled: true` to your config
 - **Breaking:** Configuration: `/media/` is now removed when `base_url` is configured, append `/media/` to your `base_url` config to keep the old behaviour if desired
+- **Breaking:** `/api/pleroma/notifications/read` is moved to `/api/v1/pleroma/notifications/read` and now supports `max_id` and responds with Mastodon API entities.
+- Configuration: added `config/description.exs`, from which `docs/config.md` is generated
 - Configuration: OpenGraph and TwitterCard providers enabled by default
 - Configuration: Filter.AnonymizeFilename added ability to retain file extension with custom text
 - Federation: Return 403 errors when trying to request pages from a user's follower/following collections if they have `hide_followers`/`hide_follows` set
 - NodeInfo: Return `skipThreadContainment` in `metadata` for the `skip_thread_containment` option
 - NodeInfo: Return `mailerEnabled` in `metadata`
 - Mastodon API: Unsubscribe followers when they unfollow a user
+- Mastodon API: `pleroma.thread_muted` key in the Status entity
 - AdminAPI: Add "godmode" while fetching user statuses (i.e. admin can see private statuses)
+- Improve digest email template
+– Pagination: (optional) return `total` alongside with `items` when paginating
+- The `Pleroma.FlakeId` module has been replaced with the `flake_id` library.
 
 ### Fixed
-- Not being able to pin unlisted posts
+- Following from Osada
-- Objects being re-embedded to activities after being updated (e.g faved/reposted). Running 'mix pleroma.database prune_objects' again is advised.
+- Favorites timeline doing database-intensive queries
 - Metadata rendering errors resulting in the entire page being inaccessible
 - `federation_incoming_replies_max_depth` option being ignored in certain cases
-- Federation/MediaProxy not working with instances that have wrong certificate order
 - Mastodon API: Handling of search timeouts (`/api/v1/search` and `/api/v2/search`)
+- Mastodon API: Misskey's endless polls being unable to render
 - Mastodon API: Embedded relationships not being properly rendered in the Account entity of Status entity
-- Mastodon API: follower/following counters not being nullified, when `hide_follows`/`hide_followers` is set
+- Mastodon API: Notifications endpoint crashing if one notification failed to render
-- Mastodon API: `muted` in the Status entity, using author's account to determine if the tread was muted
+- Mastodon API: `exclude_replies` is correctly handled again.
 - Mastodon API: Add `account_id`, `type`, `offset`, and `limit` to search API (`/api/v1/search` and `/api/v2/search`)
 - Mastodon API, streaming: Fix filtering of notifications based on blocks/mutes/thread mutes
-- ActivityPub C2S: follower/following collection pages being inaccessible even when authentifucated if `hide_followers`/ `hide_follows` was set
+- Mastodon API: Fix private and direct statuses not being filtered out from the public timeline for an authenticated user (`GET /api/v1/timelines/public`)
-- Existing user id not being preserved on insert conflict
+- Mastodon API: Ensure the `account` field is not empty when rendering Notification entities.
+- Mastodon API: Inability to get some local users by nickname in `/api/v1/accounts/:id_or_nickname`
+- Mastodon API: Blocks are now treated consistently between the Streaming API and the Timeline APIs
 - Rich Media: Parser failing when no TTL can be found by image TTL setters
 - Rich Media: The crawled URL is now spliced into the rich media data.
 - ActivityPub S2S: sharedInbox usage has been mostly aligned with the rules in the AP specification.
-- ActivityPub S2S: remote user deletions now work the same as local user deletions.
+- ActivityPub C2S: follower/following collection pages being inaccessible even when authentifucated if `hide_followers`/ `hide_follows` was set
-- ActivityPub S2S: POST requests are now signed with `(request-target)` pseudo-header.
+- ActivityPub: Deactivated user deletion
-- Not being able to access the Mastodon FE login page on private instances
+- ActivityPub: Fix `/users/:nickname/inbox` crashing without an authenticated user
-- Invalid SemVer version generation, when the current branch does not have commits ahead of tag/checked out on a tag
+- MRF: fix ability to follow a relay when AntiFollowbotPolicy was enabled
+- ActivityPub: Correct addressing of Undo.
+- ActivityPub: Correct addressing of profile update activities.
+- ActivityPub: Polls are now refreshed when necessary.
+- Report emails now include functional links to profiles of remote user accounts
+- Existing user id not being preserved on insert conflict
 - Pleroma.Upload base_url was not automatically whitelisted by MediaProxy. Now your custom CDN or file hosting will be accessed directly as expected.
 - Report email not being sent to admins when the reporter is a remote user
-- MRF: ensure that subdomain_match calls are case-insensitive
+- Reverse Proxy limiting `max_body_length` was incorrectly defined and only checked `Content-Length` headers which may not be sufficient in some circumstances
-- MRF: fix use of unserializable keyword lists in describe() implementations
 
 ### Added
+- Expiring/ephemeral activites. All activities can have expires_at value set, which controls when they should be deleted automatically.
+- Mastodon API: in post_status, the expires_in parameter lets you set the number of seconds until an activity expires. It must be at least one hour.
+- Mastodon API: all status JSON responses contain a `pleroma.expires_at` item which states when an activity will expire. The value is only shown to the user who created the activity. To everyone else it's empty.
+- Configuration: `ActivityExpiration.enabled` controls whether expired activites will get deleted at the appropriate time. Enabled by default.
 - Conversations: Add Pleroma-specific conversation endpoints and status posting extensions. Run the `bump_all_conversations` task again to create the necessary data.
-- **Breaking:** MRF describe API, which adds support for exposing configuration information about MRF policies to NodeInfo.
-  Custom modules will need to be updated by adding, at the very least, `def describe, do: {:ok, %{}}` to the MRF policy modules.
 - MRF: Support for priming the mediaproxy cache (`Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy`)
 - MRF: Support for excluding specific domains from Transparency.
 - MRF: Support for filtering posts based on who they mention (`Pleroma.Web.ActivityPub.MRF.MentionPolicy`)
-- MRF: Support for filtering posts based on ActivityStreams vocabulary (`Pleroma.Web.ActivityPub.MRF.VocabularyPolicy`)
-- MRF (Simple Policy): Support for wildcard domains.
-- Support for wildcard domains in user domain blocks setting.
-- Configuration: `quarantined_instances` support wildcard domains.
-- Configuration: `federation_incoming_replies_max_depth` option
 - Mastodon API: Support for the [`tagged` filter](https://github.com/tootsuite/mastodon/pull/9755) in [`GET /api/v1/accounts/:id/statuses`](https://docs.joinmastodon.org/api/rest/accounts/#get-api-v1-accounts-id-statuses)
 - Mastodon API, streaming: Add support for passing the token in the `Sec-WebSocket-Protocol` header
 - Mastodon API, extension: Ability to reset avatar, profile banner, and background
+- Mastodon API: Add support for `fields_attributes` API parameter (setting custom fields)
 - Mastodon API: Add support for categories for custom emojis by reusing the group feature. <https://github.com/tootsuite/mastodon/pull/11196>
 - Mastodon API: Add support for muting/unmuting notifications
 - Mastodon API: Add support for the `blocked_by` attribute in the relationship API (`GET /api/v1/accounts/relationships`). <https://github.com/tootsuite/mastodon/pull/10373>
@@ -67,6 +174,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Mastodon API: Add `pleroma.deactivated` to the Account entity
 - Mastodon API: added `/auth/password` endpoint for password reset with rate limit.
 - Mastodon API: /api/v1/accounts/:id/statuses now supports nicknames or user id
+- Mastodon API: Improve support for the user profile custom fields
+- Mastodon API: Add support for `fields_attributes` API parameter (setting custom fields)
+- Mastodon API: Added an endpoint to get multiple statuses by IDs (`GET /api/v1/statuses/?ids[]=1&ids[]=2`)
 - Admin API: Return users' tags when querying reports
 - Admin API: Return avatar and display name when querying users
 - Admin API: Allow querying user by ID
@@ -82,9 +192,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - ActivityPub: Optional signing of ActivityPub object fetches.
 - Admin API: Endpoint for fetching latest user's statuses
 - Pleroma API: Add `/api/v1/pleroma/accounts/confirmation_resend?email=<email>` for resending account confirmation.
-- Relays: Added a task to list relay subscriptions.
+- Pleroma API: Email change endpoint.
-- Mix Tasks: `mix pleroma.database fix_likes_collections`
+- Admin API: Added moderation log
-- Federation: Remove `likes` from objects.
+- Web response cache (currently, enabled for ActivityPub)
+- Reverse Proxy: Do not retry failed requests to limit pressure on the peer
 
 ### Changed
 - Configuration: Filter.AnonymizeFilename added ability to retain file extension with custom text
@@ -92,10 +203,67 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - RichMedia: parsers and their order are configured in `rich_media` config.
 - RichMedia: add the rich media ttl based on image expiration time.
 
+## [1.0.7] - 2019-09-26
+### Fixed
+- Broken federation on Erlang 22 (previous versions of hackney http client were using an option that got deprecated)
+### Changed
+- ActivityPub: The first page in inboxes/outboxes is no longer embedded.
+
+## [1.0.6] - 2019-08-14
+### Fixed
+- MRF: fix use of unserializable keyword lists in describe() implementations
+- ActivityPub S2S: POST requests are now signed with `(request-target)` pseudo-header.
+
+## [1.0.5] - 2019-08-13
+### Fixed
+- Mastodon API: follower/following counters not being nullified, when `hide_follows`/`hide_followers` is set
+- Mastodon API: `muted` in the Status entity, using author's account to determine if the thread was muted
+- Mastodon API: return the actual profile URL in the Account entity's `url` property when appropriate
+- Templates: properly style anchor tags
+- Objects being re-embedded to activities after being updated (e.g faved/reposted). Running 'mix pleroma.database prune_objects' again is advised.
+- Not being able to access the Mastodon FE login page on private instances
+- MRF: ensure that subdomain_match calls are case-insensitive
+- Fix internal server error when using the healthcheck API.
+
+### Added
+- **Breaking:** MRF describe API, which adds support for exposing configuration information about MRF policies to NodeInfo.
+  Custom modules will need to be updated by adding, at the very least, `def describe, do: {:ok, %{}}` to the MRF policy modules.
+- Relays: Added a task to list relay subscriptions.
+- MRF: Support for filtering posts based on ActivityStreams vocabulary (`Pleroma.Web.ActivityPub.MRF.VocabularyPolicy`)
+- MRF (Simple Policy): Support for wildcard domains.
+- Support for wildcard domains in user domain blocks setting.
+- Configuration: `quarantined_instances` support wildcard domains.
+- Mix Tasks: `mix pleroma.database fix_likes_collections`
+- Configuration: `federation_incoming_replies_max_depth` option
+
 ### Removed
-- Emoji: Remove longfox emojis.
+- Federation: Remove `likes` from objects.
-- Remove `Reply-To` header from report emails for admins.
+- **Breaking:** ActivityPub: The `accept_blocks` configuration setting.
-- ActivityPub: The `accept_blocks` configuration setting.
+
+## [1.0.4] - 2019-08-01
+### Fixed
+- Invalid SemVer version generation, when the current branch does not have commits ahead of tag/checked out on a tag
+
+## [1.0.3] - 2019-07-31
+### Security
+- OStatus: eliminate the possibility of a protocol downgrade attack.
+- OStatus: prevent following locked accounts, bypassing the approval process.
+- TwitterAPI: use CommonAPI to handle remote follows instead of OStatus.
+
+## [1.0.2] - 2019-07-28
+### Fixed
+- Not being able to pin unlisted posts
+- Mastodon API: represent poll IDs as strings
+- MediaProxy: fix matching filenames
+- MediaProxy: fix filename encoding
+- Migrations: fix a sporadic migration failure
+- Metadata rendering errors resulting in the entire page being inaccessible
+- Federation/MediaProxy not working with instances that have wrong certificate order
+- ActivityPub S2S: remote user deletions now work the same as local user deletions.
+
+### Changed
+- Configuration: OpenGraph and TwitterCard providers enabled by default
+- Configuration: Filter.AnonymizeFilename added ability to retain file extension with custom text
 
 ## [1.0.1] - 2019-07-14
 ### Security
@@ -186,6 +354,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Configuration: Added `extra_cookie_attrs` for setting non-standard cookie attributes. Defaults to ["SameSite=Lax"] so that remote follows work.
 - Timelines: Messages involving people you have blocked will be excluded from the timeline in all cases instead of just repeats.
 - Admin API: Move the user related API to `api/pleroma/admin/users`
+- Admin API: `POST /api/pleroma/admin/users` will take list of users
 - Pleroma API: Support for emoji tags in `/api/pleroma/emoji` resulting in a breaking API change
 - Mastodon API: Support for `exclude_types`, `limit` and `min_id` in `/api/v1/notifications`
 - Mastodon API: Add `languages` and `registrations` to `/api/v1/instance`

Dockerfile

@@ -1,4 +1,4 @@
-FROM rinpatch/elixir:1.9.0-rc.0-alpine as build
+FROM elixir:1.9-alpine as build
 
 COPY . .
 
@@ -12,7 +12,7 @@ RUN apk add git gcc g++ musl-dev make &&\
 	mkdir release &&\
 	mix release --path release
 
-FROM alpine:latest
+FROM alpine:3.9
 
 ARG HOME=/opt/pleroma
 ARG DATA=/var/lib/pleroma

README.md

@@ -8,7 +8,7 @@ Pleroma is a microblogging server software that can federate (= exchange message
 
 Pleroma is written in Elixir, high-performance and can run on small devices like a Raspberry Pi.
 
-For clients it supports both the [GNU Social API with Qvitter extensions](https://twitter-api.readthedocs.io/en/latest/index.html) and the [Mastodon client API](https://docs.joinmastodon.org/api/guidelines/).
+For clients it supports the [Mastodon client API](https://docs.joinmastodon.org/api/guidelines/) with Pleroma extensions (see "Pleroma's APIs and Mastodon API extensions" section on <https://docs-develop.pleroma.social>).
 
 - [Client Applications for Pleroma](https://docs-develop.pleroma.social/clients.html)
 
@@ -25,7 +25,7 @@ While we don’t provide docker files, other people have written very good ones.
 
 ### Dependencies
 
-* Postgresql version 9.6 or newer
+* Postgresql version 9.6 or newer, including the contrib modules
 * Elixir version 1.7 or newer. If your distribution only has an old version available, check [Elixir’s install page](https://elixir-lang.org/install.html) or use a tool like [asdf](https://github.com/asdf-vm/asdf).
 * Build-essential tools
 
@@ -71,7 +71,7 @@ This is useful for running Pleroma inside Tor or I2P.
 
 ## Customization and contribution
 
-The [Pleroma Documentation](https://docs-develop.pleroma.social/readme.html) offers manuals and guides on how to further customize your instance to your liking and how you can contribute to the project.
+The [Pleroma Documentation](https://docs-develop.pleroma.social) offers manuals and guides on how to further customize your instance to your liking and how you can contribute to the project.
 
 ## Troubleshooting
 

benchmarks/load_testing/fetcher.ex

@@ -0,0 +1,231 @@
+defmodule Pleroma.LoadTesting.Fetcher do
+  use Pleroma.LoadTesting.Helper
+
+  def fetch_user(user) do
+    Benchee.run(%{
+      "By id" => fn -> Repo.get_by(User, id: user.id) end,
+      "By ap_id" => fn -> Repo.get_by(User, ap_id: user.ap_id) end,
+      "By email" => fn -> Repo.get_by(User, email: user.email) end,
+      "By nickname" => fn -> Repo.get_by(User, nickname: user.nickname) end
+    })
+  end
+
+  def query_timelines(user) do
+    home_timeline_params = %{
+      "count" => 20,
+      "with_muted" => true,
+      "type" => ["Create", "Announce"],
+      "blocking_user" => user,
+      "muting_user" => user,
+      "user" => user
+    }
+
+    mastodon_public_timeline_params = %{
+      "count" => 20,
+      "local_only" => true,
+      "only_media" => "false",
+      "type" => ["Create", "Announce"],
+      "with_muted" => "true",
+      "blocking_user" => user,
+      "muting_user" => user
+    }
+
+    mastodon_federated_timeline_params = %{
+      "count" => 20,
+      "only_media" => "false",
+      "type" => ["Create", "Announce"],
+      "with_muted" => "true",
+      "blocking_user" => user,
+      "muting_user" => user
+    }
+
+    following = User.following(user)
+
+    Benchee.run(%{
+      "User home timeline" => fn ->
+        Pleroma.Web.ActivityPub.ActivityPub.fetch_activities(
+          following,
+          home_timeline_params
+        )
+      end,
+      "User mastodon public timeline" => fn ->
+        Pleroma.Web.ActivityPub.ActivityPub.fetch_public_activities(
+          mastodon_public_timeline_params
+        )
+      end,
+      "User mastodon federated public timeline" => fn ->
+        Pleroma.Web.ActivityPub.ActivityPub.fetch_public_activities(
+          mastodon_federated_timeline_params
+        )
+      end
+    })
+
+    home_activities =
+      Pleroma.Web.ActivityPub.ActivityPub.fetch_activities(
+        following,
+        home_timeline_params
+      )
+
+    public_activities =
+      Pleroma.Web.ActivityPub.ActivityPub.fetch_public_activities(mastodon_public_timeline_params)
+
+    public_federated_activities =
+      Pleroma.Web.ActivityPub.ActivityPub.fetch_public_activities(
+        mastodon_federated_timeline_params
+      )
+
+    Benchee.run(%{
+      "Rendering home timeline" => fn ->
+        Pleroma.Web.MastodonAPI.StatusView.render("index.json", %{
+          activities: home_activities,
+          for: user,
+          as: :activity
+        })
+      end,
+      "Rendering public timeline" => fn ->
+        Pleroma.Web.MastodonAPI.StatusView.render("index.json", %{
+          activities: public_activities,
+          for: user,
+          as: :activity
+        })
+      end,
+      "Rendering public federated timeline" => fn ->
+        Pleroma.Web.MastodonAPI.StatusView.render("index.json", %{
+          activities: public_federated_activities,
+          for: user,
+          as: :activity
+        })
+      end
+    })
+  end
+
+  def query_notifications(user) do
+    without_muted_params = %{"count" => "20", "with_muted" => "false"}
+    with_muted_params = %{"count" => "20", "with_muted" => "true"}
+
+    Benchee.run(%{
+      "Notifications without muted" => fn ->
+        Pleroma.Web.MastodonAPI.MastodonAPI.get_notifications(user, without_muted_params)
+      end,
+      "Notifications with muted" => fn ->
+        Pleroma.Web.MastodonAPI.MastodonAPI.get_notifications(user, with_muted_params)
+      end
+    })
+
+    without_muted_notifications =
+      Pleroma.Web.MastodonAPI.MastodonAPI.get_notifications(user, without_muted_params)
+
+    with_muted_notifications =
+      Pleroma.Web.MastodonAPI.MastodonAPI.get_notifications(user, with_muted_params)
+
+    Benchee.run(%{
+      "Render notifications without muted" => fn ->
+        Pleroma.Web.MastodonAPI.NotificationView.render("index.json", %{
+          notifications: without_muted_notifications,
+          for: user
+        })
+      end,
+      "Render notifications with muted" => fn ->
+        Pleroma.Web.MastodonAPI.NotificationView.render("index.json", %{
+          notifications: with_muted_notifications,
+          for: user
+        })
+      end
+    })
+  end
+
+  def query_dms(user) do
+    params = %{
+      "count" => "20",
+      "with_muted" => "true",
+      "type" => "Create",
+      "blocking_user" => user,
+      "user" => user,
+      visibility: "direct"
+    }
+
+    Benchee.run(%{
+      "Direct messages with muted" => fn ->
+        Pleroma.Web.ActivityPub.ActivityPub.fetch_activities_query([user.ap_id], params)
+        |> Pleroma.Pagination.fetch_paginated(params)
+      end,
+      "Direct messages without muted" => fn ->
+        Pleroma.Web.ActivityPub.ActivityPub.fetch_activities_query([user.ap_id], params)
+        |> Pleroma.Pagination.fetch_paginated(Map.put(params, "with_muted", false))
+      end
+    })
+
+    dms_with_muted =
+      Pleroma.Web.ActivityPub.ActivityPub.fetch_activities_query([user.ap_id], params)
+      |> Pleroma.Pagination.fetch_paginated(params)
+
+    dms_without_muted =
+      Pleroma.Web.ActivityPub.ActivityPub.fetch_activities_query([user.ap_id], params)
+      |> Pleroma.Pagination.fetch_paginated(Map.put(params, "with_muted", false))
+
+    Benchee.run(%{
+      "Rendering dms with muted" => fn ->
+        Pleroma.Web.MastodonAPI.StatusView.render("index.json", %{
+          activities: dms_with_muted,
+          for: user,
+          as: :activity
+        })
+      end,
+      "Rendering dms without muted" => fn ->
+        Pleroma.Web.MastodonAPI.StatusView.render("index.json", %{
+          activities: dms_without_muted,
+          for: user,
+          as: :activity
+        })
+      end
+    })
+  end
+
+  def query_long_thread(user, activity) do
+    Benchee.run(%{
+      "Fetch main post" => fn ->
+        Pleroma.Activity.get_by_id_with_object(activity.id)
+      end,
+      "Fetch context of main post" => fn ->
+        Pleroma.Web.ActivityPub.ActivityPub.fetch_activities_for_context(
+          activity.data["context"],
+          %{
+            "blocking_user" => user,
+            "user" => user,
+            "exclude_id" => activity.id
+          }
+        )
+      end
+    })
+
+    activity = Pleroma.Activity.get_by_id_with_object(activity.id)
+
+    context =
+      Pleroma.Web.ActivityPub.ActivityPub.fetch_activities_for_context(
+        activity.data["context"],
+        %{
+          "blocking_user" => user,
+          "user" => user,
+          "exclude_id" => activity.id
+        }
+      )
+
+    Benchee.run(%{
+      "Render status" => fn ->
+        Pleroma.Web.MastodonAPI.StatusView.render("show.json", %{
+          activity: activity,
+          for: user
+        })
+      end,
+      "Render context" => fn ->
+        Pleroma.Web.MastodonAPI.StatusView.render(
+          "index.json",
+          for: user,
+          activities: context,
+          as: :activity
+        )
+        |> Enum.reverse()
+      end
+    })
+  end
+end

benchmarks/load_testing/generator.ex

@@ -0,0 +1,350 @@
+defmodule Pleroma.LoadTesting.Generator do
+  use Pleroma.LoadTesting.Helper
+  alias Pleroma.Web.CommonAPI
+
+  def generate_users(opts) do
+    IO.puts("Starting generating #{opts[:users_max]} users...")
+    {time, _} = :timer.tc(fn -> do_generate_users(opts) end)
+
+    IO.puts("Inserting users take #{to_sec(time)} sec.\n")
+  end
+
+  defp do_generate_users(opts) do
+    max = Keyword.get(opts, :users_max)
+
+    Task.async_stream(
+      1..max,
+      &generate_user_data(&1),
+      max_concurrency: 10,
+      timeout: 30_000
+    )
+    |> Enum.to_list()
+  end
+
+  defp generate_user_data(i) do
+    remote = Enum.random([true, false])
+
+    user = %User{
+      name: "Test テスト User #{i}",
+      email: "user#{i}@example.com",
+      nickname: "nick#{i}",
+      password_hash:
+        "$pbkdf2-sha512$160000$bU.OSFI7H/yqWb5DPEqyjw$uKp/2rmXw12QqnRRTqTtuk2DTwZfF8VR4MYW2xMeIlqPR/UX1nT1CEKVUx2CowFMZ5JON8aDvURrZpJjSgqXrg",
+      bio: "Tester Number #{i}",
+      info: %{},
+      local: remote
+    }
+
+    user_urls =
+      if remote do
+        base_url =
+          Enum.random(["https://domain1.com", "https://domain2.com", "https://domain3.com"])
+
+        ap_id = "#{base_url}/users/#{user.nickname}"
+
+        %{
+          ap_id: ap_id,
+          follower_address: ap_id <> "/followers",
+          following_address: ap_id <> "/following"
+        }
+      else
+        %{
+          ap_id: User.ap_id(user),
+          follower_address: User.ap_followers(user),
+          following_address: User.ap_following(user)
+        }
+      end
+
+    user = Map.merge(user, user_urls)
+
+    Repo.insert!(user)
+  end
+
+  def generate_activities(user, users) do
+    do_generate_activities(user, users)
+  end
+
+  defp do_generate_activities(user, users) do
+    IO.puts("Starting generating 20000 common activities...")
+
+    {time, _} =
+      :timer.tc(fn ->
+        Task.async_stream(
+          1..20_000,
+          fn _ ->
+            do_generate_activity([user | users])
+          end,
+          max_concurrency: 10,
+          timeout: 30_000
+        )
+        |> Stream.run()
+      end)
+
+    IO.puts("Inserting common activities take #{to_sec(time)} sec.\n")
+
+    IO.puts("Starting generating 20000 activities with mentions...")
+
+    {time, _} =
+      :timer.tc(fn ->
+        Task.async_stream(
+          1..20_000,
+          fn _ ->
+            do_generate_activity_with_mention(user, users)
+          end,
+          max_concurrency: 10,
+          timeout: 30_000
+        )
+        |> Stream.run()
+      end)
+
+    IO.puts("Inserting activities with menthions take #{to_sec(time)} sec.\n")
+
+    IO.puts("Starting generating 10000 activities with threads...")
+
+    {time, _} =
+      :timer.tc(fn ->
+        Task.async_stream(
+          1..10_000,
+          fn _ ->
+            do_generate_threads([user | users])
+          end,
+          max_concurrency: 10,
+          timeout: 30_000
+        )
+        |> Stream.run()
+      end)
+
+    IO.puts("Inserting activities with threads take #{to_sec(time)} sec.\n")
+  end
+
+  defp do_generate_activity(users) do
+    post = %{
+      "status" => "Some status without mention with random user"
+    }
+
+    CommonAPI.post(Enum.random(users), post)
+  end
+
+  defp do_generate_activity_with_mention(user, users) do
+    mentions_cnt = Enum.random([2, 3, 4, 5])
+    with_user = Enum.random([true, false])
+    users = Enum.shuffle(users)
+    mentions_users = Enum.take(users, mentions_cnt)
+    mentions_users = if with_user, do: [user | mentions_users], else: mentions_users
+
+    mentions_str =
+      Enum.map(mentions_users, fn user -> "@" <> user.nickname end) |> Enum.join(", ")
+
+    post = %{
+      "status" => mentions_str <> "some status with mentions random users"
+    }
+
+    CommonAPI.post(Enum.random(users), post)
+  end
+
+  defp do_generate_threads(users) do
+    thread_length = Enum.random([2, 3, 4, 5])
+    actor = Enum.random(users)
+
+    post = %{
+      "status" => "Start of the thread"
+    }
+
+    {:ok, activity} = CommonAPI.post(actor, post)
+
+    Enum.each(1..thread_length, fn _ ->
+      user = Enum.random(users)
+
+      post = %{
+        "status" => "@#{actor.nickname} reply to thread",
+        "in_reply_to_status_id" => activity.id
+      }
+
+      CommonAPI.post(user, post)
+    end)
+  end
+
+  def generate_remote_activities(user, users) do
+    do_generate_remote_activities(user, users)
+  end
+
+  defp do_generate_remote_activities(user, users) do
+    IO.puts("Starting generating 10000 remote activities...")
+
+    {time, _} =
+      :timer.tc(fn ->
+        Task.async_stream(
+          1..10_000,
+          fn i ->
+            do_generate_remote_activity(i, user, users)
+          end,
+          max_concurrency: 10,
+          timeout: 30_000
+        )
+        |> Stream.run()
+      end)
+
+    IO.puts("Inserting remote activities take #{to_sec(time)} sec.\n")
+  end
+
+  defp do_generate_remote_activity(i, user, users) do
+    actor = Enum.random(users)
+    %{host: host} = URI.parse(actor.ap_id)
+    date = Date.utc_today()
+    datetime = DateTime.utc_now()
+
+    map = %{
+      "actor" => actor.ap_id,
+      "cc" => [actor.follower_address, user.ap_id],
+      "context" => "tag:mastodon.example.org,#{date}:objectId=#{i}:objectType=Conversation",
+      "id" => actor.ap_id <> "/statuses/#{i}/activity",
+      "object" => %{
+        "actor" => actor.ap_id,
+        "atomUri" => actor.ap_id <> "/statuses/#{i}",
+        "attachment" => [],
+        "attributedTo" => actor.ap_id,
+        "bcc" => [],
+        "bto" => [],
+        "cc" => [actor.follower_address, user.ap_id],
+        "content" =>
+          "<p><span class=\"h-card\"><a href=\"" <>
+            user.ap_id <>
+            "\" class=\"u-url mention\">@<span>" <> user.nickname <> "</span></a></span></p>",
+        "context" => "tag:mastodon.example.org,#{date}:objectId=#{i}:objectType=Conversation",
+        "conversation" =>
+          "tag:mastodon.example.org,#{date}:objectId=#{i}:objectType=Conversation",
+        "emoji" => %{},
+        "id" => actor.ap_id <> "/statuses/#{i}",
+        "inReplyTo" => nil,
+        "inReplyToAtomUri" => nil,
+        "published" => datetime,
+        "sensitive" => true,
+        "summary" => "cw",
+        "tag" => [
+          %{
+            "href" => user.ap_id,
+            "name" => "@#{user.nickname}@#{host}",
+            "type" => "Mention"
+          }
+        ],
+        "to" => ["https://www.w3.org/ns/activitystreams#Public"],
+        "type" => "Note",
+        "url" => "http://#{host}/@#{actor.nickname}/#{i}"
+      },
+      "published" => datetime,
+      "to" => ["https://www.w3.org/ns/activitystreams#Public"],
+      "type" => "Create"
+    }
+
+    Pleroma.Web.ActivityPub.ActivityPub.insert(map, false)
+  end
+
+  def generate_dms(user, users, opts) do
+    IO.puts("Starting generating #{opts[:dms_max]} DMs")
+    {time, _} = :timer.tc(fn -> do_generate_dms(user, users, opts) end)
+    IO.puts("Inserting dms take #{to_sec(time)} sec.\n")
+  end
+
+  defp do_generate_dms(user, users, opts) do
+    Task.async_stream(
+      1..opts[:dms_max],
+      fn _ ->
+        do_generate_dm(user, users)
+      end,
+      max_concurrency: 10,
+      timeout: 30_000
+    )
+    |> Stream.run()
+  end
+
+  defp do_generate_dm(user, users) do
+    post = %{
+      "status" => "@#{user.nickname} some direct message",
+      "visibility" => "direct"
+    }
+
+    CommonAPI.post(Enum.random(users), post)
+  end
+
+  def generate_long_thread(user, users, opts) do
+    IO.puts("Starting generating long thread with #{opts[:thread_length]} replies")
+    {time, activity} = :timer.tc(fn -> do_generate_long_thread(user, users, opts) end)
+    IO.puts("Inserting long thread replies take #{to_sec(time)} sec.\n")
+    {:ok, activity}
+  end
+
+  defp do_generate_long_thread(user, users, opts) do
+    {:ok, %{id: id} = activity} = CommonAPI.post(user, %{"status" => "Start of long thread"})
+
+    Task.async_stream(
+      1..opts[:thread_length],
+      fn _ -> do_generate_thread(users, id) end,
+      max_concurrency: 10,
+      timeout: 30_000
+    )
+    |> Stream.run()
+
+    activity
+  end
+
+  defp do_generate_thread(users, activity_id) do
+    CommonAPI.post(Enum.random(users), %{
+      "status" => "reply to main post",
+      "in_reply_to_status_id" => activity_id
+    })
+  end
+
+  def generate_non_visible_message(user, users) do
+    IO.puts("Starting generating 1000 non visible posts")
+
+    {time, _} =
+      :timer.tc(fn ->
+        do_generate_non_visible_posts(user, users)
+      end)
+
+    IO.puts("Inserting non visible posts take #{to_sec(time)} sec.\n")
+  end
+
+  defp do_generate_non_visible_posts(user, users) do
+    [not_friend | users] = users
+
+    make_friends(user, users)
+
+    Task.async_stream(1..1000, fn _ -> do_generate_non_visible_post(not_friend, users) end,
+      max_concurrency: 10,
+      timeout: 30_000
+    )
+    |> Stream.run()
+  end
+
+  defp make_friends(_user, []), do: nil
+
+  defp make_friends(user, [friend | users]) do
+    {:ok, _} = User.follow(user, friend)
+    {:ok, _} = User.follow(friend, user)
+    make_friends(user, users)
+  end
+
+  defp do_generate_non_visible_post(not_friend, users) do
+    post = %{
+      "status" => "some non visible post",
+      "visibility" => "private"
+    }
+
+    {:ok, activity} = CommonAPI.post(not_friend, post)
+
+    thread_length = Enum.random([2, 3, 4, 5])
+
+    Enum.each(1..thread_length, fn _ ->
+      user = Enum.random(users)
+
+      post = %{
+        "status" => "@#{not_friend.nickname} reply to non visible post",
+        "in_reply_to_status_id" => activity.id,
+        "visibility" => "private"
+      }
+
+      CommonAPI.post(user, post)
+    end)
+  end
+end

benchmarks/load_testing/helper.ex

@@ -0,0 +1,11 @@
+defmodule Pleroma.LoadTesting.Helper do
+  defmacro __using__(_) do
+    quote do
+      import Ecto.Query
+      alias Pleroma.Repo
+      alias Pleroma.User
+
+      defp to_sec(microseconds), do: microseconds / 1_000_000
+    end
+  end
+end

benchmarks/mix/tasks/pleroma/load_testing.ex

@@ -0,0 +1,134 @@
+defmodule Mix.Tasks.Pleroma.LoadTesting do
+  use Mix.Task
+  use Pleroma.LoadTesting.Helper
+  import Mix.Pleroma
+  import Pleroma.LoadTesting.Generator
+  import Pleroma.LoadTesting.Fetcher
+
+  @shortdoc "Factory for generation data"
+  @moduledoc """
+  Generates data like:
+  - local/remote users
+  - local/remote activities with notifications
+  - direct messages
+  - long thread
+  - non visible posts
+
+  ## Generate data
+      MIX_ENV=benchmark mix pleroma.load_testing --users 20000 --dms 20000 --thread_length 2000
+      MIX_ENV=benchmark mix pleroma.load_testing -u 20000 -d 20000 -t 2000
+
+  Options:
+  - `--users NUMBER` - number of users to generate. Defaults to: 20000. Alias: `-u`
+  - `--dms NUMBER` - number of direct messages to generate. Defaults to: 20000. Alias `-d`
+  - `--thread_length` - number of messages in thread. Defaults to: 2000. ALias `-t`
+  """
+
+  @aliases [u: :users, d: :dms, t: :thread_length]
+  @switches [
+    users: :integer,
+    dms: :integer,
+    thread_length: :integer
+  ]
+  @users_default 20_000
+  @dms_default 1_000
+  @thread_length_default 2_000
+
+  def run(args) do
+    start_pleroma()
+    Pleroma.Config.put([:instance, :skip_thread_containment], true)
+    {opts, _} = OptionParser.parse!(args, strict: @switches, aliases: @aliases)
+
+    users_max = Keyword.get(opts, :users, @users_default)
+    dms_max = Keyword.get(opts, :dms, @dms_default)
+    thread_length = Keyword.get(opts, :thread_length, @thread_length_default)
+
+    clean_tables()
+
+    opts =
+      Keyword.put(opts, :users_max, users_max)
+      |> Keyword.put(:dms_max, dms_max)
+      |> Keyword.put(:thread_length, thread_length)
+
+    generate_users(opts)
+
+    # main user for queries
+    IO.puts("Fetching local main user...")
+
+    {time, user} =
+      :timer.tc(fn ->
+        Repo.one(
+          from(u in User, where: u.local == true, order_by: fragment("RANDOM()"), limit: 1)
+        )
+      end)
+
+    IO.puts("Fetching main user take #{to_sec(time)} sec.\n")
+
+    IO.puts("Fetching local users...")
+
+    {time, users} =
+      :timer.tc(fn ->
+        Repo.all(
+          from(u in User,
+            where: u.id != ^user.id,
+            where: u.local == true,
+            order_by: fragment("RANDOM()"),
+            limit: 10
+          )
+        )
+      end)
+
+    IO.puts("Fetching local users take #{to_sec(time)} sec.\n")
+
+    IO.puts("Fetching remote users...")
+
+    {time, remote_users} =
+      :timer.tc(fn ->
+        Repo.all(
+          from(u in User,
+            where: u.id != ^user.id,
+            where: u.local == false,
+            order_by: fragment("RANDOM()"),
+            limit: 10
+          )
+        )
+      end)
+
+    IO.puts("Fetching remote users take #{to_sec(time)} sec.\n")
+
+    generate_activities(user, users)
+
+    generate_remote_activities(user, remote_users)
+
+    generate_dms(user, users, opts)
+
+    {:ok, activity} = generate_long_thread(user, users, opts)
+
+    generate_non_visible_message(user, users)
+
+    IO.puts("Users in DB: #{Repo.aggregate(from(u in User), :count, :id)}")
+
+    IO.puts("Activities in DB: #{Repo.aggregate(from(a in Pleroma.Activity), :count, :id)}")
+
+    IO.puts("Objects in DB: #{Repo.aggregate(from(o in Pleroma.Object), :count, :id)}")
+
+    IO.puts(
+      "Notifications in DB: #{Repo.aggregate(from(n in Pleroma.Notification), :count, :id)}"
+    )
+
+    fetch_user(user)
+    query_timelines(user)
+    query_notifications(user)
+    query_dms(user)
+    query_long_thread(user, activity)
+    Pleroma.Config.put([:instance, :skip_thread_containment], false)
+    query_timelines(user)
+  end
+
+  defp clean_tables do
+    IO.puts("Deleting old data...\n")
+    Ecto.Adapters.SQL.query!(Repo, "TRUNCATE users CASCADE;")
+    Ecto.Adapters.SQL.query!(Repo, "TRUNCATE activities CASCADE;")
+    Ecto.Adapters.SQL.query!(Repo, "TRUNCATE objects CASCADE;")
+  end
+end

config/benchmark.exs

@@ -0,0 +1,84 @@
+use Mix.Config
+
+# We don't run a server during test. If one is required,
+# you can enable the server option below.
+config :pleroma, Pleroma.Web.Endpoint,
+  http: [port: 4001],
+  url: [port: 4001],
+  server: true
+
+# Disable captha for tests
+config :pleroma, Pleroma.Captcha,
+  # It should not be enabled for automatic tests
+  enabled: false,
+  # A fake captcha service for tests
+  method: Pleroma.Captcha.Mock
+
+# Print only warnings and errors during test
+config :logger, level: :warn
+
+config :pleroma, :auth, oauth_consumer_strategies: []
+
+config :pleroma, Pleroma.Upload, filters: [], link_name: false
+
+config :pleroma, Pleroma.Uploaders.Local, uploads: "test/uploads"
+
+config :pleroma, Pleroma.Emails.Mailer, adapter: Swoosh.Adapters.Test, enabled: true
+
+config :pleroma, :instance,
+  email: "admin@example.com",
+  notify_email: "noreply@example.com",
+  skip_thread_containment: false,
+  federating: false,
+  external_user_synchronization: false
+
+config :pleroma, :activitypub, sign_object_fetches: false
+
+# Configure your database
+config :pleroma, Pleroma.Repo,
+  adapter: Ecto.Adapters.Postgres,
+  username: "postgres",
+  password: "postgres",
+  database: "pleroma_test",
+  hostname: System.get_env("DB_HOST") || "localhost",
+  pool_size: 10
+
+# Reduce hash rounds for testing
+config :pbkdf2_elixir, rounds: 1
+
+config :tesla, adapter: Tesla.Mock
+
+config :pleroma, :rich_media,
+  enabled: false,
+  ignore_hosts: [],
+  ignore_tld: ["local", "localdomain", "lan"]
+
+config :web_push_encryption, :vapid_details,
+  subject: "mailto:administrator@example.com",
+  public_key:
+    "BLH1qVhJItRGCfxgTtONfsOKDc9VRAraXw-3NsmjMngWSh7NxOizN6bkuRA7iLTMPS82PjwJAr3UoK9EC1IFrz4",
+  private_key: "_-XZ0iebPrRfZ_o0-IatTdszYa8VCH1yLN-JauK7HHA"
+
+config :web_push_encryption, :http_client, Pleroma.Web.WebPushHttpClientMock
+
+config :pleroma_job_queue, disabled: true
+
+config :pleroma, Pleroma.ScheduledActivity,
+  daily_user_limit: 2,
+  total_user_limit: 3,
+  enabled: false
+
+config :pleroma, :rate_limit,
+  search: [{1000, 30}, {1000, 30}],
+  app_account_creation: {10_000, 5},
+  password_reset: {1000, 30}
+
+config :pleroma, :http_security, report_uri: "https://endpoint.com"
+
+config :pleroma, :http, send_user_agent: false
+
+rum_enabled = System.get_env("RUM_ENABLED") == "true"
+config :pleroma, :database, rum_enabled: rum_enabled
+IO.puts("RUM enabled: #{rum_enabled}")
+
+config :pleroma, Pleroma.ReverseProxy.Client, Pleroma.ReverseProxy.ClientMock

config/config.exs

@@ -51,6 +51,20 @@ config :pleroma, Pleroma.Repo,
   telemetry_event: [Pleroma.Repo.Instrumenter],
   migration_lock: nil
 
+scheduled_jobs =
+  with digest_config <- Application.get_env(:pleroma, :email_notifications)[:digest],
+       true <- digest_config[:active] do
+    [{digest_config[:schedule], {Pleroma.Daemons.DigestEmailDaemon, :perform, []}}]
+  else
+    _ -> []
+  end
+
+config :pleroma, Pleroma.Scheduler,
+  global: true,
+  overlap: true,
+  timezone: :utc,
+  jobs: scheduled_jobs
+
 config :pleroma, Pleroma.Captcha,
   enabled: false,
   seconds_valid: 60,
@@ -91,6 +105,7 @@ config :pleroma, Pleroma.Uploaders.Local, uploads: "uploads"
 
 config :pleroma, Pleroma.Uploaders.S3,
   bucket: nil,
+  streaming_enabled: true,
   public_endpoint: "https://s3.amazonaws.com"
 
 config :pleroma, Pleroma.Uploaders.MDII,
@@ -104,7 +119,8 @@ config :pleroma, :emoji,
     # Put groups that have higher priority than defaults here. Example in `docs/config/custom_emoji.md`
     Custom: ["/emoji/*.png", "/emoji/**/*.png"]
   ],
-  default_manifest: "https://git.pleroma.social/pleroma/emoji-index/raw/master/index.json"
+  default_manifest: "https://git.pleroma.social/pleroma/emoji-index/raw/master/index.json",
+  shared_pack_cache_seconds_per_file: 60
 
 config :pleroma, :uri_schemes,
   valid_schemes: [
@@ -223,9 +239,7 @@ config :pleroma, :instance,
   federation_incoming_replies_max_depth: 100,
   federation_reachability_timeout_days: 7,
   federation_publisher_modules: [
-    Pleroma.Web.ActivityPub.Publisher,
+    Pleroma.Web.ActivityPub.Publisher
-    Pleroma.Web.Websub,
-    Pleroma.Web.Salmon
   ],
   allow_relay: true,
   rewrite_policy: [
@@ -257,7 +271,12 @@ config :pleroma, :instance,
   dynamic_configuration: false,
   user_bio_length: 5000,
   user_name_length: 100,
-  external_user_synchronization: true
+  max_account_fields: 10,
+  max_remote_account_fields: 20,
+  account_field_name_length: 512,
+  account_field_value_length: 2048,
+  external_user_synchronization: true,
+  extended_nickname_format: false
 
 config :pleroma, :markup,
   # XXX - unfortunately, inline images must be enabled by default right now, because
@@ -267,8 +286,8 @@ config :pleroma, :markup,
   allow_tables: false,
   allow_fonts: false,
   scrub_policy: [
-    Pleroma.HTML.Transform.MediaProxy,
+    Pleroma.HTML.Scrubber.Default,
-    Pleroma.HTML.Scrubber.Default
+    Pleroma.HTML.Transform.MediaProxy
   ]
 
 config :pleroma, :frontend_configurations,
@@ -307,12 +326,26 @@ config :pleroma, :assets,
   ],
   default_mascot: :pleroma_fox_tan
 
+config :pleroma, :manifest,
+  icons: [
+    %{
+      src: "/static/logo.png",
+      type: "image/png"
+    }
+  ],
+  theme_color: "#282c37",
+  background_color: "#191b22"
+
 config :pleroma, :activitypub,
   unfollow_blocked: true,
   outgoing_blocks: true,
   follow_handshake_timeout: 500,
   sign_object_fetches: true
 
+config :pleroma, :streamer,
+  workers: 3,
+  overflow_workers: 2
+
 config :pleroma, :user, deny_follow_blocked: true
 
 config :pleroma, :mrf_normalize_markup, scrub_policy: Pleroma.HTML.Scrubber.Default
@@ -382,6 +415,8 @@ config :pleroma, :chat, enabled: false
 
 config :phoenix, :format_encoders, json: Jason
 
+config :phoenix, :json_library, Jason
+
 config :pleroma, :gopher,
   enabled: false,
   ip: {0, 0, 0, 0},
@@ -391,7 +426,8 @@ config :pleroma, Pleroma.Web.Metadata,
   providers: [
     Pleroma.Web.Metadata.Providers.OpenGraph,
     Pleroma.Web.Metadata.Providers.TwitterCard,
-    Pleroma.Web.Metadata.Providers.RelMe
+    Pleroma.Web.Metadata.Providers.RelMe,
+    Pleroma.Web.Metadata.Providers.Feed
   ],
   unfurl_nsfw: false
 
@@ -458,13 +494,12 @@ config :pleroma, Pleroma.User,
     "web"
   ]
 
-config :pleroma, Pleroma.Web.Federator.RetryQueue,
+config :pleroma, Oban,
-  enabled: false,
+  repo: Pleroma.Repo,
-  max_jobs: 20,
+  verbose: false,
-  initial_timeout: 30,
+  prune: {:maxlen, 1500},
-  max_retries: 5
+  queues: [
-
+    activity_expiration: 10,
-config :pleroma_job_queue, :queues,
     federator_incoming: 50,
     federator_outgoing: 50,
     web_push: 50,
@@ -472,6 +507,13 @@ config :pleroma_job_queue, :queues,
     transmogrifier: 20,
     scheduled_activities: 10,
     background: 5
+  ]
+
+config :pleroma, :workers,
+  retries: [
+    federator_incoming: 5,
+    federator_outgoing: 5
+  ]
 
 config :pleroma, :fetch_initial_posts,
   enabled: false,
@@ -486,7 +528,7 @@ config :auto_linker,
     class: false,
     strip_prefix: false,
     new_window: false,
-    rel: false
+    rel: "ugc"
   ]
 
 config :pleroma, :ldap,
@@ -525,6 +567,17 @@ config :pleroma, :auth, oauth_consumer_strategies: oauth_consumer_strategies
 
 config :pleroma, Pleroma.Emails.Mailer, adapter: Swoosh.Adapters.Sendmail, enabled: false
 
+config :pleroma, Pleroma.Emails.UserEmail,
+  logo: nil,
+  styling: %{
+    link_color: "#d8a070",
+    background_color: "#2C3645",
+    content_background_color: "#1B2635",
+    header_color: "#d8a070",
+    text_color: "#b9b9ba",
+    text_muted_color: "#b9b9ba"
+  }
+
 config :prometheus, Pleroma.Web.Endpoint.MetricsExporter, path: "/api/pleroma/app_metrics"
 
 config :pleroma, Pleroma.ScheduledActivity,
@@ -553,16 +606,17 @@ config :pleroma, :env, Mix.env()
 config :http_signatures,
   adapter: Pleroma.Signature
 
-config :pleroma, :rate_limit,
+config :pleroma, :rate_limit, authentication: {60_000, 15}
-  search: [{1000, 10}, {1000, 30}],
-  app_account_creation: {1_800_000, 25},
-  relations_actions: {10_000, 10},
-  relation_id_action: {60_000, 2},
-  statuses_actions: {10_000, 15},
-  status_id_action: {60_000, 3},
-  password_reset: {1_800_000, 5},
-  account_confirmation_resend: {8_640_000, 5}
 
+config :pleroma, Pleroma.ActivityExpiration, enabled: true
+
+config :pleroma, Pleroma.Plugs.RemoteIp, enabled: false
+
+config :pleroma, :web_cache_ttl,
+  activity_pub: nil,
+  activity_pub_question: 30_000
+
+config :swarm, node_blacklist: [~r/myhtmlex_.*$/]
 # Import environment specific config. This must remain at the bottom
 # of this file so it overrides the configuration defined above.
 import_config "#{Mix.env()}.exs"

config/docker.exs

@@ -10,7 +10,7 @@ config :pleroma, :instance,
   notify_email: System.get_env("NOTIFY_EMAIL"),
   limit: 5000,
   registrations_open: false,
-  dynamic_configuration: true
+  healthcheck: true
 
 config :pleroma, Pleroma.Repo,
   adapter: Ecto.Adapters.Postgres,

config/releases.exs

@@ -1,6 +1,6 @@
 import Config
 
-config :pleroma, :instance, static_dir: "/var/lib/pleroma/static"
+config :pleroma, :instance, static: "/var/lib/pleroma/static"
 config :pleroma, Pleroma.Uploaders.Local, uploads: "/var/lib/pleroma/uploads"
 
 config_path = System.get_env("PLEROMA_CONFIG_PATH") || "/etc/pleroma/config.exs"

config/test.exs

@@ -30,7 +30,8 @@ config :pleroma, :instance,
   notify_email: "noreply@example.com",
   skip_thread_containment: false,
   federating: false,
-  external_user_synchronization: false
+  external_user_synchronization: false,
+  static_dir: "test/instance_static/"
 
 config :pleroma, :activitypub, sign_object_fetches: false
 
@@ -61,7 +62,11 @@ config :web_push_encryption, :vapid_details,
 
 config :web_push_encryption, :http_client, Pleroma.Web.WebPushHttpClientMock
 
-config :pleroma_job_queue, disabled: true
+config :pleroma, Oban,
+  queues: false,
+  prune: :disabled
+
+config :pleroma, Pleroma.Scheduler, jobs: []
 
 config :pleroma, Pleroma.ScheduledActivity,
   daily_user_limit: 2,
@@ -71,7 +76,8 @@ config :pleroma, Pleroma.ScheduledActivity,
 config :pleroma, :rate_limit,
   search: [{1000, 30}, {1000, 30}],
   app_account_creation: {10_000, 5},
-  password_reset: {1000, 30}
+  password_reset: {1000, 30},
+  ap_routes: nil
 
 config :pleroma, :http_security, report_uri: "https://endpoint.com"
 
@@ -85,10 +91,9 @@ config :joken, default_signer: "yU8uHKq+yyAkZ11Hx//jcdacWc8yQ1bxAAGrplzB0Zwwjkp3
 
 config :pleroma, Pleroma.ReverseProxy.Client, Pleroma.ReverseProxy.ClientMock
 
-try do
+if File.exists?("./config/test.secret.exs") do
   import_config "test.secret.exs"
-rescue
+else
-  _ ->
   IO.puts(
     "You may want to create test.secret.exs to declare custom database connection parameters."
   )

docs/API/admin_api.md

@@ -47,7 +47,7 @@ Authentication is required and the user must be an admin.
 }
 ```
 
-## `/api/pleroma/admin/users`
+## DEPRECATED `DELETE /api/pleroma/admin/users`
 
 ### Remove a user
 
@@ -56,13 +56,26 @@ Authentication is required and the user must be an admin.
   - `nickname`
 - Response: User’s nickname
 
+## `DELETE /api/pleroma/admin/users`
+
+### Remove a user
+
+- Method `DELETE`
+- Params:
+  - `nicknames`
+- Response: Array of user nicknames
+
 ### Create a user
 
 - Method: `POST`
 - Params:
-  - `nickname`
+  `users`: [
-  - `email`
+    {
-  - `password`
+      `nickname`,
+      `email`,
+      `password`
+    }
+  ]
 - Response: User’s nickname
 
 ## `/api/pleroma/admin/users/follow`
@@ -150,28 +163,86 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret
 }
 ```
 
-### Add user in permission group
+## DEPRECATED `POST /api/pleroma/admin/users/:nickname/permission_group/:permission_group`
+
+### Add user to permission group
 
-- Method: `POST`
 - Params: none
 - Response:
   - On failure: `{"error": "…"}`
-  - On success: JSON of the `user.info`
+  - On success: JSON of the user
+
+## `POST /api/pleroma/admin/users/permission_group/:permission_group`
+
+### Add users to permission group
+
+- Params:
+  - `nicknames`: nicknames array
+- Response:
+  - On failure: `{"error": "…"}`
+  - On success: JSON of the user
+
+## DEPRECATED `DELETE /api/pleroma/admin/users/:nickname/permission_group/:permission_group`
 
 ### Remove user from permission group
 
-- Method: `DELETE`
 - Params: none
 - Response:
   - On failure: `{"error": "…"}`
-  - On success: JSON of the `user.info`
+  - On success: JSON of the user
 - Note: An admin cannot revoke their own admin status.
 
-## `/api/pleroma/admin/users/:nickname/activation_status`
+## `DELETE /api/pleroma/admin/users/permission_group/:permission_group`
+
+### Remove users from permission group
+
+- Params:
+  - `nicknames`: nicknames array
+- Response:
+  - On failure: `{"error": "…"}`
+  - On success: JSON of the user
+- Note: An admin cannot revoke their own admin status.
+
+## `PATCH /api/pleroma/admin/users/activate`
+
+### Activate user
+
+- Params:
+  - `nicknames`: nicknames array
+- Response:
+
+```json
+{
+  users: [
+    {
+      // user object
+    }
+  ]
+}
+```
+
+## `PATCH /api/pleroma/admin/users/deactivate`
+
+### Deactivate user
+
+- Params:
+  - `nicknames`: nicknames array
+- Response:
+
+```json
+{
+  users: [
+    {
+      // user object
+    }
+  ]
+}
+```
+
+## DEPRECATED `PATCH /api/pleroma/admin/users/:nickname/activation_status`
 
 ### Active or deactivate a user
 
-- Method: `PUT`
 - Params:
   - `nickname`
   - `status` BOOLEAN field, false value means deactivation.
@@ -218,17 +289,35 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret
 - Response:
   - On success: URL of the unfollowed relay
 
+## `GET /api/pleroma/admin/relay`
+
+### List Relays
+
+- Params: none
+- Response:
+  - On success: JSON array of relays
+
 ## `/api/pleroma/admin/users/invite_token`
 
-### Get an account registration invite token
+### Create an account registration invite token
 
-- Methods: `GET`
+- Methods: `POST`
 - Params:
-  - *optional* `invite` => [
   - *optional* `max_use` (integer)
   - *optional* `expires_at` (date string e.g. "2019-04-07")
-  ]
+- Response:
-- Response: invite token (base64 string)
+
+```json
+{
+  "id": integer,
+  "token": string,
+  "used": boolean,
+  "expires_at": date,
+  "uses": integer,
+  "max_use": integer,
+  "invite_type": string (possible values: `one_time`, `reusable`, `date_limited`, `reusable_date_limited`)
+}
+```
 
 ## `/api/pleroma/admin/users/invites`
 
@@ -294,16 +383,32 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret
 
 - Methods: `GET`
 - Params: none
-- Response: password reset token (base64 string)
+- Response:
+
+```json
+{
+  "token": "base64 reset token",
+  "link": "https://pleroma.social/api/pleroma/password_reset/url-encoded-base64-token"
+}
+```
+
+
+## `/api/pleroma/admin/users/:nickname/force_password_reset`
+
+### Force passord reset for a user with a given nickname
+
+- Methods: `PATCH`
+- Params: none
+- Response: none (code `204`)
 
 ## `/api/pleroma/admin/reports`
 ### Get a list of reports
 - Method `GET`
 - Params:
-  - `state`: optional, the state of reports. Valid values are `open`, `closed` and `resolved`
+  - *optional* `state`: **string** the state of reports. Valid values are `open`, `closed` and `resolved`
-  - `limit`: optional, the number of records to retrieve
+  - *optional* `limit`: **integer** the number of records to retrieve
-  - `since_id`: optional, returns results that are more recent than the specified id
+  - *optional* `page`: **integer** page number
-  - `max_id`: optional, returns results that are older than the specified id
+  - *optional* `page_size`: **integer** number of log entries per page (default is `50`)
 - Response:
   - On failure: 403 Forbidden error `{"error": "error_msg"}` when requested by anonymous or non-admin
   - On success: JSON, returns a list of reports, where:
@@ -313,6 +418,7 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret
 
 ```json
 {
+  "total" : 1,
   "reports": [
     {
       "account": {
@@ -680,6 +786,7 @@ Compile time settings (need instance reboot):
     }
   ]
 }
+```
 
 - Response:
 
@@ -694,3 +801,38 @@ Compile time settings (need instance reboot):
   ]
 }
 ```
+
+## `/api/pleroma/admin/moderation_log`
+### Get moderation log
+- Method `GET`
+- Params:
+  - *optional* `page`: **integer** page number
+  - *optional* `page_size`: **integer** number of log entries per page (default is `50`)
+  - *optional* `start_date`: **datetime (ISO 8601)** filter logs by creation date, start from `start_date`. Accepts datetime in ISO 8601 format (YYYY-MM-DDThh:mm:ss), e.g. `2005-08-09T18:31:42`
+  - *optional* `end_date`: **datetime (ISO 8601)** filter logs by creation date, end by from `end_date`. Accepts datetime in ISO 8601 format (YYYY-MM-DDThh:mm:ss), e.g. 2005-08-09T18:31:42
+  - *optional* `user_id`: **integer** filter logs by actor's id
+  - *optional* `search`: **string** search logs by the log message
+- Response:
+
+```json
+[
+  {
+    "data": {
+      "actor": {
+        "id": 1,
+        "nickname": "lain"
+      },
+      "action": "relay_follow"
+    },
+    "time": 1502812026, // timestamp
+    "message": "[2017-08-15 15:47:06] @nick0 followed relay: https://example.org/relay" // log message
+  }
+]
+```
+
+## `POST /api/pleroma/admin/reload_emoji`
+### Reload the instance's custom emoji
+* Method `POST`
+* Authentication: required
+* Params: None
+* Response: JSON, "ok" and 200 status

docs/API/differences_in_mastoapi_responses.md

@@ -13,6 +13,7 @@ Some apps operate under the assumption that no more than 4 attachments can be re
 ## Timelines
 
 Adding the parameter `with_muted=true` to the timeline queries will also return activities by muted (not by blocked!) users.
+Adding the parameter `exclude_visibilities` to the timeline queries will exclude the statuses with the given visibilities. The parameter accepts an array of visibility types (`public`, `unlisted`, `private`, `direct`), e.g., `exclude_visibilities[]=direct&exclude_visibilities[]=private`.
 
 ## Statuses
 
@@ -21,10 +22,13 @@ Adding the parameter `with_muted=true` to the timeline queries will also return
 Has these additional fields under the `pleroma` object:
 
 - `local`: true if the post was made on the local instance
-- `conversation_id`: the ID of the conversation the status is associated with (if any)
+- `conversation_id`: the ID of the AP context the status is associated with (if any)
+- `direct_conversation_id`: the ID of the Mastodon direct message conversation the status is associated with (if any)
 - `in_reply_to_account_acct`: the `acct` property of User entity for replied user (if any)
 - `content`: a map consisting of alternate representations of the `content` property with the key being it's mimetype. Currently the only alternate representation supported is `text/plain`
 - `spoiler_text`: a map consisting of alternate representations of the `spoiler_text` property with the key being it's mimetype. Currently the only alternate representation supported is `text/plain`
+- `expires_at`: a datetime (iso8601) that states when the post will expire (be deleted automatically), or empty if the post won't expire
+- `thread_muted`: true if the thread the post belongs to is muted
 
 ## Attachments
 
@@ -48,9 +52,12 @@ Has these additional fields under the `pleroma` object:
 - `confirmation_pending`: boolean, true if a new user account is waiting on email confirmation to be activated
 - `hide_followers`: boolean, true when the user has follower hiding enabled
 - `hide_follows`: boolean, true when the user has follow hiding enabled
+- `hide_followers_count`: boolean, true when the user has follower stat hiding enabled
+- `hide_follows_count`: boolean, true when the user has follow stat hiding enabled
 - `settings_store`: A generic map of settings for frontends. Opaque to the backend. Only returned in `verify_credentials` and `update_credentials`
 - `chat_token`: The token needed for Pleroma chat. Only returned in `verify_credentials`
 - `deactivated`: boolean, true when the user is deactivated
+- `unread_conversation_count`: The count of unread conversations. Only returned to the account owner.
 
 ### Source
 
@@ -78,6 +85,12 @@ Has these additional fields under the `pleroma` object:
 
 - `is_seen`: true if the notification was read by the user
 
+## GET `/api/v1/notifications`
+
+Accepts additional parameters:
+
+- `exclude_visibilities`: will exclude the notifications for activities with the given visibilities. The parameter accepts an array of visibility types (`public`, `unlisted`, `private`, `direct`). Usage example: `GET /api/v1/notifications?exclude_visibilities[]=direct&exclude_visibilities[]=private`.
+
 ## POST `/api/v1/statuses`
 
 Additional parameters can be added to the JSON body/Form data:
@@ -86,8 +99,23 @@ Additional parameters can be added to the JSON body/Form data:
 - `content_type`: string, contain the MIME type of the status, it is transformed into HTML by the backend. You can get the list of the supported MIME types with the nodeinfo endpoint.
 - `to`: A list of nicknames (like `lain@soykaf.club` or `lain` on the local server) that will be used to determine who is going to be addressed by this post. Using this will disable the implicit addressing by mentioned names in the `status` body, only the people in the `to` list will be addressed. The normal rules for for post visibility are not affected by this and will still apply.
 - `visibility`: string, besides standard MastoAPI values (`direct`, `private`, `unlisted` or `public`) it can be used to address a List by setting it to `list:LIST_ID`.
+- `expires_in`: The number of seconds the posted activity should expire in. When a posted activity expires it will be deleted from the server, and a delete request for it will be federated. This needs to be longer than an hour.
 - `in_reply_to_conversation_id`: Will reply to a given conversation, addressing only the people who are part of the recipient set of that conversation. Sets the visibility to `direct`.
 
+## GET `/api/v1/statuses`
+
+An endpoint to get multiple statuses by IDs.
+
+Required parameters:
+
+- `ids`: array of activity ids
+
+Usage example: `GET /api/v1/statuses/?ids[]=1&ids[]=2`.
+
+Returns: array of Status.
+
+The maximum number of statuses is limited to 100 per request.
+
 ## PATCH `/api/v1/update_credentials`
 
 Additional parameters can be added to the JSON body/Form data:
@@ -95,6 +123,8 @@ Additional parameters can be added to the JSON body/Form data:
 - `no_rich_text` - if true, html tags are stripped from all statuses requested from the API
 - `hide_followers` - if true, user's followers will be hidden
 - `hide_follows` - if true, user's follows will be hidden
+- `hide_followers_count` - if true, user's follower count will be hidden
+- `hide_follows_count` - if true, user's follow count will be hidden
 - `hide_favorites` - if true, user's favorites timeline will be hidden
 - `show_role` - if true, user's role (e.g admin, moderator) will be exposed to anyone in the API
 - `default_scope` - the scope returned under `privacy` key in Source subentity

docs/API/pleroma_api.md

@@ -124,15 +124,16 @@ Request parameters can be passed via [query strings](https://en.wikipedia.org/wi
 ```
 
 ## `/api/pleroma/admin/`…
-See [Admin-API](Admin-API.md)
+See [Admin-API](admin_api.md)
 
-## `/api/pleroma/notifications/read`
+## `/api/v1/pleroma/notifications/read`
-### Mark a single notification as read
+### Mark notifications as read
 * Method `POST`
 * Authentication: required
-* Params:
+* Params (mutually exclusive):
-    * `id`: notification's id
+    * `id`: a single notification id to read
-* Response: JSON. Returns `{"status": "success"}` if the reading was successful, otherwise returns `{"error": "error_msg"}`
+    * `max_id`: read all notifications up to this id
+* Response: Notification entity/Array of Notification entities that were read. In case of `max_id`, only the first 80 read notifications will be returned.
 
 ## `/api/v1/pleroma/accounts/:id/subscribe`
 ### Subscribe to receive notifications for all statuses posted by a user
@@ -316,10 +317,21 @@ See [Admin-API](Admin-API.md)
   "active": 0, # active processes
   "idle": 0, # idle processes
   "memory_used": 0.00, # Memory used
-  "healthy": true # Instance state
+  "healthy": true, # Instance state
+  "job_queue_stats": {} # Job queue stats
 }
 ```
 
+## `/api/pleroma/change_email`
+### Change account email
+* Method `POST`
+* Authentication: required
+* Params:
+    * `password`: user's password
+    * `email`: new email
+* Response: JSON. Returns `{"status": "success"}` if the change was successful, `{"error": "[error message]"}` otherwise
+* Note: Currently, Mastodon has no API for changing email. If they add it in future it might be incompatible with Pleroma.
+
 # Pleroma Conversations
 
 Pleroma Conversations have the same general structure that Mastodon Conversations have. The behavior differs in the following ways when using these endpoints:
@@ -354,3 +366,116 @@ The status posting endpoint takes an additional parameter, `in_reply_to_conversa
 * Params:
     * `recipients`: A list of ids of users that should receive posts to this conversation. This will replace the current list of recipients, so submit the full list. The owner of owner of the conversation will always be part of the set of recipients, though.
 * Response: JSON, statuses (200 - healthy, 503 unhealthy)
+
+## `GET /api/v1/pleroma/conversations/read`
+### Marks all user's conversations as read.
+* Method `POST`
+* Authentication: required
+* Params: None
+* Response: JSON, returns a list of Mastodon Conversation entities that were marked as read (200 - healthy, 503 unhealthy).
+
+## `GET /api/pleroma/emoji/packs`
+### Lists the custom emoji packs on the server
+* Method `GET`
+* Authentication: not required
+* Params: None
+* Response: JSON, "ok" and 200 status and the JSON hashmap of "pack name" to "pack contents"
+
+## `PUT /api/pleroma/emoji/packs/:name`
+### Creates an empty custom emoji pack
+* Method `PUT`
+* Authentication: required
+* Params: None
+* Response: JSON, "ok" and 200 status or 409 if the pack with that name already exists
+
+## `DELETE /api/pleroma/emoji/packs/:name`
+### Delete a custom emoji pack
+* Method `DELETE`
+* Authentication: required
+* Params: None
+* Response: JSON, "ok" and 200 status or 500 if there was an error deleting the pack
+
+## `POST /api/pleroma/emoji/packs/:name/update_file`
+### Update a file in a custom emoji pack
+* Method `POST`
+* Authentication: required
+* Params:
+    * if the `action` is `add`, adds an emoji named `shortcode` to the pack `pack_name`,
+      that means that the emoji file needs to be uploaded with the request
+      (thus requiring it to be a multipart request) and be named `file`.
+      There can also be an optional `filename` that will be the new emoji file name
+      (if it's not there, the name will be taken from the uploaded file).
+    * if the `action` is `update`, changes emoji shortcode
+      (from `shortcode` to `new_shortcode` or moves the file (from the current filename to `new_filename`)
+    * if the `action` is `remove`, removes the emoji named `shortcode` and it's associated file
+* Response: JSON, updated "files" section of the pack and 200 status, 409 if the trying to use a shortcode
+  that is already taken, 400 if there was an error with the shortcode, filename or file (additional info
+  in the "error" part of the response JSON)
+
+## `POST /api/pleroma/emoji/packs/:name/update_metadata`
+### Updates (replaces) pack metadata
+* Method `POST`
+* Authentication: required
+* Params:
+  * `new_data`: new metadata to replace the old one
+* Response: JSON, updated "metadata" section of the pack and 200 status or 400 if there was a
+  problem with the new metadata (the error is specified in the "error" part of the response JSON)
+
+## `POST /api/pleroma/emoji/packs/download_from`
+### Requests the instance to download the pack from another instance
+* Method `POST`
+* Authentication: required
+* Params:
+  * `instance_address`: the address of the instance to download from
+  * `pack_name`: the pack to download from that instance
+* Response: JSON, "ok" and 200 status if the pack was downloaded, or 500 if there were
+  errors downloading the pack
+
+## `POST /api/pleroma/emoji/packs/list_from`
+### Requests the instance to list the packs from another instance
+* Method `POST`
+* Authentication: required
+* Params:
+  * `instance_address`: the address of the instance to download from
+* Response: JSON with the pack list, same as if the request was made to that instance's
+  list endpoint directly + 200 status
+
+## `GET /api/pleroma/emoji/packs/:name/download_shared`
+### Requests a local pack from the instance
+* Method `GET`
+* Authentication: not required
+* Params: None
+* Response: the archive of the pack with a 200 status code, 403 if the pack is not set as shared,
+  404 if the pack does not exist
+
+## `GET /api/v1/pleroma/accounts/:id/scrobbles`
+### Requests a list of current and recent Listen activities for an account
+* Method `GET`
+* Authentication: not required
+* Params: None
+* Response: An array of media metadata entities.
+* Example response:
+```json
+[
+   {
+       "account": {...},
+       "id": "1234",
+       "title": "Some Title",
+       "artist": "Some Artist",
+       "album": "Some Album",
+       "length": 180000,
+       "created_at": "2019-09-28T12:40:45.000Z"
+   }
+]
+```
+
+## `POST /api/v1/pleroma/scrobble`
+### Creates a new Listen activity for an account
+* Method `POST`
+* Authentication: required
+* Params:
+  * `title`: the title of the media playing
+  * `album`: the album of the media playing [optional]
+  * `artist`: the artist of the media playing [optional]
+  * `length`: the length of the media playing [optional]
+* Response: the newly created media metadata entity representing the Listen activity

docs/administration/CLI_tasks/config.md

@@ -0,0 +1,19 @@
+# Transfering the config to/from the database
+
+!!! danger
+    This is a Work In Progress, not usable just yet.
+
+Every command should be ran with a prefix, in case of OTP releases it is `./bin/pleroma_ctl config` and in case of source installs it's
+`mix pleroma.config`.
+
+## Transfer config from file to DB.
+
+```sh
+$PREFIX migrate_to_db
+```
+
+## Transfer config from DB to `config/env.exported_from_db.secret.exs`
+
+```sh
+$PREFIX migrate_from_db <env>
+```

docs/administration/CLI_tasks/database.md

@@ -0,0 +1,48 @@
+# Database maintenance tasks
+
+Every command should be ran with a prefix, in case of OTP releases it is `./bin/pleroma_ctl database` and in case of source installs it's `mix pleroma.database`.
+
+## Replace embedded objects with their references
+
+Replaces embedded objects with references to them in the `objects` table. Only needs to be ran once if the instance was created before Pleroma 1.0.5. The reason why this is not a migration is because it could significantly increase the database size after being ran, however after this `VACUUM FULL` will be able to reclaim about 20% (really depends on what is in the database, your mileage may vary) of the db size before the migration.
+
+```sh
+$PREFIX remove_embedded_objects [<options>]
+```
+
+### Options
+- `--vacuum` - run `VACUUM FULL` after the embedded objects are replaced with their references
+
+## Prune old remote posts from the database
+
+This will prune remote posts older than 90 days (configurable with [`config :pleroma, :instance, remote_post_retention_days`](../../configuration/cheatsheet.md#instance)) from the database, they will be refetched from source when accessed.
+
+!!! note
+    The disk space will only be reclaimed after `VACUUM FULL`
+
+```sh
+$PREFIX pleroma.database prune_objects [<options>]
+```
+
+### Options
+- `--vacuum` - run `VACUUM FULL` after the objects are pruned
+
+## Create a conversation for all existing DMs
+
+Can be safely re-run
+
+```sh
+$PREFIX bump_all_conversations
+```
+
+## Remove duplicated items from following and update followers count for all users
+
+```sh
+$PREFIX update_users_following_followers_counts
+```
+
+## Fix the pre-existing "likes" collections for all objects
+
+```sh
+$PREFIX fix_likes_collections
+```

docs/administration/CLI_tasks/digest.md

@@ -0,0 +1,13 @@
+# Managing digest emails
+Every command should be ran with a prefix, in case of OTP releases it is `./bin/pleroma_ctl digest` and in case of source installs it's `mix pleroma.digest`.
+
+## Send digest email since given date (user registration date by default) ignoring user activity status.
+
+```sh
+$PREFIX test <nickname> [<since_date>]
+```
+
+Example: 
+```sh
+$PREFIX test donaldtheduck 2019-05-20
+```

docs/administration/CLI_tasks/emoji.md

@@ -0,0 +1,30 @@
+# Managing emoji packs
+
+Every command should be ran with a prefix, in case of OTP releases it is `./bin/pleroma_ctl emoji` and in case of source installs it's `mix pleroma.emoji`.
+
+## Lists emoji packs and metadata specified in the manifest
+
+```sh
+$PREFIX ls-packs [<options>]
+```
+
+### Options
+- `-m, --manifest PATH/URL` - path to a custom manifest, it can either be an URL starting with `http`, in that case the manifest will be fetched from that address, or a local path
+
+## Fetch, verify and install the specified packs from the manifest into `STATIC-DIR/emoji/PACK-NAME`
+```sh
+$PREFIX get-packs [<options>] <packs>
+```
+
+### Options
+- `-m, --manifest PATH/URL` - same as [`ls-packs`](#ls-packs)
+
+## Create a new manifest entry and a file list from the specified remote pack file
+```sh
+$PREFIX gen-pack PACK-URL
+```
+Currently, only .zip archives are recognized as remote pack files and packs are therefore assumed to be zip archives. This command is intended to run interactively and will first ask you some basic questions about the pack, then download the remote file and generate an SHA256 checksum for it, then generate an emoji file list for you. 
+
+  The manifest entry will either be written to a newly created `index.json` file or appended to the existing one, *replacing* the old pack with the same name if it was in the file previously.
+
+  The file list will be written to the file specified previously, *replacing* that file. You _should_ check that the file list doesn't contain anything you don't need in the pack, that is, anything that is not an emoji (the whole pack is downloaded, but only emoji files are extracted).

docs/administration/CLI_tasks/instance.md

@@ -0,0 +1,30 @@
+# Managing instance configuration
+
+Every command should be ran with a prefix, in case of OTP releases it is `./bin/pleroma_ctl instance` and in case of source installs it's `mix pleroma.instance`.
+
+## Generate a new configuration file
+```sh
+$PREFIX gen [<options>]
+```
+
+If any of the options are left unspecified, you will be prompted interactively.
+
+### Options
+- `-f`, `--force` - overwrite any output files
+- `-o <path>`, `--output <path>` - the output file for the generated configuration
+- `--output-psql <path>` - the output file for the generated PostgreSQL setup
+- `--domain <domain>` - the domain of your instance
+- `--instance-name <instance_name>` - the name of your instance
+- `--admin-email <email>` - the email address of the instance admin
+- `--notify-email <email>` - email address for notifications
+- `--dbhost <hostname>` - the hostname of the PostgreSQL database to use
+- `--dbname <database_name>` - the name of the database to use
+- `--dbuser <username>` - the user (aka role) to use for the database connection
+- `--dbpass <password>` - the password to use for the database connection
+- `--rum <Y|N>` - Whether to enable RUM indexes
+- `--indexable <Y|N>` - Allow/disallow indexing site by search engines
+- `--db-configurable <Y|N>` - Allow/disallow configuring instance from admin part
+- `--uploads-dir <path>` - the directory uploads go in when using a local uploader
+- `--static-dir <path>` - the directory custom public files should be read from (custom emojis, frontend bundle overrides, robots.txt, etc.)
+- `--listen-ip <ip>` - the ip the app should listen to, defaults to 127.0.0.1
+- `--listen-port <port>` - the port the app should listen to, defaults to 4000

docs/administration/CLI_tasks/relay.md

@@ -0,0 +1,30 @@
+# Managing relays
+
+Every command should be ran with a prefix, in case of OTP releases it is `./bin/pleroma_ctl relay` and in case of source installs it's `mix pleroma.relay`.
+
+## Follow a relay
+```sh
+$PREFIX follow <relay_url>
+```
+
+Example:
+```sh
+$PREFIX follow https://example.org/relay
+```
+
+## Unfollow a remote relay
+
+```sh
+$PREFIX unfollow <relay_url>
+```
+
+Example:
+```sh
+$PREFIX unfollow https://example.org/relay
+```
+
+## List relay subscriptions
+
+```sh
+$PREFIX list
+```

docs/administration/CLI_tasks/uploads.md

@@ -0,0 +1,12 @@
+# Managing uploads
+
+Every command should be ran with a prefix, in case of OTP releases it is `./bin/pleroma_ctl uploads` and in case of source installs it's `mix pleroma.uploads`.
+
+## Migrate uploads from local to remote storage
+```sh
+$PREFIX migrate_local <target_uploader> [<options>]
+```
+### Options
+- `--delete` - delete local uploads after migrating them to the target uploader
+
+A list of available uploaders can be seen in [Configuration Cheat Sheet](../../configuration/cheatsheet.md#pleromaupload)

docs/administration/CLI_tasks/user.md

@@ -0,0 +1,94 @@
+# Managing users
+
+Every command should be ran with a prefix, in case of OTP releases it is `./bin/pleroma_ctl user` and in case of source installs it's `mix pleroma.user`.
+
+## Create a user
+```sh
+$PREFIX new <nickname> <email> [<options>]
+```
+
+### Options
+- `--name <name>` - the user's display name
+- `--bio <bio>` - the user's bio
+- `--password <password>` - the user's password
+- `--moderator`/`--no-moderator` - whether the user should be a moderator
+- `--admin`/`--no-admin` - whether the user should be an admin
+- `-y`, `--assume-yes`/`--no-assume-yes` - whether to assume yes to all questions
+
+## Generate an invite link
+```sh
+$PREFIX invite [<options>]
+```
+
+### Options
+- `--expires-at DATE` - last day on which token is active (e.g. "2019-04-05")
+- `--max-use NUMBER` - maximum numbers of token uses
+
+## List generated invites
+```sh
+$PREFIX invites
+```
+
+## Revoke invite
+```sh
+$PREFIX revoke_invite <token_or_id>
+```
+
+## Delete a user
+```sh
+$PREFIX rm <nickname>
+```
+
+## Delete user's posts and interactions
+```sh
+$PREFIX delete_activities <nickname>
+```
+
+## Sign user out from all applications (delete user's OAuth tokens and authorizations)
+```sh
+$PREFIX sign_out <nickname>
+```
+
+## Deactivate or activate a user 
+```sh
+$PREFIX toggle_activated <nickname> 
+```
+
+## Unsubscribe local users from a user and deactivate the user
+```sh
+$PREFIX unsubscribe NICKNAME
+```
+
+## Unsubscribe local users from an instance and deactivate all accounts on it
+```sh
+$PREFIX unsubscribe_all_from_instance <instance>
+```
+
+## Create a password reset link for user
+```sh
+$PREFIX reset_password <nickname>
+```
+
+## Set the value of the given user's settings
+```sh
+$PREFIX set <nickname> [<options>]
+```
+### Options
+- `--locked`/`--no-locked` - whether the user should be locked
+- `--moderator`/`--no-moderator` - whether the user should be a moderator
+- `--admin`/`--no-admin` - whether the user should be an admin
+
+## Add tags to a user
+```sh
+$PREFIX tag <nickname> <tags>
+```
+
+## Delete tags from a user
+```sh
+$PREFIX untag <nickname> <tags>
+```
+
+## Toggle confirmation status of the user
+```sh
+$PREFIX toggle_confirmed <nickname>
+```

docs/clients.md

@@ -39,7 +39,7 @@ Feel free to contact us to be added to this list!
 
 ### Nekonium
 - Homepage: [F-Droid Repository](https://repo.gdgd.jp.net/), [Google Play](https://play.google.com/store/apps/details?id=com.apps.nekonium), [Amazon](https://www.amazon.co.jp/dp/B076FXPRBC/)
-- Source: <https://git.gdgd.jp.net/lin/nekonium/>
+- Source: <https://gogs.gdgd.jp.net/lin/nekonium>
 - Contact: [@lin@pleroma.gdgd.jp.net](https://pleroma.gdgd.jp.net/users/lin)
 - Platforms: Android
 - Features: Streaming Ready
@@ -67,7 +67,7 @@ Feel free to contact us to be added to this list!
 ## Alternative Web Interfaces
 ### Brutaldon
 - Homepage: <https://jfm.carcosa.net/projects/software/brutaldon/>
-- Source Code: <https://github.com/jfmcbrayer/brutaldon>
+- Source Code: <https://git.carcosa.net/jmcbray/brutaldon>
 - Contact: [@gcupc@glitch.social](https://glitch.social/users/gcupc)
 - Features: No Streaming
 

docs/config/General-tips-for-customizing-Pleroma-FE.md

@@ -1,17 +0,0 @@
-# General tips for customizing Pleroma FE
-There are some configuration scripts for Pleroma BE and FE:
-
-1. `config/prod.secret.exs`
-1. `config/config.exs`
-1. `priv/static/static/config.json`
-
-The `prod.secret.exs` affects first. `config.exs` is for fallback or default. `config.json` is for GNU-social-BE-Pleroma-FE instances.
-
-Usually all you have to do is:
-
-1. Copy the section in the `config/config.exs` which you want to activate.
-1. Paste into `config/prod.secret.exs`.
-1. Edit `config/prod.secret.exs`.
-1. Restart the Pleroma daemon.
-
-`prod.secret.exs` is for the `MIX_ENV=prod` environment. `dev.secret.exs` is for the `MIX_ENV=dev` environment respectively.

docs/config/small_customizations.md

@@ -1,12 +0,0 @@
-# Small customizations
-
-See also static_dir.md for visual settings.
-
-## Theme
-
-All users of your instance will be able to change the theme they use by going to the settings (the cog in the top-right hand corner). However, if you wish to change the default theme, you can do so by editing `theme` in `config/dev.secret.exs` accordingly.
-
-## Message Visibility
-
-To enable message visibility options when posting like in the Mastodon frontend, set
-`scope_options_enabled` to `true` in `config/dev.secret.exs`.

docs/configuration/cheatsheet.md

@@ -1,17 +1,22 @@
-# Configuration
+# Configuration Cheat Sheet
+
+This is a cheat sheet for Pleroma configuration file, any setting possible to configure should be listed here.
+
+Pleroma configuration works by first importing the base config (`config/config.exs` on source installs, compiled-in on OTP releases), then overriding it by the environment config (`config/$MIX_ENV.exs` on source installs, N/A to OTP releases) and then overriding it by user config (`config/$MIX_ENV.secret.exs` on source installs, typically `/etc/pleroma/config.exs` on OTP releases).
+
+You shouldn't edit the base config directly to avoid breakages and merge conflicts, but it can be used as a reference if you don't understand how an option is supposed to be formatted, the latest version of it can be viewed [here](https://git.pleroma.social/pleroma/pleroma/blob/develop/config/config.exs).
 
-This file describe the configuration, it is recommended to edit the relevant *.secret.exs file instead of the others founds in the ``config`` directory.
-If you run Pleroma with ``MIX_ENV=prod`` the file is ``prod.secret.exs``, otherwise it is ``dev.secret.exs``.
 
 ## Pleroma.Upload
 * `uploader`: Select which `Pleroma.Uploaders` to use
 * `filters`: List of `Pleroma.Upload.Filter` to use.
 * `link_name`: When enabled Pleroma will add a `name` parameter to the url of the upload, for example `https://instance.tld/media/corndog.png?name=corndog.png`. This is needed to provide the correct filename in Content-Disposition headers when using filters like `Pleroma.Upload.Filter.Dedupe`
 * `base_url`: The base URL to access a user-uploaded file. Useful when you want to proxy the media files via another host.
-* `proxy_remote`: If you\'re using a remote uploader, Pleroma will proxy media requests instead of redirecting to it.
+* `proxy_remote`: If you're using a remote uploader, Pleroma will proxy media requests instead of redirecting to it.
 * `proxy_opts`: Proxy options, see `Pleroma.ReverseProxy` documentation.
 
-Note: `strip_exif` has been replaced by `Pleroma.Upload.Filter.Mogrify`.
+!!! warning
+    `strip_exif` has been replaced by `Pleroma.Upload.Filter.Mogrify`.
 
 ## Pleroma.Uploaders.Local
 * `uploads`: Which directory to store the user-uploads in, relative to pleroma’s working directory
@@ -23,6 +28,7 @@ Note: `strip_exif` has been replaced by `Pleroma.Upload.Filter.Mogrify`.
 * `truncated_namespace`: If you use S3 compatible service such as Digital Ocean Spaces or CDN, set folder name or "" etc.
 For example, when using CDN to S3 virtual host format, set "".
 At this time, write CNAME to CDN in public_endpoint.
+* `streaming_enabled`: Enable streaming uploads, when enabled the file will be sent to the server in chunks as it's being read. This may be unsupported by some providers, try disabling this if you have upload problems.
 
 ## Pleroma.Upload.Filter.Mogrify
 
@@ -110,12 +116,6 @@ config :pleroma, Pleroma.Emails.Mailer,
 * `allowed_post_formats`: MIME-type list of formats allowed to be posted (transformed into HTML)
 * `mrf_transparency`: Make the content of your Message Rewrite Facility settings public (via nodeinfo).
 * `mrf_transparency_exclusions`: Exclude specific instance names from MRF transparency.  The use of the exclusions feature will be disclosed in nodeinfo as a boolean value.
-* `scope_copy`: Copy the scope (private/unlisted/public) in replies to posts by default.
-* `subject_line_behavior`: Allows changing the default behaviour of subject lines in replies. Valid values:
-  * "email": Copy and preprend re:, as in email.
-  * "masto": Copy verbatim, as in Mastodon.
-  * "noop": Don't copy the subject.
-* `always_show_subject_input`: When set to false, auto-hide the subject field when it's empty.
 * `extended_nickname_format`: Set to `true` to use extended local nicknames format (allows underscores/dashes). This will break federation with
     older software for theses nicknames.
 * `max_pinned_statuses`: The maximum number of pinned statuses. `0` will disable the feature.
@@ -131,9 +131,17 @@ config :pleroma, Pleroma.Emails.Mailer,
 * `user_name_length`: A user name maximum length (default: `100`)
 * `skip_thread_containment`: Skip filter out broken threads. The default is `false`.
 * `limit_to_local_content`: Limit unauthenticated users to search for local statutes and users only. Possible values: `:unauthenticated`, `:all` and `false`. The default is `:unauthenticated`.
-* `dynamic_configuration`: Allow transferring configuration to DB with the subsequent customization from Admin api.
+* `max_account_fields`: The maximum number of custom fields in the user profile (default: `10`)
+* `max_remote_account_fields`: The maximum number of custom fields in the remote user profile (default: `20`)
+* `account_field_name_length`: An account field name maximum length (default: `512`)
+* `account_field_value_length`: An account field value maximum length (default: `2048`)
 * `external_user_synchronization`: Enabling following/followers counters synchronization for external users.
 
+!!! danger
+    This is a Work In Progress, not usable just yet
+
+* `dynamic_configuration`: Allow transferring configuration to DB with the subsequent customization from Admin api.
+
 
 
 ## :logger
@@ -181,7 +189,7 @@ See the [Quack Github](https://github.com/azohra/quack) for more details
 
 ## :frontend_configurations
 
-This can be used to configure a keyword list that keeps the configuration data for any kind of frontend. By default, settings for `pleroma_fe` and `masto_fe` are configured.
+This can be used to configure a keyword list that keeps the configuration data for any kind of frontend. By default, settings for `pleroma_fe` and `masto_fe` are configured. You can find the documentation for `pleroma_fe` configuration into [Pleroma-FE configuration and customization for instance administrators](/frontend/CONFIGURATION/#options).
 
 Frontends can access these settings at `/api/pleroma/frontend_configurations`
 
@@ -203,14 +211,15 @@ These settings **need to be complete**, they will override the defaults.
 NOTE: for versions < 1.0, you need to set [`:fe`](#fe) to false, as shown a few lines below.
 
 ## :fe
-__THIS IS DEPRECATED__
+!!! warning
+    __THIS IS DEPRECATED__
 
-If you are using this method, please change it to the [`frontend_configurations`](#frontend_configurations) method.
+    If you are using this method, please change it to the [`frontend_configurations`](#frontend_configurations) method.
-Please **set this option to false** in your config like this:
+    Please **set this option to false** in your config like this:
 
-```elixir
+    ```elixir
-config :pleroma, :fe, false
+    config :pleroma, :fe, false
-```
+    ```
 
 This section is used to configure Pleroma-FE, unless ``:managed_config`` in ``:instance`` is set to false.
 
@@ -238,6 +247,35 @@ relates to mascots on the mastodon frontend
 * `default_mascot`: An element from `mascots` - This will be used as the default mascot
   on MastoFE (default: `:pleroma_fox_tan`)
 
+## :manifest
+
+This section describe PWA manifest instance-specific values. Currently this option relate only for MastoFE.
+
+* `icons`: Describe the icons of the app, this a list of maps describing icons in the same way as the
+  [spec](https://www.w3.org/TR/appmanifest/#imageresource-and-its-members) describes it.
+
+  Example:
+
+  ```elixir
+  config :pleroma, :manifest,
+    icons: [
+      %{
+        src: "/static/logo.png"
+      },
+      %{
+        src: "/static/icon.png",
+        type: "image/png"
+      },
+      %{
+        src: "/static/icon.ico",
+        sizes: "72x72 96x96 128x128 256x256"
+      }
+    ]
+  ```
+
+* `theme_color`: Describe the theme color of the app. (Example: `"#282c37"`, `"rebeccapurple"`)
+* `background_color`: Describe the background color of the app. (Example: `"#191b22"`, `"aliceblue"`)
+
 ## :mrf_simple
 * `media_removal`: List of instances to remove medias from
 * `media_nsfw`: List of instances to put medias as NSFW(sensitive) from
@@ -256,7 +294,7 @@ All criteria are configured as a map of regular expressions to lists of policy m
 
 Example:
 
-```
+```elixir
 config :pleroma, :mrf_subchain,
   match_actor: %{
     ~r/https:\/\/example.com/s => [Pleroma.Web.ActivityPub.MRF.DropPolicy]
@@ -296,7 +334,10 @@ config :pleroma, :mrf_subchain,
 * `dstport`: Port advertised in urls (optional, defaults to `port`)
 
 ## Pleroma.Web.Endpoint
-`Phoenix` endpoint configuration, all configuration options can be viewed [here](https://hexdocs.pm/phoenix/Phoenix.Endpoint.html#module-dynamic-configuration), only common options are listed here
+
+!!! note
+    `Phoenix` endpoint configuration, all configuration options can be viewed [here](https://hexdocs.pm/phoenix/Phoenix.Endpoint.html#module-dynamic-configuration), only common options are listed here.
+
 * `http` - a list containing http protocol configuration, all configuration options can be viewed [here](https://hexdocs.pm/plug_cowboy/Plug.Cowboy.html#module-options), only common options are listed here. For deployment using docker, you need to set this to `[ip: {0,0,0,0}, port: 4000]` to make pleroma accessible from other containers (such as your nginx server).
   - `ip` - a tuple consisting of 4 integers
   - `port`
@@ -309,7 +350,8 @@ config :pleroma, :mrf_subchain,
 
 
 
-**Important note**: if you modify anything inside these lists, default `config.exs` values will be overwritten, which may result in breakage, to make sure this does not happen please copy the default value for the list from `config.exs` and modify/add only what you need
+!!! warning
+    If you modify anything inside these lists, default `config.exs` values will be overwritten, which may result in breakage, to make sure this does not happen please copy the default value for the list from `config.exs` and modify/add only what you need
 
 Example:
 ```elixir
@@ -396,41 +438,73 @@ You can then do
 curl "http://localhost:4000/api/pleroma/admin/invite_token?admin_token=somerandomtoken"
 ```
 
-## :pleroma_job_queue
+## Oban
 
-[Pleroma Job Queue](https://git.pleroma.social/pleroma/pleroma_job_queue) configuration: a list of queues with maximum concurrent jobs.
+[Oban](https://github.com/sorentwo/oban) asynchronous job processor configuration.
+
+Configuration options described in [Oban readme](https://github.com/sorentwo/oban#usage):
+* `repo` - app's Ecto repo (`Pleroma.Repo`)
+* `verbose` - logs verbosity
+* `prune` - non-retryable jobs [pruning settings](https://github.com/sorentwo/oban#pruning) (`:disabled` / `{:maxlen, value}` / `{:maxage, value}`)
+* `queues` - job queues (see below)
 
 Pleroma has the following queues:
 
+* `activity_expiration` - Activity expiration
 * `federator_outgoing` - Outgoing federation
 * `federator_incoming` - Incoming federation
-* `mailer` - Email sender, see [`Pleroma.Emails.Mailer`](#pleroma-emails-mailer)
+* `mailer` - Email sender, see [`Pleroma.Emails.Mailer`](#pleromaemailsmailer)
 * `transmogrifier` - Transmogrifier
 * `web_push` - Web push notifications
-* `scheduled_activities` - Scheduled activities, see [`Pleroma.ScheduledActivities`](#pleromascheduledactivity)
+* `scheduled_activities` - Scheduled activities, see [`Pleroma.ScheduledActivity`](#pleromascheduledactivity)
 
 Example:
 
 ```elixir
-config :pleroma_job_queue, :queues,
+config :pleroma, Oban,
+  repo: Pleroma.Repo,
+  verbose: false,
+  prune: {:maxlen, 1500},
+  queues: [
     federator_incoming: 50,
     federator_outgoing: 50
+  ]
 ```
 
-This config contains two queues: `federator_incoming` and `federator_outgoing`. Both have the `max_jobs` set to `50`.
+This config contains two queues: `federator_incoming` and `federator_outgoing`. Both have the number of max concurrent jobs set to `50`.
 
-## Pleroma.Web.Federator.RetryQueue
+### Migrating `pleroma_job_queue` settings
 
-* `enabled`: If set to `true`, failed federation jobs will be retried
+`config :pleroma_job_queue, :queues` is replaced by `config :pleroma, Oban, :queues` and uses the same format (keys are queues' names, values are max concurrent jobs numbers).
-* `max_jobs`: The maximum amount of parallel federation jobs running at the same time.
+
-* `initial_timeout`: The initial timeout in seconds
+## :workers
-* `max_retries`: The maximum number of times a federation job is retried
+
+Includes custom worker options not interpretable directly by `Oban`.
+
+* `retries` — keyword lists where keys are `Oban` queues (see above) and values are numbers of max attempts for failed jobs.
+
+Example:
+
+```elixir
+config :pleroma, :workers,
+  retries: [
+    federator_incoming: 5,
+    federator_outgoing: 5
+  ]
+```
+
+### Migrating `Pleroma.Web.Federator.RetryQueue` settings
+
+* `max_retries` is replaced with `config :pleroma, :workers, retries: [federator_outgoing: 5]`
+* `enabled: false` corresponds to `config :pleroma, :workers, retries: [federator_outgoing: 1]`
+* deprecated options: `max_jobs`, `initial_timeout`
 
 ## Pleroma.Web.Metadata
 * `providers`: a list of metadata providers to enable. Providers available:
   * Pleroma.Web.Metadata.Providers.OpenGraph
   * Pleroma.Web.Metadata.Providers.TwitterCard
   * Pleroma.Web.Metadata.Providers.RelMe - add links from user bio with rel=me into the `<header>` as `<link rel=me>`
+  * Pleroma.Web.Metadata.Providers.Feed - add a link to a user's Atom feed into the `<header>` as `<link rel=alternate>`
 * `unfurl_nsfw`: If set to `true` nsfw attachments will be shown in previews
 
 ## :rich_media
@@ -481,16 +555,38 @@ config :auto_linker,
     class: false,
     strip_prefix: false,
     new_window: false,
-    rel: false
+    rel: "ugc"
   ]
 ```
 
+## Pleroma.Scheduler
+
+Configuration for [Quantum](https://github.com/quantum-elixir/quantum-core) jobs scheduler.
+
+See [Quantum readme](https://github.com/quantum-elixir/quantum-core#usage) for the list of supported options. 
+
+Example:
+
+```elixir
+config :pleroma, Pleroma.Scheduler,
+  global: true,
+  overlap: true,
+  timezone: :utc,
+  jobs: [{"0 */6 * * * *", {Pleroma.Web.Websub, :refresh_subscriptions, []}}]
+```
+
+The above example defines a single job which invokes `Pleroma.Web.Websub.refresh_subscriptions()` every 6 hours ("0 */6 * * * *", [crontab format](https://en.wikipedia.org/wiki/Cron)).
+
 ## Pleroma.ScheduledActivity
 
 * `daily_user_limit`: the number of scheduled activities a user is allowed to create in a single day (Default: `25`)
 * `total_user_limit`: the number of scheduled activities a user is allowed to create in total (Default: `300`)
 * `enabled`: whether scheduled activities are sent to the job queue to be executed
 
+## Pleroma.ActivityExpiration
+
+* `enabled`: whether expired activities will be sent to the job queue to be deleted
+
 ## Pleroma.Web.Auth.Authenticator
 
 * `Pleroma.Web.Auth.PleromaAuthenticator`: default database authenticator
@@ -555,18 +651,24 @@ Email notifications settings.
     - interval: Minimum interval between digest emails to one user
     - inactivity_threshold: Minimum user inactivity threshold
 
+## Pleroma.Emails.UserEmail
+
+- `:logo` - a path to a custom logo. Set it to `nil` to use the default Pleroma logo.
+- `:styling` - a map with color settings for email templates.
+
 ## OAuth consumer mode
 
 OAuth consumer mode allows sign in / sign up via external OAuth providers (e.g. Twitter, Facebook, Google, Microsoft, etc.).
 Implementation is based on Ueberauth; see the list of [available strategies](https://github.com/ueberauth/ueberauth/wiki/List-of-Strategies).
 
-Note: each strategy is shipped as a separate dependency; in order to get the strategies, run `OAUTH_CONSUMER_STRATEGIES="..." mix deps.get`,
+!!! note
-e.g. `OAUTH_CONSUMER_STRATEGIES="twitter facebook google microsoft" mix deps.get`.
+    Each strategy is shipped as a separate dependency; in order to get the strategies, run `OAUTH_CONSUMER_STRATEGIES="..." mix deps.get`, e.g. `OAUTH_CONSUMER_STRATEGIES="twitter facebook google microsoft" mix deps.get`.  The server should also be started with `OAUTH_CONSUMER_STRATEGIES="..." mix phx.server` in case you enable any strategies.
-The server should also be started with `OAUTH_CONSUMER_STRATEGIES="..." mix phx.server` in case you enable any strategies.
 
-Note: each strategy requires separate setup (on external provider side and Pleroma side). Below are the guidelines on setting up most popular strategies.
+!!! note
+    Each strategy requires separate setup (on external provider side and Pleroma side). Below are the guidelines on setting up most popular strategies.
 
-Note: make sure that `"SameSite=Lax"` is set in `extra_cookie_attrs` when you have this feature enabled. OAuth consumer mode will not work with `"SameSite=Strict"`
+!!! note
+    Make sure that `"SameSite=Lax"` is set in `extra_cookie_attrs` when you have this feature enabled. OAuth consumer mode will not work with `"SameSite=Strict"`
 
 * For Twitter, [register an app](https://developer.twitter.com/en/apps), configure callback URL to https://<your_host>/oauth/twitter/callback
 
@@ -640,6 +742,8 @@ Configure OAuth 2 provider capabilities:
 * `pack_extensions`: A list of file extensions for emojis, when no emoji.txt for a pack is present. Example `[".png", ".gif"]`
 * `groups`: Emojis are ordered in groups (tags). This is an array of key-value pairs where the key is the groupname and the value the location or array of locations. `*` can be used as a wildcard. Example `[Custom: ["/emoji/*.png", "/emoji/custom/*.png"]]`
 * `default_manifest`: Location of the JSON-manifest. This manifest contains information about the emoji-packs you can download. Currently only one manifest can be added (no arrays).
+* `shared_pack_cache_seconds_per_file`: When an emoji pack is shared, the archive is created and cached in
+  memory for this amount of seconds multiplied by the number of files.
 
 ## Database options
 
@@ -658,6 +762,10 @@ This will probably take a long time.
 
 ## :rate_limit
 
+This is an advanced feature and disabled by default.
+
+If your instance is behind a reverse proxy you must enable and configure [`Pleroma.Plugs.RemoteIp`](#pleroma-plugs-remoteip).
+
 A keyword list of rate limiters where a key is a limiter name and value is the limiter configuration. The basic configuration is a tuple where:
 
 * The first element: `scale` (Integer). The time scale in milliseconds.
@@ -665,8 +773,6 @@ A keyword list of rate limiters where a key is a limiter name and value is the l
 
 It is also possible to have different limits for unauthenticated and authenticated users: the keyword value must be a list of two tuples where the first one is a config for unauthenticated users and the second one is for authenticated.
 
-See [`Pleroma.Plugs.RateLimiter`](Pleroma.Plugs.RateLimiter.html) documentation for examples.
-
 Supported rate limiters:
 
 * `:search` for the search requests (account & status search etc.)
@@ -675,3 +781,26 @@ Supported rate limiters:
 * `:relation_id_action` for actions on relation with a specific user (follow, unfollow)
 * `:statuses_actions` for create / delete / fav / unfav / reblog / unreblog actions on any statuses
 * `:status_id_action` for fav / unfav or reblog / unreblog actions on the same status by the same user
+
+## :web_cache_ttl
+
+The expiration time for the web responses cache. Values should be in milliseconds or `nil` to disable expiration.
+
+Available caches:
+
+* `:activity_pub` - activity pub routes (except question activities). Defaults to `nil` (no expiration).
+* `:activity_pub_question` - activity pub routes (question activities). Defaults to `30_000` (30 seconds).
+
+## Pleroma.Plugs.RemoteIp
+
+!!! warning
+    If your instance is not behind at least one reverse proxy, you should not enable this plug.
+
+`Pleroma.Plugs.RemoteIp` is a shim to call [`RemoteIp`](https://git.pleroma.social/pleroma/remote_ip) but with runtime configuration.
+
+Available options:
+
+* `enabled` - Enable/disable the plug. Defaults to `false`.
+* `headers` - A list of strings naming the `req_headers` to use when deriving the `remote_ip`. Order does not matter. Defaults to `~w[forwarded x-forwarded-for x-client-ip x-real-ip]`.
+* `proxies` - A list of strings in [CIDR](https://en.wikipedia.org/wiki/CIDR) notation specifying the IPs of known proxies. Defaults to `[]`.
+* `reserved` - Defaults to [localhost](https://en.wikipedia.org/wiki/Localhost) and [private network](https://en.wikipedia.org/wiki/Private_network).

docs/configuration/custom_emoji.md

@@ -4,6 +4,7 @@ Before you add your own custom emoji, check if they are available in an existing
 See `Mix.Tasks.Pleroma.Emoji` for information about emoji packs.
 
 To add custom emoji:
+
 * Create the `STATIC-DIR/emoji/` directory if it doesn't exist
   (`STATIC-DIR` is configurable, `instance/static/` by default)
 * Create a directory with whatever name you want (custom is a good name to show the purpose of it).

docs/installation/alpine_linux_en.md

@@ -1,7 +1,9 @@
 # Installing on Alpine Linux
 ## Installation
 
-This guide is a step-by-step installation guide for Alpine Linux. It also assumes that you have administrative rights, either as root or a user with [sudo permissions](https://www.linode.com/docs/tools-reference/custom-kernels-distros/install-alpine-linux-on-your-linode/#configuration). If you want to run this guide with root, ignore the `sudo` at the beginning of the lines, unless it calls a user like `sudo -Hu pleroma`; in this case, use `su -l <username> -s $SHELL -c 'command'` instead.
+This guide is a step-by-step installation guide for Alpine Linux. The instructions were verified against Alpine v3.10 standard image. You might miss additional dependencies if you use `netboot` instead.
+
+It assumes that you have administrative rights, either as root or a user with [sudo permissions](https://www.linode.com/docs/tools-reference/custom-kernels-distros/install-alpine-linux-on-your-linode/#configuration). If you want to run this guide with root, ignore the `sudo` at the beginning of the lines, unless it calls a user like `sudo -Hu pleroma`; in this case, use `su -l <username> -s $SHELL -c 'command'` instead.
 
 ### Required packages
 
@@ -20,12 +22,13 @@ This guide is a step-by-step installation guide for Alpine Linux. It also assume
 
 ### Prepare the system
 
-* First make sure to have the community repository enabled:
+* The community repository must be enabled in `/etc/apk/repositories`. Depending on which version and mirror you use this looks like `http://alpine.42.fr/v3.10/community`. If you autogenerated the mirror during installation:
 
 ```shell
-echo "https://nl.alpinelinux.org/alpine/latest-stable/community" | sudo tee -a /etc/apk/repository
+awk 'NR==2' /etc/apk/repositories | sed 's/main/community/' | tee -a /etc/apk/repositories
 ```
 
+
 * Then update the system, if not already done:
 
 ```shell
@@ -77,7 +80,8 @@ sudo rc-update add postgresql
 * Add a new system user for the Pleroma service:
 
 ```shell
-sudo adduser -S -s /bin/false -h /opt/pleroma -H pleroma
+sudo addgroup pleroma
+sudo adduser -S -s /bin/false -h /opt/pleroma -H -G pleroma pleroma
 ```
 
 **Note**: To execute a single command as the Pleroma system user, use `sudo -Hu pleroma command`. You can also switch to a shell by using `sudo -Hu pleroma $SHELL`. If you don’t have and want `sudo` on your system, you can use `su` as root user (UID 0) for a single command by using `su -l pleroma -s $SHELL -c 'command'` and `su -l pleroma -s $SHELL` for starting a shell.
@@ -87,7 +91,7 @@ sudo adduser -S -s /bin/false -h /opt/pleroma -H pleroma
 ```shell
 sudo mkdir -p /opt/pleroma
 sudo chown -R pleroma:pleroma /opt/pleroma
-sudo -Hu pleroma git clone -b master https://git.pleroma.social/pleroma/pleroma /opt/pleroma
+sudo -Hu pleroma git clone -b stable https://git.pleroma.social/pleroma/pleroma /opt/pleroma
 ```
 
 * Change to the new directory:
@@ -164,7 +168,26 @@ If that doesn’t work, make sure, that nginx is not already running. If it stil
 sudo cp /opt/pleroma/installation/pleroma.nginx /etc/nginx/conf.d/pleroma.conf
 ```
 
-* Before starting nginx edit the configuration and change it to your needs (e.g. change servername, change cert paths)
+* Before starting nginx edit the configuration and change it to your needs. You must change change `server_name` and the paths to the certificates. You can use `nano` (install with `apk add nano` if missing).
+
+```
+server {
+    server_name    your.domain;
+    listen         80;
+    ...
+}
+
+server {
+    server_name your.domain;
+    listen 443 ssl http2;
+    ...
+    ssl_trusted_certificate   /etc/letsencrypt/live/your.domain/chain.pem;
+    ssl_certificate           /etc/letsencrypt/live/your.domain/fullchain.pem;
+    ssl_certificate_key       /etc/letsencrypt/live/your.domain/privkey.pem;
+    ...
+}
+```
+
 * Enable and start nginx:
 
 ```shell
@@ -202,12 +225,10 @@ sudo -Hu pleroma MIX_ENV=prod mix pleroma.user new <username> <your@emailaddress
 
 #### Further reading
 
-* [Backup your instance](backup.html)
+* [Backup your instance](../administration/backup.md)
-* [Configuration tips](general-tips-for-customizing-pleroma-fe.html)
+* [Hardening your instance](../configuration/hardening.md)
-* [Hardening your instance](hardening.html)
+* [How to activate mediaproxy](../configuration/howto_mediaproxy.md)
-* [How to activate mediaproxy](howto_mediaproxy.html)
+* [Updating your instance](../administration/updating.md)
-* [Small Pleroma-FE customizations](small_customizations.html)
-* [Updating your instance](updating.html)
 
 ## Questions
 

docs/installation/arch_linux_en.md

@@ -66,7 +66,7 @@ sudo useradd -r -s /bin/false -m -d /var/lib/pleroma -U pleroma
 ```shell
 sudo mkdir -p /opt/pleroma
 sudo chown -R pleroma:pleroma /opt/pleroma
-sudo -Hu pleroma git clone -b master https://git.pleroma.social/pleroma/pleroma /opt/pleroma
+sudo -Hu pleroma git clone -b stable https://git.pleroma.social/pleroma/pleroma /opt/pleroma
 ```
 
 * Change to the new directory:
@@ -200,12 +200,10 @@ sudo -Hu pleroma MIX_ENV=prod mix pleroma.user new <username> <your@emailaddress
 
 #### Further reading
 
-* [Backup your instance](backup.html)
+* [Backup your instance](../administration/backup.md)
-* [Configuration tips](general-tips-for-customizing-pleroma-fe.html)
+* [Hardening your instance](../configuration/hardening.md)
-* [Hardening your instance](hardening.html)
+* [How to activate mediaproxy](../configuration/howto_mediaproxy.md)
-* [How to activate mediaproxy](howto_mediaproxy.html)
+* [Updating your instance](../administration/updating.md)
-* [Small Pleroma-FE customizations](small_customizations.html)
-* [Updating your instance](updating.html)
 
 ## Questions
 

docs/installation/centos7_en.md

@@ -143,7 +143,7 @@ sudo useradd -r -s /bin/false -m -d /var/lib/pleroma -U pleroma
 ```shell
 sudo mkdir -p /opt/pleroma
 sudo chown -R pleroma:pleroma /opt/pleroma
-sudo -Hu pleroma git clone -b master https://git.pleroma.social/pleroma/pleroma /opt/pleroma
+sudo -Hu pleroma git clone -b stable https://git.pleroma.social/pleroma/pleroma /opt/pleroma
 ```
 
 * Change to the new directory:
@@ -264,12 +264,10 @@ sudo -Hu pleroma MIX_ENV=prod mix pleroma.user new <username> <your@emailaddress
 
 #### Further reading
 
-* [Backup your instance](backup.html)
+* [Backup your instance](../administration/backup.md)
-* [Configuration tips](general-tips-for-customizing-pleroma-fe.html)
+* [Hardening your instance](../configuration/hardening.md)
-* [Hardening your instance](hardening.html)
+* [How to activate mediaproxy](../configuration/howto_mediaproxy.md)
-* [How to activate mediaproxy](howto_mediaproxy.html)
+* [Updating your instance](../administration/updating.md)
-* [Small Pleroma-FE customizations](small_customizations.html)
-* [Updating your instance](updating.html)
 
 ## Questions
 

docs/installation/debian_based_en.md

@@ -68,7 +68,7 @@ sudo useradd -r -s /bin/false -m -d /var/lib/pleroma -U pleroma
 ```shell
 sudo mkdir -p /opt/pleroma
 sudo chown -R pleroma:pleroma /opt/pleroma
-sudo -Hu pleroma git clone -b master https://git.pleroma.social/pleroma/pleroma /opt/pleroma
+sudo -Hu pleroma git clone -b stable https://git.pleroma.social/pleroma/pleroma /opt/pleroma
 ```
 
 * Change to the new directory:
@@ -190,12 +190,10 @@ sudo -Hu pleroma MIX_ENV=prod mix pleroma.user new <username> <your@emailaddress
 
 #### Further reading
 
-* [Backup your instance](backup.html)
+* [Backup your instance](../administration/backup.md)
-* [Configuration tips](general-tips-for-customizing-pleroma-fe.html)
+* [Hardening your instance](../configuration/hardening.md)
-* [Hardening your instance](hardening.html)
+* [How to activate mediaproxy](../configuration/howto_mediaproxy.md)
-* [How to activate mediaproxy](howto_mediaproxy.html)
+* [Updating your instance](../administration/updating.md)
-* [Small Pleroma-FE customizations](small_customizations.html)
-* [Updating your instance](updating.html)
 
 ## Questions
 

docs/installation/debian_based_jp.md

@@ -5,187 +5,184 @@
 
 ## インストール
 
-このガイドはDebian Stretchを仮定しています。Ubuntu 16.04でも可能です。
+このガイドはDebian Stretchを利用することを想定しています。Ubuntu 16.04や18.04でもおそらく動作します。また、ユーザはrootもしくはsudoにより管理者権限を持っていることを前提とします。もし、以下の操作をrootユーザで行う場合は、 `sudo` を無視してください。ただし、`sudo -Hu pleroma` のようにユーザを指定している場合には `su <username> -s $SHELL -c 'command'` を代わりに使ってください。
 
 ### 必要なソフトウェア
 
-- PostgreSQL 9.6+ (postgresql-contrib-9.6 または他のバージョンの PSQL をインストールしてください)
+- PostgreSQL 9.6以上 (Ubuntu16.04では9.5しか提供されていないので，[](https://www.postgresql.org/download/linux/ubuntu/)こちらから新しいバージョンを入手してください)
-- Elixir 1.5 以上 ([Debianのリポジトリからインストールしないこと！！！ ここからインストールすること！](https://elixir-lang.org/install.html#unix-and-unix-like))。または [asdf](https://github.com/asdf-vm/asdf) を pleroma ユーザーでインストール。
+- postgresql-contrib 9.6以上 (同上)
-- erlang-dev
+- Elixir 1.5 以上 ([Debianのリポジトリからインストールしないこと！！！ ここからインストールすること！](https://elixir-lang.org/install.html#unix-and-unix-like)。または [asdf](https://github.com/asdf-vm/asdf) をpleromaユーザーでインストールしてください)
+  - erlang-dev
 - erlang-tools
 - erlang-parsetools
+- erlang-eldap (LDAP認証を有効化するときのみ必要)
 - erlang-ssh
-- erlang-xmerl (Jessieではバックポートからインストールすること！)
+- erlang-xmerl
 - git
 - build-essential
-- openssh
+
-- openssl
+#### このガイドで利用している追加パッケージ
-- nginx prefered (Apacheも動くかもしれませんが、誰もテストしていません！)
+
-- certbot (または何らかのACME Let's encryptクライアント)
+- nginx (おすすめです。他のリバースプロキシを使う場合は、参考となる設定をこのリポジトリから探してください)
+- certbot (または何らかのLet's Encrypt向けACMEクライアント)
 
 ### システムを準備する
 
 * まずシステムをアップデートしてください。
 ```
-apt update && apt dist-upgrade
+sudo apt update
+sudo apt full-upgrade
 ```
 
-* 複数のツールとpostgresqlをインストールします。あとで必要になるので。
+* 上記に挙げたパッケージをインストールしておきます。
 ```
-apt install git build-essential openssl ssh sudo postgresql-9.6 postgresql-contrib-9.6
+sudo apt install git build-essential postgresql postgresql-contrib
 ```
-(postgresqlのバージョンは、あなたのディストロにあわせて変えてください。または、バージョン番号がいらないかもしれません。)
+
 
 ### ElixirとErlangをインストールします
 
 * Erlangのリポジトリをダウンロードおよびインストールします。
 ```
-wget -P /tmp/ https://packages.erlang-solutions.com/erlang-solutions_1.0_all.deb && sudo dpkg -i /tmp/erlang-solutions_1.0_all.deb
+wget -P /tmp/ https://packages.erlang-solutions.com/erlang-solutions_1.0_all.deb
+sudo dpkg -i /tmp/erlang-solutions_1.0_all.deb
 ```
 
 * ElixirとErlangをインストールします、
 ```
-apt update && apt install elixir erlang-dev erlang-parsetools erlang-xmerl erlang-tools erlang-ssh
+sudo apt update
+sudo apt install elixir erlang-dev erlang-parsetools erlang-xmerl erlang-tools erlang-ssh
 ```
 
 ### Pleroma BE (バックエンド) をインストールします
 
-*  新しいユーザーを作ります。
+*  Pleroma用に新しいユーザーを作ります。
-```
-adduser pleroma
-``` 
-(Give it any password you want, make it STRONG)
 
-*  新しいユーザーをsudoグループに入れます。
 ```
-usermod -aG sudo pleroma
+sudo useradd -r -s /bin/false -m -d /var/lib/pleroma -U pleroma
 ```
 
-*  新しいユーザーに変身し、ホームディレクトリに移動します。
+**注意**: Pleromaユーザとして単発のコマンドを実行したい場合はは、`sudo -Hu pleroma command` を使ってください。シェルを使いたい場合は `sudo -Hu pleroma $SHELL`です。もし `sudo` を使わない場合は、rootユーザで `su -l pleroma -s $SHELL -c 'command'` とすることでコマンドを、`su -l pleroma -s $SHELL` とすることでシェルを開始できます。
-```
-su pleroma
-cd ~
-```
 
 *  Gitリポジトリをクローンします。
 ```
-git clone -b master https://git.pleroma.social/pleroma/pleroma
+sudo mkdir -p /opt/pleroma
+sudo chown -R pleroma:pleroma /opt/pleroma
+sudo -Hu pleroma git clone -b stable https://git.pleroma.social/pleroma/pleroma /opt/pleroma
 ```
 
 *  新しいディレクトリに移動します。
 ```
-cd pleroma/
+cd /opt/pleroma
 ```
 
 * Pleromaが依存するパッケージをインストールします。Hexをインストールしてもよいか聞かれたら、yesを入力してください。
 ```
-mix deps.get
+sudo -Hu pleroma mix deps.get
 ```
 
 * コンフィギュレーションを生成します。
 ```
-mix pleroma.instance gen
+sudo -Hu pleroma mix pleroma.instance gen
 ```
     * rebar3をインストールしてもよいか聞かれたら、yesを入力してください。
-    * この処理には時間がかかります。私もよく分かりませんが、何らかのコンパイルが行われているようです。
+    * このときにpleromaの一部がコンパイルされるため、この処理には時間がかかります。
-    * あなたのインスタンスについて、いくつかの質問があります。その回答は `config/generated_config.exs` というコンフィギュレーションファイルに保存されます。
+    * あなたのインスタンスについて、いくつかの質問されます。この質問により `config/generated_config.exs` という設定ファイルが生成されます。
 
-**注意**: メディアプロクシを有効にすると回答して、なおかつ、キャッシュのURLは空欄のままにしている場合は、`generated_config.exs` を編集して、`base_url` で始まる行をコメントアウトまたは削除してください。そして、上にある行の `true` の後にあるコンマを消してください。
 
 * コンフィギュレーションを確認して、もし問題なければ、ファイル名を変更してください。
 ```
 mv config/{generated_config.exs,prod.secret.exs}
 ```
 
-* これまでのコマンドで、すでに `config/setup_db.psql` というファイルが作られています。このファイルをもとに、データベースを作成します。
+* 先程のコマンドで、すでに `config/setup_db.psql` というファイルが作られています。このファイルをもとに、データベースを作成します。
 ```
-sudo su postgres -c 'psql -f config/setup_db.psql'
+sudo -Hu pleroma mix pleroma.instance gen
 ```
 
-* そして、データベースのミグレーションを実行します。
+* そして、データベースのマイグレーションを実行します。
 ```
-MIX_ENV=prod mix ecto.migrate
+sudo -Hu pleroma MIX_ENV=prod mix ecto.migrate
 ```
 
-* Pleromaを起動できるようになりました。
+* これでPleromaを起動できるようになりました。
 ```
-MIX_ENV=prod mix phx.server
+sudo -Hu pleroma MIX_ENV=prod mix phx.server
 ```
 
-### インストールを終わらせる
+### インストールの最終段階
 
-あなたの新しいインスタンスを世界に向けて公開するには、nginxまたは何らかのウェブサーバー (プロクシ) を使用する必要があります。また、Pleroma のためにシステムサービスファイルを作成する必要があります。
+あなたの新しいインスタンスを世界に向けて公開するには、nginx等のWebサーバやプロキシサーバをPleromaの前段に使用する必要があります。また、Pleroma のためにシステムサービスファイルを作成する必要があります。
 
 #### Nginx
 
 * まだインストールしていないなら、nginxをインストールします。
 ```
-apt install nginx
+sudo apt install nginx
 ```
 
 * SSLをセットアップします。他の方法でもよいですが、ここではcertbotを説明します。
 certbotを使うならば、まずそれをインストールします。
 ```
-apt install certbot
+sudo apt install certbot
 ```
 そしてセットアップします。
 ```
-mkdir -p /var/lib/letsencrypt/.well-known
+sudo mkdir -p /var/lib/letsencrypt/
-% certbot certonly --email your@emailaddress --webroot -w /var/lib/letsencrypt/ -d yourdomain
+sudo certbot certonly --email <your@emailaddress> -d <yourdomain> --standalone
 ```
-もしうまくいかないときは、先にnginxを設定してください。ssl "on" を "off" に変えてから再試行してください。
+もしうまくいかないときは、nginxが正しく動いていない可能性があります。先にnginxを設定してください。ssl "on" を "off" に変えてから再試行してください。
 
 ---
 
-* nginxコンフィギュレーションの例をnginxフォルダーにコピーします。
+* nginxの設定ファイルサンプルをnginxフォルダーにコピーします。
 ```
-cp /home/pleroma/pleroma/installation/pleroma.nginx /etc/nginx/sites-enabled/pleroma.nginx
+sudo cp /opt/pleroma/installation/pleroma.nginx /etc/nginx/sites-available/pleroma.nginx
+sudo ln -s /etc/nginx/sites-available/pleroma.nginx /etc/nginx/sites-enabled/pleroma.nginx
 ```
 
-* nginxを起動する前に、コンフィギュレーションを編集してください。例えば、サーバー名、証明書のパスなどを変更する必要があります。
+* nginxを起動する前に、設定ファイルを編集してください。例えば、サーバー名、証明書のパスなどを変更する必要があります。
 * nginxを再起動します。
 ```
-systemctl reload nginx.service
+sudo systemctl enable --now nginx.service
 ```
 
+もし証明書を更新する必要が出てきた場合には、nginxの関連するlocationブロックのコメントアウトを外し、以下のコマンドを動かします。
+
+```
+sudo certbot certonly --email <your@emailaddress> -d <yourdomain> --webroot -w /var/lib/letsencrypt/
+```
+
+#### 他のWebサーバやプロキシ
+これに関してはサンプルが `/opt/pleroma/installation/` にあるので、探してみてください。
+
 #### Systemd サービス
 
-* サービスファイルの例をコピーします。
+* サービスファイルのサンプルをコピーします。
 ```
-cp /home/pleroma/pleroma/installation/pleroma.service /usr/lib/systemd/system/pleroma.service
+sudo cp /opt/pleroma/installation/pleroma.service /etc/systemd/system/pleroma.service
 ```
 
-* サービスファイルを変更します。すべてのパスが正しいことを確認してください。また、`[Service]` セクションに以下の行があることを確認してください。
+* サービスファイルを変更します。すべてのパスが正しいことを確認してください
+* サービスを有効化し `pleroma.service` を開始してください
 ```
-Environment="MIX_ENV=prod"
+sudo systemctl enable --now pleroma.service
 ```
 
-* `pleroma.service` を enable および start してください。
+#### 初期ユーザの作成
+
+新たにインスタンスを作成したら、以下のコマンドにより管理者権限を持った初期ユーザを作成できます。
+
 ```
-systemctl enable --now pleroma.service
+sudo -Hu pleroma MIX_ENV=prod mix pleroma.user new <username> <your@emailaddress> --admin
 ```
 
-#### モデレーターを作る
+#### その他の設定とカスタマイズ
 
-新たにユーザーを作ったら、モデレーター権限を与えたいかもしれません。以下のタスクで可能です。
+* [Backup your instance](../administration/backup.md)
-```
+* [Hardening your instance](../configuration/hardening.md)
-mix set_moderator username [true|false]
+* [How to activate mediaproxy](../configuration/howto_mediaproxy.md)
-```
+* [Updating your instance](../administration/updating.md)
-
-モデレーターはすべてのポストを消すことができます。将来的には他のことも可能になるかもしれません。
-
-#### メディアプロクシを有効にする
-
-`generate_config` でメディアプロクシを有効にしているなら、すでにメディアプロクシが動作しています。あとから設定を変更したいなら、[How to activate mediaproxy](How-to-activate-mediaproxy) を見てください。
-
-#### コンフィギュレーションとカスタマイズ
-
-* [Backup your instance](backup.html)
-* [Configuration tips](general-tips-for-customizing-pleroma-fe.html)
-* [Hardening your instance](hardening.html)
-* [How to activate mediaproxy](howto_mediaproxy.html)
-* [Small Pleroma-FE customizations](small_customizations.html)
-* [Updating your instance](updating.html)
 
 ## 質問ある？
 

docs/installation/gentoo_en.md

@@ -106,7 +106,7 @@ It is highly recommended you use your own fork for the `https://path/to/repo` pa
 
 ```shell
  pleroma$ cd ~
- pleroma$ git clone -b master https://path/to/repo
+ pleroma$ git clone -b stable https://path/to/repo
 ```
 
 * Change to the new directory:
@@ -283,12 +283,10 @@ If you opted to allow sudo for the `pleroma` user but would like to remove the a
 
 #### Further reading
 
-* [Backup your instance](backup.html)
+* [Backup your instance](../administration/backup.md)
-* [Configuration tips](general-tips-for-customizing-pleroma-fe.html)
+* [Hardening your instance](../configuration/hardening.md)
-* [Hardening your instance](hardening.html)
+* [How to activate mediaproxy](../configuration/howto_mediaproxy.md)
-* [How to activate mediaproxy](howto_mediaproxy.html)
+* [Updating your instance](../administration/updating.md)
-* [Small Pleroma-FE customizations](small_customizations.html)
-* [Updating your instance](updating.html)
 
 ## Questions
 

docs/installation/migrating_from_source_otp_en.md

@@ -11,7 +11,7 @@ Benefits of OTP releases over from-source installs include:
 * **Faster and less bug-prone mix tasks.** On a from-source install one has to wait untill a new Pleroma node is started for each mix task and they execute outside of the instance context (for example if a user was deleted via a mix task, the instance will have no knowledge of that and continue to display status count and follows before the cache expires). Mix tasks in OTP releases are executed by calling into a running instance via RPC, which solves both of these problems.
 
 ### Sounds great, how do I switch?
-Currently we support Linux machines with GNU (e.g. Debian, Ubuntu) or musl (e.g. Alpine) libc and `x86_64`, `aarch64` or `armv7l` CPUs. If you are unsure, check the [Detecting flavour](otp_en.html#detecting-flavour) section in OTP install guide. If your platform is supported, proceed with the guide, if not check the [My platform is not supported](#my-platform-is-not-supported) section.
+Currently we support Linux machines with GNU (e.g. Debian, Ubuntu) or musl (e.g. Alpine) libc and `x86_64`, `aarch64` or `armv7l` CPUs. If you are unsure, check the [Detecting flavour](otp_en.md#detecting-flavour) section in OTP install guide. If your platform is supported, proceed with the guide, if not check the [My platform is not supported](#my-platform-is-not-supported) section.
 ### I don't think it is worth the effort, can I stay on a from-source install?
 Yes, currently there are no plans to deprecate them.
 
@@ -70,7 +70,7 @@ and then copy custom emojis to `/var/lib/pleroma/static/emoji/custom`.
 
 This is needed because storing custom emojis in the root directory is deprecated, but if you just move them to `/var/lib/pleroma/static/emoji/custom` it will break emoji urls on old posts.
 
-Note that globs have been replaced with `pack_extensions`, so if your emojis are not in png/gif you should [modify the default value](config.html#emoji).
+Note that globs have been replaced with `pack_extensions`, so if your emojis are not in png/gif you should [modify the default value](../configuration/cheatsheet.md#emoji).
 
 ### Moving the config
 ```sh
@@ -86,7 +86,7 @@ mv ~pleroma/config/prod.secret.exs /etc/pleroma/config.exs
 $EDITOR /etc/pleroma/config.exs
 ```
 ## Installing the release
-Before proceeding, get the flavour from [Detecting flavour](otp_en.html#detecting-flavour) section in OTP installation guide.
+Before proceeding, get the flavour from [Detecting flavour](otp_en.md#detecting-flavour) section in OTP installation guide.
 ```sh
 # Delete all files in pleroma user's directory
 rm -r ~pleroma/*
@@ -96,9 +96,9 @@ rm -r ~pleroma/*
 export FLAVOUR="arm64-musl"
 
 # Clone the release build into a temporary directory and unpack it
-# Replace `master` with `develop` if you want to run the develop branch
+# Replace `stable` with `unstable` if you want to run the unstable branch
 su pleroma -s $SHELL -lc "
-curl 'https://git.pleroma.social/api/v4/projects/2/jobs/artifacts/master/download?job=$FLAVOUR' -o /tmp/pleroma.zip
+curl 'https://git.pleroma.social/api/v4/projects/2/jobs/artifacts/stable/download?job=$FLAVOUR' -o /tmp/pleroma.zip
 unzip /tmp/pleroma.zip -d /tmp/
 "
 
@@ -148,6 +148,6 @@ cp -f ~pleroma/installation/init.d/pleroma /etc/init.d/pleroma
 rc-service pleroma start
 ```
 ## Running mix tasks
-Refer to [Running mix tasks](otp_en.html#running-mix-tasks) section from OTP release installation guide.
+Refer to [Running mix tasks](otp_en.md#running-mix-tasks) section from OTP release installation guide.
 ## Updating
-Refer to [Updating](otp_en.html#updating) section from OTP release installation guide.
+Refer to [Updating](otp_en.md#updating) section from OTP release installation guide.

docs/installation/netbsd_en.md

@@ -58,7 +58,7 @@ Clone the repository:
 
 ```
 $ cd /home/pleroma
-$ git clone -b master https://git.pleroma.social/pleroma/pleroma.git
+$ git clone -b stable https://git.pleroma.social/pleroma/pleroma.git
 ```
 
 Configure Pleroma. Note that you need a domain name at this point:

docs/installation/openbsd_en.md

@@ -29,7 +29,7 @@ This creates a "pleroma" login class and sets higher values than default for dat
 Create the \_pleroma user, assign it the pleroma login class and create its home directory (/home/\_pleroma/): `useradd -m -L pleroma _pleroma`
 
 #### Clone pleroma's directory
-Enter a shell as the \_pleroma user. As root, run `su _pleroma -;cd`. Then clone the repository with `git clone -b master https://git.pleroma.social/pleroma/pleroma.git`. Pleroma is now installed in /home/\_pleroma/pleroma/, it will be configured and started at the end of this guide.
+Enter a shell as the \_pleroma user. As root, run `su _pleroma -;cd`. Then clone the repository with `git clone -b stable https://git.pleroma.social/pleroma/pleroma.git`. Pleroma is now installed in /home/\_pleroma/pleroma/, it will be configured and started at the end of this guide.
 
 #### Postgresql
 Start a shell as the \_postgresql user (as root run `su _postgresql -` then run the `initdb` command to initialize postgresql:  

docs/installation/openbsd_fi.md

@@ -44,7 +44,7 @@ Vaihda pleroma-käyttäjään ja mene kotihakemistoosi:
 
 Lataa pleroman lähdekoodi:
 
-`$ git clone -b master https://git.pleroma.social/pleroma/pleroma.git`
+`$ git clone -b stable https://git.pleroma.social/pleroma/pleroma.git`
 
 `$ cd pleroma`
 

docs/installation/otp_en.md

@@ -42,7 +42,7 @@ apk add curl unzip ncurses postgresql postgresql-contrib nginx certbot
 ## Setup
 ### Configuring PostgreSQL
 #### (Optional) Installing RUM indexes
-RUM indexes are an alternative indexing scheme that is not included in PostgreSQL by default. You can read more about them on the [Configuration page](config.html#rum-indexing-for-full-text-search). They are completely optional and most of the time are not worth it, especially if you are running a single user instance (unless you absolutely need ordered search results).
+RUM indexes are an alternative indexing scheme that is not included in PostgreSQL by default. You can read more about them on the [Configuration page](../configuration/cheatsheet.md#rum-indexing-for-full-text-search). They are completely optional and most of the time are not worth it, especially if you are running a single user instance (unless you absolutely need ordered search results).
 
 Debian/Ubuntu (available only on Buster/19.04):
 ```sh
@@ -80,7 +80,7 @@ export FLAVOUR="arm64-musl"
 
 # Clone the release build into a temporary directory and unpack it
 su pleroma -s $SHELL -lc "
-curl 'https://git.pleroma.social/api/v4/projects/2/jobs/artifacts/master/download?job=$FLAVOUR' -o /tmp/pleroma.zip
+curl 'https://git.pleroma.social/api/v4/projects/2/jobs/artifacts/stable/download?job=$FLAVOUR' -o /tmp/pleroma.zip
 unzip /tmp/pleroma.zip -d /tmp/
 "
 
@@ -262,8 +262,8 @@ su pleroma -s $SHELL -lc "./bin/pleroma_ctl migrate"
 But you should **always check the release notes/changelog** in case there are config deprecations, special update steps, etc.
 
 ## Further reading
-* [Configuration](config.html)
+
-* [Pleroma's base config.exs](https://git.pleroma.social/pleroma/pleroma/blob/master/config/config.exs)
+* [Backup your instance](../administration/backup.md)
-* [Hardening your instance](hardening.html)
+* [Hardening your instance](../configuration/hardening.md)
-* [Pleroma Clients](clients.html)
+* [How to activate mediaproxy](../configuration/howto_mediaproxy.md)
-* [Emoji pack manager](Mix.Tasks.Pleroma.Emoji.html)
+* [Updating your instance](../administration/updating.md)

installation/pleroma-mongooseim.cfg

@@ -215,7 +215,9 @@
       ]}
   ]},
 
-  { 5222, ejabberd_c2s, [
+  %% If you want dual stack, you have to clone this entire config stanza
+  %% and change the bind to "::"
+  { {5222, "0.0.0.0"}, ejabberd_c2s, [
 
 			%%
 			%% If TLS is compiled in and you installed a SSL
@@ -246,7 +248,9 @@
   %%			{max_stanza_size, 65536}
   %%		       ]},
 
-  { 5269, ejabberd_s2s_in, [
+  %% If you want dual stack, you have to clone this entire config stanza
+  %% and change the bind to "::"
+  { {5269, "0.0.0.0"}, ejabberd_s2s_in, [
 			   {shaper, s2s_shaper},
 			   {max_stanza_size, 131072},
 			   {protocol_options, ["no_sslv3"]}

installation/pleroma.nginx

@@ -70,6 +70,7 @@ server {
         proxy_set_header Upgrade $http_upgrade;
         proxy_set_header Connection "upgrade";
         proxy_set_header Host $http_host;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
 
         # this is explicitly IPv4 since Pleroma.Web.Endpoint binds on IPv4 only
         # and `localhost.` resolves to [::0] on some systems: see issue #930
@@ -91,6 +92,6 @@ server {
         chunked_transfer_encoding on;
         proxy_ignore_headers Cache-Control;
         proxy_hide_header  Cache-Control;
-        proxy_pass http://localhost:4000;
+        proxy_pass         http://127.0.0.1:4000;
     }
 }

lib/mix/pleroma.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Mix.Pleroma do

lib/mix/tasks/pleroma/benchmark.ex

@@ -27,7 +27,7 @@ defmodule Mix.Tasks.Pleroma.Benchmark do
     })
   end
 
-  def run(["render_timeline", nickname]) do
+  def run(["render_timeline", nickname | _] = args) do
     start_pleroma()
     user = Pleroma.User.get_by_nickname(nickname)
 
@@ -37,33 +37,37 @@ defmodule Mix.Tasks.Pleroma.Benchmark do
       |> Map.put("blocking_user", user)
       |> Map.put("muting_user", user)
       |> Map.put("user", user)
-      |> Map.put("limit", 80)
+      |> Map.put("limit", 4096)
       |> Pleroma.Web.ActivityPub.ActivityPub.fetch_public_activities()
       |> Enum.reverse()
 
     inputs = %{
-      "One activity" => Enum.take_random(activities, 1),
+      "1 activity" => Enum.take_random(activities, 1),
-      "Ten activities" => Enum.take_random(activities, 10),
+      "10 activities" => Enum.take_random(activities, 10),
-      "Twenty activities" => Enum.take_random(activities, 20),
+      "20 activities" => Enum.take_random(activities, 20),
-      "Forty activities" => Enum.take_random(activities, 40),
+      "40 activities" => Enum.take_random(activities, 40),
-      "Eighty activities" => Enum.take_random(activities, 80)
+      "80 activities" => Enum.take_random(activities, 80)
     }
 
+    inputs =
+      if Enum.at(args, 2) == "extended" do
+        Map.merge(inputs, %{
+          "200 activities" => Enum.take_random(activities, 200),
+          "500 activities" => Enum.take_random(activities, 500),
+          "2000 activities" => Enum.take_random(activities, 2000),
+          "4096 activities" => Enum.take_random(activities, 4096)
+        })
+      else
+        inputs
+      end
+
     Benchee.run(
       %{
-        "Parallel rendering" => fn activities ->
-          Pleroma.Web.MastodonAPI.StatusView.render("index.json", %{
-            activities: activities,
-            for: user,
-            as: :activity
-          })
-        end,
         "Standart rendering" => fn activities ->
           Pleroma.Web.MastodonAPI.StatusView.render("index.json", %{
             activities: activities,
             for: user,
-            as: :activity,
+            as: :activity
-            parallel: false
           })
         end
       },

lib/mix/tasks/pleroma/config.ex

@@ -8,18 +8,7 @@ defmodule Mix.Tasks.Pleroma.Config do
   alias Pleroma.Repo
   alias Pleroma.Web.AdminAPI.Config
   @shortdoc "Manages the location of the config"
-  @moduledoc """
+  @moduledoc File.read!("docs/administration/CLI_tasks/config.md")
-  Manages the location of the config.
-
-  ## Transfers config from file to DB.
-
-      mix pleroma.config migrate_to_db
-
-  ## Transfers config from DB to file `config/env.exported_from_db.secret.exs`
-
-      mix pleroma.config migrate_from_db ENV
-  """
-
   def run(["migrate_to_db"]) do
     start_pleroma()
 

lib/mix/tasks/pleroma/count_statuses.ex

@@ -0,0 +1,22 @@
+defmodule Mix.Tasks.Pleroma.CountStatuses do
+  @shortdoc "Re-counts statuses for all users"
+
+  use Mix.Task
+  alias Pleroma.User
+  import Ecto.Query
+
+  def run([]) do
+    Mix.Pleroma.start_pleroma()
+
+    stream =
+      User
+      |> where(local: true)
+      |> Pleroma.Repo.stream()
+
+    Pleroma.Repo.transaction(fn ->
+      Enum.each(stream, &User.update_note_count/1)
+    end)
+
+    Mix.Pleroma.shell_info("Done")
+  end
+end

lib/mix/tasks/pleroma/database.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Mix.Tasks.Pleroma.Database do
@@ -13,34 +13,8 @@ defmodule Mix.Tasks.Pleroma.Database do
   use Mix.Task
 
   @shortdoc "A collection of database related tasks"
-  @moduledoc """
+  @moduledoc File.read!("docs/administration/CLI_tasks/database.md")
-   A collection of database related tasks
 
-   ## Replace embedded objects with their references
-
-   Replaces embedded objects with references to them in the `objects` table. Only needs to be ran once. The reason why this is not a migration is because it could significantly increase the database size after being ran, however after this `VACUUM FULL` will be able to reclaim about 20% (really depends on what is in the database, your mileage may vary) of the db size before the migration.
-
-       mix pleroma.database remove_embedded_objects
-
-    Options:
-    - `--vacuum` - run `VACUUM FULL` after the embedded objects are replaced with their references
-
-  ## Prune old objects from the database
-
-      mix pleroma.database prune_objects
-
-  ## Create a conversation for all existing DMs. Can be safely re-run.
-
-      mix pleroma.database bump_all_conversations
-
-  ## Remove duplicated items from following and update followers count for all users
-
-      mix pleroma.database update_users_following_followers_counts
-
-  ## Fix the pre-existing "likes" collections for all objects
-
-      mix pleroma.database fix_likes_collections
-  """
   def run(["remove_embedded_objects" | args]) do
     {options, [], []} =
       OptionParser.parse(
@@ -54,7 +28,7 @@ defmodule Mix.Tasks.Pleroma.Database do
     Logger.info("Removing embedded objects")
 
     Repo.query!(
-      "update activities set data = jsonb_set(data, '{object}'::text[], data->'object'->'id') where data->'object'->>'id' is not null;",
+      "update activities set data = safe_jsonb_set(data, '{object}'::text[], data->'object'->'id') where data->'object'->>'id' is not null;",
       [],
       timeout: :infinity
     )
@@ -78,9 +52,9 @@ defmodule Mix.Tasks.Pleroma.Database do
   def run(["update_users_following_followers_counts"]) do
     start_pleroma()
 
-    users = Repo.all(User)
+    User
-    Enum.each(users, &User.remove_duplicated_following/1)
+    |> Repo.all()
-    Enum.each(users, &User.update_follower_count/1)
+    |> Enum.each(&User.update_follower_count/1)
   end
 
   def run(["prune_objects" | args]) do
@@ -152,7 +126,7 @@ defmodule Mix.Tasks.Pleroma.Database do
         set: [
           data:
             fragment(
-              "jsonb_set(?, '{likes}', '[]'::jsonb, true)",
+              "safe_jsonb_set(?, '{likes}', '[]'::jsonb, true)",
               object.data
             )
         ]

lib/mix/tasks/pleroma/digest.ex

@@ -2,16 +2,8 @@ defmodule Mix.Tasks.Pleroma.Digest do
   use Mix.Task
 
   @shortdoc "Manages digest emails"
-  @moduledoc """
+  @moduledoc File.read!("docs/administration/CLI_tasks/digest.md")
-  Manages digest emails
 
-  ## Send digest email since given date (user registration date by default)
-  ignoring user activity status.
-
-  ``mix pleroma.digest test <nickname> <since_date>``
-
-  Example: ``mix pleroma.digest test donaldtheduck 2019-05-20``
-  """
   def run(["test", nickname | opts]) do
     Mix.Pleroma.start_pleroma()
 
@@ -27,7 +19,15 @@ defmodule Mix.Tasks.Pleroma.Digest do
 
     patched_user = %{user | last_digest_emailed_at: last_digest_emailed_at}
 
-    _user = Pleroma.DigestEmailWorker.perform(patched_user)
+    with %Swoosh.Email{} = email <- Pleroma.Emails.UserEmail.digest_email(patched_user) do
+      {:ok, _} = Pleroma.Emails.Mailer.deliver(email)
+
       Mix.shell().info("Digest email have been sent to #{nickname} (#{user.email})")
+    else
+      _ ->
+        Mix.shell().info(
+          "Cound't find any mentions for #{nickname} since #{last_digest_emailed_at}"
+        )
+    end
   end
 end

lib/mix/tasks/pleroma/docs.ex

@@ -0,0 +1,42 @@
+defmodule Mix.Tasks.Pleroma.Docs do
+  use Mix.Task
+  import Mix.Pleroma
+
+  @shortdoc "Generates docs from descriptions.exs"
+  @moduledoc """
+  Generates docs from `descriptions.exs`.
+
+  Supports two formats: `markdown` and `json`.
+
+  ## Generate Markdown docs
+
+  `mix pleroma.docs`
+
+  ## Generate JSON docs
+
+  `mix pleroma.docs json`
+  """
+
+  def run(["json"]) do
+    do_run(Pleroma.Docs.JSON)
+  end
+
+  def run(_) do
+    do_run(Pleroma.Docs.Markdown)
+  end
+
+  defp do_run(implementation) do
+    start_pleroma()
+
+    with {descriptions, _paths} <- Mix.Config.eval!("config/description.exs"),
+         {:ok, file_path} <-
+           Pleroma.Docs.Generator.process(
+             implementation,
+             descriptions[:pleroma][:config_description]
+           ) do
+      type = if implementation == Pleroma.Docs.Markdown, do: "Markdown", else: "JSON"
+
+      Mix.shell().info([:green, "#{type} docs successfully generated to #{file_path}."])
+    end
+  end
+end

lib/mix/tasks/pleroma/ecto/ecto.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-onl
 
 defmodule Mix.Tasks.Pleroma.Ecto do

lib/mix/tasks/pleroma/ecto/migrate.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-onl
 
 defmodule Mix.Tasks.Pleroma.Ecto.Migrate do

lib/mix/tasks/pleroma/ecto/rollback.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-onl
 
 defmodule Mix.Tasks.Pleroma.Ecto.Rollback do

lib/mix/tasks/pleroma/emoji.ex

@@ -1,59 +1,12 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Mix.Tasks.Pleroma.Emoji do
   use Mix.Task
 
   @shortdoc "Manages emoji packs"
-  @moduledoc """
+  @moduledoc File.read!("docs/administration/CLI_tasks/emoji.md")
-  Manages emoji packs
-
-  ## ls-packs
-
-      mix pleroma.emoji ls-packs [OPTION...]
-
-  Lists the emoji packs and metadata specified in the manifest.
-
-  ### Options
-
-  - `-m, --manifest PATH/URL` - path to a custom manifest, it can
-    either be an URL starting with `http`, in that case the
-    manifest will be fetched from that address, or a local path
-
-  ## get-packs
-
-      mix pleroma.emoji get-packs [OPTION...] PACKS
-
-  Fetches, verifies and installs the specified PACKS from the
-  manifest into the `STATIC-DIR/emoji/PACK-NAME`
-
-  ### Options
-
-  - `-m, --manifest PATH/URL` - same as ls-packs
-
-  ## gen-pack
-
-      mix pleroma.emoji gen-pack PACK-URL
-
-  Creates a new manifest entry and a file list from the specified
-  remote pack file. Currently, only .zip archives are recognized
-  as remote pack files and packs are therefore assumed to be zip
-  archives. This command is intended to run interactively and will
-  first ask you some basic questions about the pack, then download
-  the remote file and generate an SHA256 checksum for it, then
-  generate an emoji file list for you.
-
-  The manifest entry will either be written to a newly created
-  `index.json` file or appended to the existing one, *replacing*
-  the old pack with the same name if it was in the file previously.
-
-  The file list will be written to the file specified previously,
-  *replacing* that file. You _should_ check that the file list doesn't
-  contain anything you don't need in the pack, that is, anything that is
-  not an emoji (the whole pack is downloaded, but only emoji files
-  are extracted).
-  """
 
   def run(["ls-packs" | args]) do
     Application.ensure_all_started(:hackney)
@@ -158,19 +111,21 @@ defmodule Mix.Tasks.Pleroma.Emoji do
             file_list: files_to_unzip
           )
 
-        IO.puts(IO.ANSI.format(["Writing emoji.txt for ", :bright, pack_name]))
+        IO.puts(IO.ANSI.format(["Writing pack.json for ", :bright, pack_name]))
 
-        emoji_txt_str =
+        pack_json = %{
-          Enum.map(
+          pack: %{
-            files,
+            "license" => pack["license"],
-            fn {shortcode, path} ->
+            "homepage" => pack["homepage"],
-              emojo_path = Path.join("/emoji/#{pack_name}", path)
+            "description" => pack["description"],
-              "#{shortcode}, #{emojo_path}"
+            "fallback-src" => pack["src"],
-            end
+            "fallback-src-sha256" => pack["src_sha256"],
-          )
+            "share-files" => true
-          |> Enum.join("\n")
+          },
+          files: files
+        }
 
-        File.write!(Path.join(pack_path, "emoji.txt"), emoji_txt_str)
+        File.write!(Path.join(pack_path, "pack.json"), Jason.encode!(pack_json, pretty: true))
       else
         IO.puts(IO.ANSI.format([:bright, :red, "No pack named \"#{pack_name}\" found"]))
       end
@@ -235,7 +190,7 @@ defmodule Mix.Tasks.Pleroma.Emoji do
         cwd: tmp_pack_dir
       )
 
-    emoji_map = Pleroma.Emoji.make_shortcode_to_file_map(tmp_pack_dir, exts)
+    emoji_map = Pleroma.Emoji.Loader.make_shortcode_to_file_map(tmp_pack_dir, exts)
 
     File.write!(files_name, Jason.encode!(emoji_map, pretty: true))
 

lib/mix/tasks/pleroma/instance.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Mix.Tasks.Pleroma.Instance do
@@ -7,36 +7,7 @@ defmodule Mix.Tasks.Pleroma.Instance do
   import Mix.Pleroma
 
   @shortdoc "Manages Pleroma instance"
-  @moduledoc """
+  @moduledoc File.read!("docs/administration/CLI_tasks/instance.md")
-  Manages Pleroma instance.
-
-  ## Generate a new instance config.
-
-    mix pleroma.instance gen [OPTION...]
-
-  If any options are left unspecified, you will be prompted interactively
-
-  ## Options
-
-  - `-f`, `--force` - overwrite any output files
-  - `-o PATH`, `--output PATH` - the output file for the generated configuration
-  - `--output-psql PATH` - the output file for the generated PostgreSQL setup
-  - `--domain DOMAIN` - the domain of your instance
-  - `--instance-name INSTANCE_NAME` - the name of your instance
-  - `--admin-email ADMIN_EMAIL` - the email address of the instance admin
-  - `--notify-email NOTIFY_EMAIL` - email address for notifications
-  - `--dbhost HOSTNAME` - the hostname of the PostgreSQL database to use
-  - `--dbname DBNAME` - the name of the database to use
-  - `--dbuser DBUSER` - the user (aka role) to use for the database connection
-  - `--dbpass DBPASS` - the password to use for the database connection
-  - `--rum Y/N` - Whether to enable RUM indexes
-  - `--indexable Y/N` - Allow/disallow indexing site by search engines
-  - `--db-configurable Y/N` - Allow/disallow configuring instance from admin part
-  - `--uploads-dir` - the directory uploads go in when using a local uploader
-  - `--static-dir` - the directory custom public files should be read from (custom emojis, frontend bundle overrides, robots.txt, etc.)
-  - `--listen-ip` - the ip the app should listen to, defaults to 127.0.0.1
-  - `--listen-port` - the port the app should listen to, defaults to 4000
-  """
 
   def run(["gen" | rest]) do
     {options, [], []} =

lib/mix/tasks/pleroma/relay.ex

@@ -1,33 +1,15 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Mix.Tasks.Pleroma.Relay do
   use Mix.Task
   import Mix.Pleroma
-  alias Pleroma.User
   alias Pleroma.Web.ActivityPub.Relay
 
   @shortdoc "Manages remote relays"
-  @moduledoc """
+  @moduledoc File.read!("docs/administration/CLI_tasks/relay.md")
-  Manages remote relays
 
-  ## Follow a remote relay
-
-  ``mix pleroma.relay follow <relay_url>``
-
-  Example: ``mix pleroma.relay follow  https://example.org/relay``
-
-  ## Unfollow a remote relay
-
-  ``mix pleroma.relay unfollow <relay_url>``
-
-  Example: ``mix pleroma.relay unfollow https://example.org/relay``
-
-  ## List relay subscriptions
-
-  ``mix pleroma.relay list``
-  """
   def run(["follow", target]) do
     start_pleroma()
 
@@ -53,15 +35,10 @@ defmodule Mix.Tasks.Pleroma.Relay do
   def run(["list"]) do
     start_pleroma()
 
-    with %User{} = user <- Relay.get_actor() do
+    with {:ok, list} <- Relay.list() do
-      user.following
+      list |> Enum.each(&shell_info(&1))
-      |> Enum.each(fn entry ->
-        URI.parse(entry)
-        |> Map.get(:host)
-        |> shell_info()
-      end)
     else
-      e -> shell_error("Error while fetching relay subscription list: #{inspect(e)}")
+      {:error, e} -> shell_error("Error while fetching relay subscription list: #{inspect(e)}")
     end
   end
 end

lib/mix/tasks/pleroma/uploads.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Mix.Tasks.Pleroma.Uploads do
@@ -12,16 +12,8 @@ defmodule Mix.Tasks.Pleroma.Uploads do
   @log_every 50
 
   @shortdoc "Migrates uploads from local to remote storage"
-  @moduledoc """
+  @moduledoc File.read!("docs/administration/CLI_tasks/uploads.md")
-   Manages uploads
 
-   ## Migrate uploads from local to remote storage
-       mix pleroma.uploads migrate_local TARGET_UPLOADER [OPTIONS...]
-   Options:
-   - `--delete` - delete local uploads after migrating them to the target uploader
-
-   A list of available uploaders can be seen in config.exs
-  """
   def run(["migrate_local", target_uploader | args]) do
     delete? = Enum.member?(args, "--delete")
     start_pleroma()

lib/mix/tasks/pleroma/user.ex

@@ -1,96 +1,18 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Mix.Tasks.Pleroma.User do
   use Mix.Task
-  import Ecto.Changeset
   import Mix.Pleroma
+  alias Ecto.Changeset
   alias Pleroma.User
   alias Pleroma.UserInviteToken
   alias Pleroma.Web.OAuth
 
   @shortdoc "Manages Pleroma users"
-  @moduledoc """
+  @moduledoc File.read!("docs/administration/CLI_tasks/user.md")
-  Manages Pleroma users.
 
-  ## Create a new user.
-
-      mix pleroma.user new NICKNAME EMAIL [OPTION...]
-
-  Options:
-  - `--name NAME` - the user's name (i.e., "Lain Iwakura")
-  - `--bio BIO` - the user's bio
-  - `--password PASSWORD` - the user's password
-  - `--moderator`/`--no-moderator` - whether the user is a moderator
-  - `--admin`/`--no-admin` - whether the user is an admin
-  - `-y`, `--assume-yes`/`--no-assume-yes` - whether to assume yes to all questions
-
-  ## Generate an invite link.
-
-      mix pleroma.user invite [OPTION...]
-
-    Options:
-    - `--expires-at DATE` - last day on which token is active (e.g. "2019-04-05")
-    - `--max-use NUMBER` - maximum numbers of token uses
-
-  ## List generated invites
-
-      mix pleroma.user invites
-
-  ## Revoke invite
-
-      mix pleroma.user revoke_invite TOKEN OR TOKEN_ID
-
-  ## Delete the user's account.
-
-      mix pleroma.user rm NICKNAME
-
-  ## Delete the user's activities.
-
-      mix pleroma.user delete_activities NICKNAME
-
-  ## Sign user out from all applications (delete user's OAuth tokens and authorizations).
-
-      mix pleroma.user sign_out NICKNAME
-
-  ## Deactivate or activate the user's account.
-
-      mix pleroma.user toggle_activated NICKNAME
-
-  ## Unsubscribe local users from user's account and deactivate it
-
-      mix pleroma.user unsubscribe NICKNAME
-
-  ## Unsubscribe local users from an entire instance and deactivate all accounts
-
-      mix pleroma.user unsubscribe_all_from_instance INSTANCE
-
-  ## Create a password reset link.
-
-      mix pleroma.user reset_password NICKNAME
-
-  ## Set the value of the given user's settings.
-
-      mix pleroma.user set NICKNAME [OPTION...]
-
-  Options:
-  - `--locked`/`--no-locked` - whether the user's account is locked
-  - `--moderator`/`--no-moderator` - whether the user is a moderator
-  - `--admin`/`--no-admin` - whether the user is an admin
-
-  ## Add tags to a user.
-
-      mix pleroma.user tag NICKNAME TAGS
-
-  ## Delete tags from a user.
-
-      mix pleroma.user untag NICKNAME TAGS
-
-  ## Toggle confirmation of the user's account.
-
-      mix pleroma.user toggle_confirmed NICKNAME
-  """
   def run(["new", nickname, email | rest]) do
     {options, [], []} =
       OptionParser.parse(
@@ -188,10 +110,10 @@ defmodule Mix.Tasks.Pleroma.User do
     start_pleroma()
 
     with %User{} = user <- User.get_cached_by_nickname(nickname) do
-      {:ok, user} = User.deactivate(user, !user.info.deactivated)
+      {:ok, user} = User.deactivate(user, !user.deactivated)
 
       shell_info(
-        "Activation status of #{nickname}: #{if(user.info.deactivated, do: "de", else: "")}activated"
+        "Activation status of #{nickname}: #{if(user.deactivated, do: "de", else: "")}activated"
       )
     else
       _ ->
@@ -228,9 +150,9 @@ defmodule Mix.Tasks.Pleroma.User do
       shell_info("Deactivating #{user.nickname}")
       User.deactivate(user)
 
-      {:ok, friends} = User.get_friends(user)
+      user
-
+      |> User.get_friends()
-      Enum.each(friends, fn friend ->
+      |> Enum.each(fn friend ->
         user = User.get_cached_by_id(user.id)
 
         shell_info("Unsubscribing #{friend.nickname} from #{user.nickname}")
@@ -241,7 +163,7 @@ defmodule Mix.Tasks.Pleroma.User do
 
       user = User.get_cached_by_id(user.id)
 
-      if Enum.empty?(user.following) do
+      if Enum.empty?(User.get_friends(user)) do
         shell_info("Successfully unsubscribed all followers from #{user.nickname}")
       end
     else
@@ -405,7 +327,7 @@ defmodule Mix.Tasks.Pleroma.User do
     start_pleroma()
 
     with %User{local: true} = user <- User.get_cached_by_nickname(nickname) do
-      {:ok, _} = User.delete_user_activities(user)
+      User.delete_user_activities(user)
       shell_info("User #{nickname} statuses deleted.")
     else
       _ ->
@@ -419,7 +341,7 @@ defmodule Mix.Tasks.Pleroma.User do
     with %User{} = user <- User.get_cached_by_nickname(nickname) do
       {:ok, user} = User.toggle_confirmation(user)
 
-      message = if user.info.confirmation_pending, do: "needs", else: "doesn't need"
+      message = if user.confirmation_pending, do: "needs", else: "doesn't need"
 
       shell_info("#{nickname} #{message} confirmation.")
     else
@@ -443,41 +365,32 @@ defmodule Mix.Tasks.Pleroma.User do
   end
 
   defp set_moderator(user, value) do
-    info_cng = User.Info.admin_api_update(user.info, %{is_moderator: value})
+    {:ok, user} =
+      user
+      |> Changeset.change(%{is_moderator: value})
+      |> User.update_and_set_cache()
 
-    user_cng =
+    shell_info("Moderator status of #{user.nickname}: #{user.is_moderator}")
-      Ecto.Changeset.change(user)
-      |> put_embed(:info, info_cng)
-
-    {:ok, user} = User.update_and_set_cache(user_cng)
-
-    shell_info("Moderator status of #{user.nickname}: #{user.info.is_moderator}")
     user
   end
 
   defp set_admin(user, value) do
-    info_cng = User.Info.admin_api_update(user.info, %{is_admin: value})
+    {:ok, user} =
+      user
+      |> Changeset.change(%{is_admin: value})
+      |> User.update_and_set_cache()
 
-    user_cng =
+    shell_info("Admin status of #{user.nickname}: #{user.is_admin}")
-      Ecto.Changeset.change(user)
-      |> put_embed(:info, info_cng)
-
-    {:ok, user} = User.update_and_set_cache(user_cng)
-
-    shell_info("Admin status of #{user.nickname}: #{user.info.is_admin}")
     user
   end
 
   defp set_locked(user, value) do
-    info_cng = User.Info.user_upgrade(user.info, %{locked: value})
+    {:ok, user} =
+      user
+      |> Changeset.change(%{locked: value})
+      |> User.update_and_set_cache()
 
-    user_cng =
+    shell_info("Locked status of #{user.nickname}: #{user.locked}")
-      Ecto.Changeset.change(user)
-      |> put_embed(:info, info_cng)
-
-    {:ok, user} = User.update_and_set_cache(user_cng)
-
-    shell_info("Locked status of #{user.nickname}: #{user.info.locked}")
     user
   end
 end

lib/pleroma/activity.ex

@@ -6,6 +6,8 @@ defmodule Pleroma.Activity do
   use Ecto.Schema
 
   alias Pleroma.Activity
+  alias Pleroma.Activity.Queries
+  alias Pleroma.ActivityExpiration
   alias Pleroma.Bookmark
   alias Pleroma.Notification
   alias Pleroma.Object
@@ -19,7 +21,7 @@ defmodule Pleroma.Activity do
   @type t :: %__MODULE__{}
   @type actor :: String.t()
 
-  @primary_key {:id, Pleroma.FlakeId, autogenerate: true}
+  @primary_key {:id, FlakeId.Ecto.CompatType, autogenerate: true}
 
   # https://github.com/tootsuite/mastodon/blob/master/app/models/notification.rb#L19
   @mastodon_notification_types %{
@@ -59,11 +61,13 @@ defmodule Pleroma.Activity do
     # typical case.
     has_one(:object, Object, on_delete: :nothing, foreign_key: :id)
 
+    has_one(:expiration, ActivityExpiration, on_delete: :delete_all)
+
     timestamps()
   end
 
-  def with_joined_object(query) do
+  def with_joined_object(query, join_type \\ :inner) do
-    join(query, :inner, [activity], o in Object,
+    join(query, join_type, [activity], o in Object,
       on:
         fragment(
           "(?->>'id') = COALESCE(?->'object'->>'id', ?->>'object')",
@@ -75,10 +79,10 @@ defmodule Pleroma.Activity do
     )
   end
 
-  def with_preloaded_object(query) do
+  def with_preloaded_object(query, join_type \\ :inner) do
     query
     |> has_named_binding?(:object)
-    |> if(do: query, else: with_joined_object(query))
+    |> if(do: query, else: with_joined_object(query, join_type))
     |> preload([activity, object: object], object: object)
   end
 
@@ -96,6 +100,7 @@ defmodule Pleroma.Activity do
     from([a] in query,
       left_join: tm in ThreadMute,
       on: tm.user_id == ^user.id and tm.context == fragment("?->>'context'", a.data),
+      as: :thread_mute,
       select: %Activity{a | thread_muted?: not is_nil(tm.id)}
     )
   end
@@ -103,12 +108,9 @@ defmodule Pleroma.Activity do
   def with_set_thread_muted_field(query, _), do: query
 
   def get_by_ap_id(ap_id) do
-    Repo.one(
+    ap_id
-      from(
+    |> Queries.by_ap_id()
-        activity in Activity,
+    |> Repo.one()
-        where: fragment("(?)->>'id' = ?", activity.data, ^to_string(ap_id))
-      )
-    )
   end
 
   def get_bookmark(%Activity{} = activity, %User{} = user) do
@@ -129,91 +131,55 @@ defmodule Pleroma.Activity do
   end
 
   def get_by_ap_id_with_object(ap_id) do
-    Repo.one(
+    ap_id
-      from(
+    |> Queries.by_ap_id()
-        activity in Activity,
+    |> with_preloaded_object(:left)
-        where: fragment("(?)->>'id' = ?", activity.data, ^to_string(ap_id)),
+    |> Repo.one()
-        left_join: o in Object,
-        on:
-          fragment(
-            "(?->>'id') = COALESCE(?->'object'->>'id', ?->>'object')",
-            o.data,
-            activity.data,
-            activity.data
-          ),
-        preload: [object: o]
-      )
-    )
   end
 
+  @spec get_by_id(String.t()) :: Activity.t() | nil
   def get_by_id(id) do
+    case FlakeId.flake_id?(id) do
+      true ->
         Activity
         |> where([a], a.id == ^id)
         |> restrict_deactivated_users()
         |> Repo.one()
+
+      _ ->
+        nil
+    end
   end
 
   def get_by_id_with_object(id) do
-    from(activity in Activity,
+    Activity
-      where: activity.id == ^id,
+    |> where(id: ^id)
-      inner_join: o in Object,
+    |> with_preloaded_object()
-      on:
-        fragment(
-          "(?->>'id') = COALESCE(?->'object'->>'id', ?->>'object')",
-          o.data,
-          activity.data,
-          activity.data
-        ),
-      preload: [object: o]
-    )
     |> Repo.one()
   end
 
-  def by_object_ap_id(ap_id) do
+  def all_by_ids_with_object(ids) do
-    from(
+    Activity
-      activity in Activity,
+    |> where([a], a.id in ^ids)
-      where:
+    |> with_preloaded_object()
-        fragment(
+    |> Repo.all()
-          "coalesce((?)->'object'->>'id', (?)->>'object') = ?",
-          activity.data,
-          activity.data,
-          ^to_string(ap_id)
-        )
-    )
   end
 
-  def create_by_object_ap_id(ap_ids) when is_list(ap_ids) do
+  @doc """
-    from(
+  Accepts `ap_id` or list of `ap_id`.
-      activity in Activity,
+  Returns a query.
-      where:
+  """
-        fragment(
+  @spec create_by_object_ap_id(String.t() | [String.t()]) :: Ecto.Queryable.t()
-          "coalesce((?)->'object'->>'id', (?)->>'object') = ANY(?)",
+  def create_by_object_ap_id(ap_id) do
-          activity.data,
+    ap_id
-          activity.data,
+    |> Queries.by_object_id()
-          ^ap_ids
+    |> Queries.by_type("Create")
-        ),
-      where: fragment("(?)->>'type' = 'Create'", activity.data)
-    )
   end
 
-  def create_by_object_ap_id(ap_id) when is_binary(ap_id) do
-    from(
-      activity in Activity,
-      where:
-        fragment(
-          "coalesce((?)->'object'->>'id', (?)->>'object') = ?",
-          activity.data,
-          activity.data,
-          ^to_string(ap_id)
-        ),
-      where: fragment("(?)->>'type' = 'Create'", activity.data)
-    )
-  end
-
-  def create_by_object_ap_id(_), do: nil
-
   def get_all_create_by_object_ap_id(ap_id) do
-    Repo.all(create_by_object_ap_id(ap_id))
+    ap_id
+    |> create_by_object_ap_id()
+    |> Repo.all()
   end
 
   def get_create_by_object_ap_id(ap_id) when is_binary(ap_id) do
@@ -224,54 +190,17 @@ defmodule Pleroma.Activity do
 
   def get_create_by_object_ap_id(_), do: nil
 
-  def create_by_object_ap_id_with_object(ap_ids) when is_list(ap_ids) do
+  @doc """
-    from(
+  Accepts `ap_id` or list of `ap_id`.
-      activity in Activity,
+  Returns a query.
-      where:
+  """
-        fragment(
+  @spec create_by_object_ap_id_with_object(String.t() | [String.t()]) :: Ecto.Queryable.t()
-          "coalesce((?)->'object'->>'id', (?)->>'object') = ANY(?)",
+  def create_by_object_ap_id_with_object(ap_id) do
-          activity.data,
+    ap_id
-          activity.data,
+    |> create_by_object_ap_id()
-          ^ap_ids
+    |> with_preloaded_object()
-        ),
-      where: fragment("(?)->>'type' = 'Create'", activity.data),
-      inner_join: o in Object,
-      on:
-        fragment(
-          "(?->>'id') = COALESCE(?->'object'->>'id', ?->>'object')",
-          o.data,
-          activity.data,
-          activity.data
-        ),
-      preload: [object: o]
-    )
   end
 
-  def create_by_object_ap_id_with_object(ap_id) when is_binary(ap_id) do
-    from(
-      activity in Activity,
-      where:
-        fragment(
-          "coalesce((?)->'object'->>'id', (?)->>'object') = ?",
-          activity.data,
-          activity.data,
-          ^to_string(ap_id)
-        ),
-      where: fragment("(?)->>'type' = 'Create'", activity.data),
-      inner_join: o in Object,
-      on:
-        fragment(
-          "(?->>'id') = COALESCE(?->'object'->>'id', ?->>'object')",
-          o.data,
-          activity.data,
-          activity.data
-        ),
-      preload: [object: o]
-    )
-  end
-
-  def create_by_object_ap_id_with_object(_), do: nil
-
   def get_create_by_object_ap_id_with_object(ap_id) when is_binary(ap_id) do
     ap_id
     |> create_by_object_ap_id_with_object()
@@ -295,7 +224,8 @@ defmodule Pleroma.Activity do
   def normalize(_), do: nil
 
   def delete_by_ap_id(id) when is_binary(id) do
-    by_object_ap_id(id)
+    id
+    |> Queries.by_object_id()
     |> select([u], u)
     |> Repo.delete_all()
     |> elem(1)
@@ -304,10 +234,19 @@ defmodule Pleroma.Activity do
       %{data: %{"type" => "Create", "object" => %{"id" => ap_id}}} -> ap_id == id
       _ -> nil
     end)
+    |> purge_web_resp_cache()
   end
 
   def delete_by_ap_id(_), do: nil
 
+  defp purge_web_resp_cache(%Activity{} = activity) do
+    %{path: path} = URI.parse(activity.data["id"])
+    Cachex.del(:web_resp_cache, path)
+    activity
+  end
+
+  defp purge_web_resp_cache(nil), do: nil
+
   for {ap_type, type} <- @mastodon_notification_types do
     def mastodon_notification_type(%Activity{data: %{"type" => unquote(ap_type)}}),
       do: unquote(type)
@@ -330,40 +269,19 @@ defmodule Pleroma.Activity do
   end
 
   def follow_requests_for_actor(%Pleroma.User{ap_id: ap_id}) do
-    from(
+    ap_id
-      a in Activity,
+    |> Queries.by_object_id()
-      where:
+    |> Queries.by_type("Follow")
-        fragment(
+    |> where([a], fragment("? ->> 'state' = 'pending'", a.data))
-          "? ->> 'type' = 'Follow'",
-          a.data
-        ),
-      where:
-        fragment(
-          "? ->> 'state' = 'pending'",
-          a.data
-        ),
-      where:
-        fragment(
-          "coalesce((?)->'object'->>'id', (?)->>'object') = ?",
-          a.data,
-          a.data,
-          ^ap_id
-        )
-    )
-  end
-
-  @spec query_by_actor(actor()) :: Ecto.Query.t()
-  def query_by_actor(actor) do
-    from(a in Activity, where: a.actor == ^actor)
   end
 
   def restrict_deactivated_users(query) do
+    deactivated_users =
+      from(u in User.Query.build(deactivated: true), select: u.ap_id)
+      |> Repo.all()
+
     from(activity in query,
-      where:
+      where: activity.actor not in ^deactivated_users
-        fragment(
-          "? not in (SELECT ap_id FROM users WHERE info->'deactivated' @> 'true')",
-          activity.actor
-        )
     )
   end
 

lib/pleroma/activity/ir/topics.ex

@@ -0,0 +1,63 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Activity.Ir.Topics do
+  alias Pleroma.Object
+  alias Pleroma.Web.ActivityPub.Visibility
+
+  def get_activity_topics(activity) do
+    activity
+    |> Object.normalize()
+    |> generate_topics(activity)
+    |> List.flatten()
+  end
+
+  defp generate_topics(%{data: %{"type" => "Answer"}}, _) do
+    []
+  end
+
+  defp generate_topics(object, activity) do
+    ["user", "list"] ++ visibility_tags(object, activity)
+  end
+
+  defp visibility_tags(object, activity) do
+    case Visibility.get_visibility(activity) do
+      "public" ->
+        if activity.local do
+          ["public", "public:local"]
+        else
+          ["public"]
+        end
+        |> item_creation_tags(object, activity)
+
+      "direct" ->
+        ["direct"]
+
+      _ ->
+        []
+    end
+  end
+
+  defp item_creation_tags(tags, %{data: %{"type" => "Create"}} = object, activity) do
+    tags ++ hashtags_to_topics(object) ++ attachment_topics(object, activity)
+  end
+
+  defp item_creation_tags(tags, _, _) do
+    tags
+  end
+
+  defp hashtags_to_topics(%{data: %{"tag" => tags}}) do
+    tags
+    |> Enum.filter(&is_bitstring(&1))
+    |> Enum.map(fn tag -> "hashtag:" <> tag end)
+  end
+
+  defp hashtags_to_topics(_), do: []
+
+  defp attachment_topics(%{data: %{"attachment" => []}}, _act), do: []
+
+  defp attachment_topics(_object, %{local: true}), do: ["public:media", "public:local:media"]
+
+  defp attachment_topics(_object, _act), do: ["public:media"]
+end

lib/pleroma/activity/queries.ex

@@ -0,0 +1,67 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Activity.Queries do
+  @moduledoc """
+  Contains queries for Activity.
+  """
+
+  import Ecto.Query, only: [from: 2]
+
+  @type query :: Ecto.Queryable.t() | Activity.t()
+
+  alias Pleroma.Activity
+
+  @spec by_ap_id(query, String.t()) :: query
+  def by_ap_id(query \\ Activity, ap_id) do
+    from(
+      activity in query,
+      where: fragment("(?)->>'id' = ?", activity.data, ^to_string(ap_id))
+    )
+  end
+
+  @spec by_actor(query, String.t()) :: query
+  def by_actor(query \\ Activity, actor) do
+    from(
+      activity in query,
+      where: fragment("(?)->>'actor' = ?", activity.data, ^actor)
+    )
+  end
+
+  @spec by_object_id(query, String.t() | [String.t()]) :: query
+  def by_object_id(query \\ Activity, object_id)
+
+  def by_object_id(query, object_ids) when is_list(object_ids) do
+    from(
+      activity in query,
+      where:
+        fragment(
+          "coalesce((?)->'object'->>'id', (?)->>'object') = ANY(?)",
+          activity.data,
+          activity.data,
+          ^object_ids
+        )
+    )
+  end
+
+  def by_object_id(query, object_id) when is_binary(object_id) do
+    from(activity in query,
+      where:
+        fragment(
+          "coalesce((?)->'object'->>'id', (?)->>'object') = ?",
+          activity.data,
+          activity.data,
+          ^object_id
+        )
+    )
+  end
+
+  @spec by_type(query, String.t()) :: query
+  def by_type(query \\ Activity, activity_type) do
+    from(
+      activity in query,
+      where: fragment("(?)->>'type' = ?", activity.data, ^activity_type)
+    )
+  end
+end

lib/pleroma/activity_expiration.ex

@@ -0,0 +1,67 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.ActivityExpiration do
+  use Ecto.Schema
+
+  alias Pleroma.Activity
+  alias Pleroma.ActivityExpiration
+  alias Pleroma.Repo
+
+  import Ecto.Changeset
+  import Ecto.Query
+
+  @type t :: %__MODULE__{}
+  @min_activity_lifetime :timer.hours(1)
+
+  schema "activity_expirations" do
+    belongs_to(:activity, Activity, type: FlakeId.Ecto.CompatType)
+    field(:scheduled_at, :naive_datetime)
+  end
+
+  def changeset(%ActivityExpiration{} = expiration, attrs) do
+    expiration
+    |> cast(attrs, [:scheduled_at])
+    |> validate_required([:scheduled_at])
+    |> validate_scheduled_at()
+  end
+
+  def get_by_activity_id(activity_id) do
+    ActivityExpiration
+    |> where([exp], exp.activity_id == ^activity_id)
+    |> Repo.one()
+  end
+
+  def create(%Activity{} = activity, scheduled_at) do
+    %ActivityExpiration{activity_id: activity.id}
+    |> changeset(%{scheduled_at: scheduled_at})
+    |> Repo.insert()
+  end
+
+  def due_expirations(offset \\ 0) do
+    naive_datetime =
+      NaiveDateTime.utc_now()
+      |> NaiveDateTime.add(offset, :millisecond)
+
+    ActivityExpiration
+    |> where([exp], exp.scheduled_at < ^naive_datetime)
+    |> Repo.all()
+  end
+
+  def validate_scheduled_at(changeset) do
+    validate_change(changeset, :scheduled_at, fn _, scheduled_at ->
+      if not expires_late_enough?(scheduled_at) do
+        [scheduled_at: "an ephemeral activity must live for at least one hour"]
+      else
+        []
+      end
+    end)
+  end
+
+  def expires_late_enough?(scheduled_at) do
+    now = NaiveDateTime.utc_now()
+    diff = NaiveDateTime.diff(scheduled_at, now, :millisecond)
+    diff >= @min_activity_lifetime
+  end
+end

lib/pleroma/application.ex

@@ -31,33 +31,21 @@ defmodule Pleroma.Application do
     children =
       [
         Pleroma.Repo,
+        Pleroma.Scheduler,
         Pleroma.Config.TransferTask,
         Pleroma.Emoji,
         Pleroma.Captcha,
-        Pleroma.FlakeId,
+        Pleroma.Daemons.ScheduledActivityDaemon,
-        Pleroma.ScheduledActivityWorker
+        Pleroma.Daemons.ActivityExpirationDaemon
       ] ++
         cachex_children() ++
         hackney_pool_children() ++
         [
-          Pleroma.Web.Federator.RetryQueue,
           Pleroma.Stats,
-          %{
+          Pleroma.JobQueueMonitor,
-            id: :web_push_init,
+          {Oban, Pleroma.Config.get(Oban)}
-            start: {Task, :start_link, [&Pleroma.Web.Push.init/0]},
-            restart: :temporary
-          },
-          %{
-            id: :federator_init,
-            start: {Task, :start_link, [&Pleroma.Web.Federator.init/0]},
-            restart: :temporary
-          },
-          %{
-            id: :internal_fetch_init,
-            start: {Task, :start_link, [&Pleroma.Web.ActivityPub.InternalFetchActor.init/0]},
-            restart: :temporary
-          }
         ] ++
+        task_children(@env) ++
         oauth_cleanup_child(oauth_cleanup_enabled?()) ++
         streamer_child(@env) ++
         chat_child(@env, chat_enabled?()) ++
@@ -69,9 +57,7 @@ defmodule Pleroma.Application do
     # See http://elixir-lang.org/docs/stable/elixir/Supervisor.html
     # for other strategies and supported options
     opts = [strategy: :one_for_one, name: Pleroma.Supervisor]
-    result = Supervisor.start_link(children, opts)
+    Supervisor.start_link(children, opts)
-    :ok = after_supervisor_start()
-    result
   end
 
   defp setup_instrumenters do
@@ -115,10 +101,16 @@ defmodule Pleroma.Application do
       build_cachex("object", default_ttl: 25_000, ttl_interval: 1000, limit: 2500),
       build_cachex("rich_media", default_ttl: :timer.minutes(120), limit: 5000),
       build_cachex("scrubber", limit: 2500),
-      build_cachex("idempotency", expiration: idempotency_expiration(), limit: 2500)
+      build_cachex("idempotency", expiration: idempotency_expiration(), limit: 2500),
+      build_cachex("web_resp", limit: 2500),
+      build_cachex("emoji_packs", expiration: emoji_packs_expiration(), limit: 10),
+      build_cachex("failed_proxy_url", limit: 2500)
     ]
   end
 
+  defp emoji_packs_expiration,
+    do: expiration(default: :timer.seconds(5 * 60), interval: :timer.seconds(60))
+
   defp idempotency_expiration,
     do: expiration(default: :timer.seconds(6 * 60 * 60), interval: :timer.seconds(60))
 
@@ -140,7 +132,7 @@ defmodule Pleroma.Application do
   defp streamer_child(:test), do: []
 
   defp streamer_child(_) do
-    [Pleroma.Web.Streamer]
+    [Pleroma.Web.Streamer.supervisor()]
   end
 
   defp oauth_cleanup_child(true),
@@ -163,16 +155,28 @@ defmodule Pleroma.Application do
     end
   end
 
-  defp after_supervisor_start do
+  defp task_children(:test) do
-    with digest_config <- Application.get_env(:pleroma, :email_notifications)[:digest],
+    [
-         true <- digest_config[:active] do
+      %{
-      PleromaJobQueue.schedule(
+        id: :web_push_init,
-        digest_config[:schedule],
+        start: {Task, :start_link, [&Pleroma.Web.Push.init/0]},
-        :digest_emails,
+        restart: :temporary
-        Pleroma.DigestEmailWorker
+      }
-      )
+    ]
   end
 
-    :ok
+  defp task_children(_) do
+    [
+      %{
+        id: :web_push_init,
+        start: {Task, :start_link, [&Pleroma.Web.Push.init/0]},
+        restart: :temporary
+      },
+      %{
+        id: :internal_fetch_init,
+        start: {Task, :start_link, [&Pleroma.Web.ActivityPub.InternalFetchActor.init/0]},
+        restart: :temporary
+      }
+    ]
   end
 end

lib/pleroma/bbs/handler.ex

@@ -5,6 +5,7 @@
 defmodule Pleroma.BBS.Handler do
   use Sshd.ShellHandler
   alias Pleroma.Activity
+  alias Pleroma.HTML
   alias Pleroma.Web.ActivityPub.ActivityPub
   alias Pleroma.Web.CommonAPI
 
@@ -42,9 +43,9 @@ defmodule Pleroma.BBS.Handler do
   end
 
   def puts_activity(activity) do
-    status = Pleroma.Web.MastodonAPI.StatusView.render("status.json", %{activity: activity})
+    status = Pleroma.Web.MastodonAPI.StatusView.render("show.json", %{activity: activity})
     IO.puts("-- #{status.id} by #{status.account.display_name} (#{status.account.acct})")
-    IO.puts(HtmlSanitizeEx.strip_tags(status.content))
+    IO.puts(HTML.strip_tags(status.content))
     IO.puts("")
   end
 
@@ -97,7 +98,7 @@ defmodule Pleroma.BBS.Handler do
       |> Map.put("user", user)
 
     activities =
-      [user.ap_id | user.following]
+      [user.ap_id | Pleroma.User.following(user)]
       |> ActivityPub.fetch_activities(params)
 
     Enum.each(activities, fn activity ->

lib/pleroma/bookmark.ex

@@ -10,20 +10,20 @@ defmodule Pleroma.Bookmark do
 
   alias Pleroma.Activity
   alias Pleroma.Bookmark
-  alias Pleroma.FlakeId
   alias Pleroma.Repo
   alias Pleroma.User
 
   @type t :: %__MODULE__{}
 
   schema "bookmarks" do
-    belongs_to(:user, User, type: FlakeId)
+    belongs_to(:user, User, type: FlakeId.Ecto.CompatType)
-    belongs_to(:activity, Activity, type: FlakeId)
+    belongs_to(:activity, Activity, type: FlakeId.Ecto.CompatType)
 
     timestamps()
   end
 
-  @spec create(FlakeId.t(), FlakeId.t()) :: {:ok, Bookmark.t()} | {:error, Changeset.t()}
+  @spec create(FlakeId.Ecto.CompatType.t(), FlakeId.Ecto.CompatType.t()) ::
+          {:ok, Bookmark.t()} | {:error, Changeset.t()}
   def create(user_id, activity_id) do
     attrs = %{
       user_id: user_id,
@@ -37,7 +37,7 @@ defmodule Pleroma.Bookmark do
     |> Repo.insert()
   end
 
-  @spec for_user_query(FlakeId.t()) :: Ecto.Query.t()
+  @spec for_user_query(FlakeId.Ecto.CompatType.t()) :: Ecto.Query.t()
   def for_user_query(user_id) do
     Bookmark
     |> where(user_id: ^user_id)
@@ -52,7 +52,8 @@ defmodule Pleroma.Bookmark do
     |> Repo.one()
   end
 
-  @spec destroy(FlakeId.t(), FlakeId.t()) :: {:ok, Bookmark.t()} | {:error, Changeset.t()}
+  @spec destroy(FlakeId.Ecto.CompatType.t(), FlakeId.Ecto.CompatType.t()) ::
+          {:ok, Bookmark.t()} | {:error, Changeset.t()}
   def destroy(user_id, activity_id) do
     from(b in Bookmark,
       where: b.user_id == ^user_id,

lib/pleroma/constants.ex

@@ -6,4 +6,16 @@ defmodule Pleroma.Constants do
   use Const
 
   const(as_public, do: "https://www.w3.org/ns/activitystreams#Public")
+
+  const(object_internal_fields,
+    do: [
+      "likes",
+      "like_count",
+      "announcements",
+      "announcement_count",
+      "emoji",
+      "context_id",
+      "deleted_activity_id"
+    ]
+  )
 end

lib/pleroma/conversation.ex

@@ -67,6 +67,14 @@ defmodule Pleroma.Conversation do
 
       participations =
         Enum.map(users, fn user ->
+          invisible_conversation = Enum.any?(users, &User.blocks?(user, &1))
+
+          unless invisible_conversation do
+            User.increment_unread_conversation_count(conversation, user)
+          end
+
+          opts = Keyword.put(opts, :invisible_conversation, invisible_conversation)
+
           {:ok, participation} =
             Participation.create_for_user_and_conversation(user, conversation, opts)
 

lib/pleroma/conversation/participation.ex

@@ -13,10 +13,10 @@ defmodule Pleroma.Conversation.Participation do
   import Ecto.Query
 
   schema "conversation_participations" do
-    belongs_to(:user, User, type: Pleroma.FlakeId)
+    belongs_to(:user, User, type: FlakeId.Ecto.CompatType)
     belongs_to(:conversation, Conversation)
     field(:read, :boolean, default: false)
-    field(:last_activity_id, Pleroma.FlakeId, virtual: true)
+    field(:last_activity_id, FlakeId.Ecto.CompatType, virtual: true)
 
     has_many(:recipient_ships, RecipientShip)
     has_many(:recipients, through: [:recipient_ships, :user])
@@ -32,11 +32,20 @@ defmodule Pleroma.Conversation.Participation do
 
   def create_for_user_and_conversation(user, conversation, opts \\ []) do
     read = !!opts[:read]
+    invisible_conversation = !!opts[:invisible_conversation]
+
+    update_on_conflict =
+      if(invisible_conversation, do: [], else: [read: read])
+      |> Keyword.put(:updated_at, NaiveDateTime.utc_now())
 
     %__MODULE__{}
-    |> creation_cng(%{user_id: user.id, conversation_id: conversation.id, read: read})
+    |> creation_cng(%{
+      user_id: user.id,
+      conversation_id: conversation.id,
+      read: invisible_conversation || read
+    })
     |> Repo.insert(
-      on_conflict: [set: [read: read, updated_at: NaiveDateTime.utc_now()]],
+      on_conflict: [set: update_on_conflict],
       returning: true,
       conflict_target: [:user_id, :conversation_id]
     )
@@ -48,10 +57,57 @@ defmodule Pleroma.Conversation.Participation do
     |> validate_required([:read])
   end
 
+  def mark_as_read(%User{} = user, %Conversation{} = conversation) do
+    with %__MODULE__{} = participation <- for_user_and_conversation(user, conversation) do
+      mark_as_read(participation)
+    end
+  end
+
   def mark_as_read(participation) do
     participation
     |> read_cng(%{read: true})
     |> Repo.update()
+    |> case do
+      {:ok, participation} ->
+        participation = Repo.preload(participation, :user)
+        User.set_unread_conversation_count(participation.user)
+        {:ok, participation}
+
+      error ->
+        error
+    end
+  end
+
+  def mark_all_as_read(%User{local: true} = user, %User{} = target_user) do
+    target_conversation_ids =
+      __MODULE__
+      |> where([p], p.user_id == ^target_user.id)
+      |> select([p], p.conversation_id)
+      |> Repo.all()
+
+    __MODULE__
+    |> where([p], p.user_id == ^user.id)
+    |> where([p], p.conversation_id in ^target_conversation_ids)
+    |> update([p], set: [read: true])
+    |> Repo.update_all([])
+
+    {:ok, user} = User.set_unread_conversation_count(user)
+    {:ok, user, []}
+  end
+
+  def mark_all_as_read(%User{} = user, %User{}), do: {:ok, user, []}
+
+  def mark_all_as_read(%User{} = user) do
+    {_, participations} =
+      __MODULE__
+      |> where([p], p.user_id == ^user.id)
+      |> where([p], not p.read)
+      |> update([p], set: [read: true])
+      |> select([p], p)
+      |> Repo.update_all([])
+
+    {:ok, user} = User.set_unread_conversation_count(user)
+    {:ok, user, participations}
   end
 
   def mark_as_unread(participation) do
@@ -135,4 +191,12 @@ defmodule Pleroma.Conversation.Participation do
 
     {:ok, Repo.preload(participation, :recipients, force: true)}
   end
+
+  def unread_conversation_count_for_user(user) do
+    from(p in __MODULE__,
+      where: p.user_id == ^user.id,
+      where: not p.read,
+      select: %{count: count(p.id)}
+    )
+  end
 end

lib/pleroma/conversation/participation_recipient_ship.ex

@@ -12,7 +12,7 @@ defmodule Pleroma.Conversation.Participation.RecipientShip do
   import Ecto.Changeset
 
   schema "conversation_participation_recipient_ships" do
-    belongs_to(:user, User, type: Pleroma.FlakeId)
+    belongs_to(:user, User, type: FlakeId.Ecto.CompatType)
     belongs_to(:participation, Participation)
   end
 

lib/pleroma/daemons/activity_expiration_daemon.ex

@@ -0,0 +1,66 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Daemons.ActivityExpirationDaemon do
+  alias Pleroma.Activity
+  alias Pleroma.ActivityExpiration
+  alias Pleroma.Config
+  alias Pleroma.Repo
+  alias Pleroma.User
+  alias Pleroma.Web.CommonAPI
+
+  require Logger
+  use GenServer
+  import Ecto.Query
+
+  @schedule_interval :timer.minutes(1)
+
+  def start_link(_) do
+    GenServer.start_link(__MODULE__, nil)
+  end
+
+  @impl true
+  def init(_) do
+    if Config.get([ActivityExpiration, :enabled]) do
+      schedule_next()
+      {:ok, nil}
+    else
+      :ignore
+    end
+  end
+
+  def perform(:execute, expiration_id) do
+    try do
+      expiration =
+        ActivityExpiration
+        |> where([e], e.id == ^expiration_id)
+        |> Repo.one!()
+
+      activity = Activity.get_by_id_with_object(expiration.activity_id)
+      user = User.get_by_ap_id(activity.object.data["actor"])
+      CommonAPI.delete(activity.id, user)
+    rescue
+      error ->
+        Logger.error("#{__MODULE__} Couldn't delete expired activity: #{inspect(error)}")
+    end
+  end
+
+  @impl true
+  def handle_info(:perform, state) do
+    ActivityExpiration.due_expirations(@schedule_interval)
+    |> Enum.each(fn expiration ->
+      Pleroma.Workers.ActivityExpirationWorker.enqueue(
+        "activity_expiration",
+        %{"activity_expiration_id" => expiration.id}
+      )
+    end)
+
+    schedule_next()
+    {:noreply, state}
+  end
+
+  defp schedule_next do
+    Process.send_after(self(), :perform, @schedule_interval)
+  end
+end

lib/pleroma/daemons/digest_email_daemon.ex

@@ -1,7 +1,12 @@
-defmodule Pleroma.DigestEmailWorker do
+# Pleroma: A lightweight social networking server
-  import Ecto.Query
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
 
-  @queue_name :digest_emails
+defmodule Pleroma.Daemons.DigestEmailDaemon do
+  alias Pleroma.Repo
+  alias Pleroma.Workers.DigestEmailsWorker
+
+  import Ecto.Query
 
   def perform do
     config = Pleroma.Config.get([:email_notifications, :digest])
@@ -12,12 +17,14 @@ defmodule Pleroma.DigestEmailWorker do
     now = NaiveDateTime.truncate(NaiveDateTime.utc_now(), :second)
 
     from(u in inactive_users_query,
-      where: fragment(~s(? #> '{"email_notifications","digest"}' @> 'true'), u.info),
+      where: fragment(~s(? ->'digest' @> 'true'), u.email_notifications),
       where: u.last_digest_emailed_at < datetime_add(^now, ^negative_interval, "day"),
       select: u
     )
-    |> Pleroma.Repo.all()
+    |> Repo.all()
-    |> Enum.each(&PleromaJobQueue.enqueue(@queue_name, __MODULE__, [&1]))
+    |> Enum.each(fn user ->
+      DigestEmailsWorker.enqueue("digest_email", %{"user_id" => user.id})
+    end)
   end
 
   @doc """

lib/pleroma/daemons/scheduled_activity_daemon.ex

@@ -2,7 +2,7 @@
 # Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
-defmodule Pleroma.ScheduledActivityWorker do
+defmodule Pleroma.Daemons.ScheduledActivityDaemon do
   @moduledoc """
   Sends scheduled activities to the job queue.
   """
@@ -11,6 +11,7 @@ defmodule Pleroma.ScheduledActivityWorker do
   alias Pleroma.ScheduledActivity
   alias Pleroma.User
   alias Pleroma.Web.CommonAPI
+
   use GenServer
   require Logger
 
@@ -45,7 +46,10 @@ defmodule Pleroma.ScheduledActivityWorker do
   def handle_info(:perform, state) do
     ScheduledActivity.due_activities(@schedule_interval)
     |> Enum.each(fn scheduled_activity ->
-      PleromaJobQueue.enqueue(:scheduled_activities, __MODULE__, [:execute, scheduled_activity.id])
+      Pleroma.Workers.ScheduledActivityWorker.enqueue(
+        "execute",
+        %{"activity_id" => scheduled_activity.id}
+      )
     end)
 
     schedule_next()

lib/pleroma/delivery.ex

@@ -0,0 +1,50 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Delivery do
+  use Ecto.Schema
+
+  alias Pleroma.Delivery
+  alias Pleroma.Object
+  alias Pleroma.Repo
+  alias Pleroma.User
+  alias Pleroma.User
+
+  import Ecto.Changeset
+  import Ecto.Query
+
+  schema "deliveries" do
+    belongs_to(:user, User, type: FlakeId.Ecto.CompatType)
+    belongs_to(:object, Object)
+  end
+
+  def changeset(delivery, params \\ %{}) do
+    delivery
+    |> cast(params, [:user_id, :object_id])
+    |> validate_required([:user_id, :object_id])
+    |> foreign_key_constraint(:object_id)
+    |> foreign_key_constraint(:user_id)
+    |> unique_constraint(:user_id, name: :deliveries_user_id_object_id_index)
+  end
+
+  def create(object_id, user_id) do
+    %Delivery{}
+    |> changeset(%{user_id: user_id, object_id: object_id})
+    |> Repo.insert(on_conflict: :nothing)
+  end
+
+  def get(object_id, user_id) do
+    from(d in Delivery, where: d.user_id == ^user_id and d.object_id == ^object_id)
+    |> Repo.one()
+  end
+
+  # A hack because user delete activities have a fake id for whatever reason
+  # TODO: Get rid of this
+  def delete_all_by_object_id("pleroma:fake_object_id"), do: {0, []}
+
+  def delete_all_by_object_id(object_id) do
+    from(d in Delivery, where: d.object_id == ^object_id)
+    |> Repo.delete_all()
+  end
+end

lib/pleroma/docs/generator.ex

@@ -0,0 +1,73 @@
+defmodule Pleroma.Docs.Generator do
+  @callback process(keyword()) :: {:ok, String.t()}
+
+  @spec process(module(), keyword()) :: {:ok, String.t()}
+  def process(implementation, descriptions) do
+    implementation.process(descriptions)
+  end
+
+  @spec uploaders_list() :: [module()]
+  def uploaders_list do
+    {:ok, modules} = :application.get_key(:pleroma, :modules)
+
+    Enum.filter(modules, fn module ->
+      name_as_list = Module.split(module)
+
+      List.starts_with?(name_as_list, ["Pleroma", "Uploaders"]) and
+        List.last(name_as_list) != "Uploader"
+    end)
+  end
+
+  @spec filters_list() :: [module()]
+  def filters_list do
+    {:ok, modules} = :application.get_key(:pleroma, :modules)
+
+    Enum.filter(modules, fn module ->
+      name_as_list = Module.split(module)
+
+      List.starts_with?(name_as_list, ["Pleroma", "Upload", "Filter"])
+    end)
+  end
+
+  @spec mrf_list() :: [module()]
+  def mrf_list do
+    {:ok, modules} = :application.get_key(:pleroma, :modules)
+
+    Enum.filter(modules, fn module ->
+      name_as_list = Module.split(module)
+
+      List.starts_with?(name_as_list, ["Pleroma", "Web", "ActivityPub", "MRF"]) and
+        length(name_as_list) > 4
+    end)
+  end
+
+  @spec richmedia_parsers() :: [module()]
+  def richmedia_parsers do
+    {:ok, modules} = :application.get_key(:pleroma, :modules)
+
+    Enum.filter(modules, fn module ->
+      name_as_list = Module.split(module)
+
+      List.starts_with?(name_as_list, ["Pleroma", "Web", "RichMedia", "Parsers"]) and
+        length(name_as_list) == 5
+    end)
+  end
+end
+
+defimpl Jason.Encoder, for: Tuple do
+  def encode(tuple, opts) do
+    Jason.Encode.list(Tuple.to_list(tuple), opts)
+  end
+end
+
+defimpl Jason.Encoder, for: [Regex, Function] do
+  def encode(term, opts) do
+    Jason.Encode.string(inspect(term), opts)
+  end
+end
+
+defimpl String.Chars, for: Regex do
+  def to_string(term) do
+    inspect(term)
+  end
+end

lib/pleroma/docs/json.ex

@@ -0,0 +1,20 @@
+defmodule Pleroma.Docs.JSON do
+  @behaviour Pleroma.Docs.Generator
+
+  @spec process(keyword()) :: {:ok, String.t()}
+  def process(descriptions) do
+    config_path = "docs/generate_config.json"
+
+    with {:ok, file} <- File.open(config_path, [:write]),
+         json <- generate_json(descriptions),
+         :ok <- IO.write(file, json),
+         :ok <- File.close(file) do
+      {:ok, config_path}
+    end
+  end
+
+  @spec generate_json([keyword()]) :: String.t()
+  def generate_json(descriptions) do
+    Jason.encode!(descriptions)
+  end
+end

lib/pleroma/docs/markdown.ex

@@ -0,0 +1,88 @@
+defmodule Pleroma.Docs.Markdown do
+  @behaviour Pleroma.Docs.Generator
+
+  @spec process(keyword()) :: {:ok, String.t()}
+  def process(descriptions) do
+    config_path = "docs/generated_config.md"
+    {:ok, file} = File.open(config_path, [:utf8, :write])
+    IO.write(file, "# Generated configuration\n")
+    IO.write(file, "Date of generation: #{Date.utc_today()}\n\n")
+
+    IO.write(
+      file,
+      "This file describe the configuration, it is recommended to edit the relevant `*.secret.exs` file instead of the others founds in the ``config`` directory.\n\n" <>
+        "If you run Pleroma with ``MIX_ENV=prod`` the file is ``prod.secret.exs``, otherwise it is ``dev.secret.exs``.\n\n"
+    )
+
+    for group <- descriptions do
+      if is_nil(group[:key]) do
+        IO.write(file, "## #{inspect(group[:group])}\n")
+      else
+        IO.write(file, "## #{inspect(group[:key])}\n")
+      end
+
+      IO.write(file, "#{group[:description]}\n")
+
+      for child <- group[:children] || [] do
+        print_child_header(file, child)
+
+        print_suggestions(file, child[:suggestions])
+
+        if child[:children] do
+          for subchild <- child[:children] do
+            print_child_header(file, subchild)
+
+            print_suggestions(file, subchild[:suggestions])
+          end
+        end
+      end
+
+      IO.write(file, "\n")
+    end
+
+    :ok = File.close(file)
+    {:ok, config_path}
+  end
+
+  defp print_child_header(file, %{key: key, type: type, description: description} = _child) do
+    IO.write(
+      file,
+      "- `#{inspect(key)}` (`#{inspect(type)}`): #{description}  \n"
+    )
+  end
+
+  defp print_child_header(file, %{key: key, type: type} = _child) do
+    IO.write(file, "- `#{inspect(key)}` (`#{inspect(type)}`)  \n")
+  end
+
+  defp print_suggestion(file, suggestion) when is_list(suggestion) do
+    IO.write(file, "  `#{inspect(suggestion)}`\n")
+  end
+
+  defp print_suggestion(file, suggestion) when is_function(suggestion) do
+    IO.write(file, "  `#{inspect(suggestion.())}`\n")
+  end
+
+  defp print_suggestion(file, suggestion, as_list \\ false) do
+    list_mark = if as_list, do: "- ", else: ""
+    IO.write(file, "  #{list_mark}`#{inspect(suggestion)}`\n")
+  end
+
+  defp print_suggestions(_file, nil), do: nil
+
+  defp print_suggestions(_file, ""), do: nil
+
+  defp print_suggestions(file, suggestions) do
+    if length(suggestions) > 1 do
+      IO.write(file, "Suggestions:\n")
+
+      for suggestion <- suggestions do
+        print_suggestion(file, suggestion, true)
+      end
+    else
+      IO.write(file, "  Suggestion: ")
+
+      print_suggestion(file, List.first(suggestions))
+    end
+  end
+end

lib/pleroma/emails/admin_email.ex

@@ -17,7 +17,7 @@ defmodule Pleroma.Emails.AdminEmail do
   end
 
   defp user_url(user) do
-    Helpers.o_status_url(Pleroma.Web.Endpoint, :feed_redirect, user.nickname)
+    Helpers.feed_url(Pleroma.Web.Endpoint, :feed_redirect, user.id)
   end
 
   def report(to, reporter, account, statuses, comment) do

lib/pleroma/emails/mailer.ex

@@ -9,6 +9,7 @@ defmodule Pleroma.Emails.Mailer do
   The module contains functions to delivery email using Swoosh.Mailer.
   """
 
+  alias Pleroma.Workers.MailerWorker
   alias Swoosh.DeliveryError
 
   @otp_app :pleroma
@@ -19,7 +20,12 @@ defmodule Pleroma.Emails.Mailer do
 
   @doc "add email to queue"
   def deliver_async(email, config \\ []) do
-    PleromaJobQueue.enqueue(:mailer, __MODULE__, [:deliver_async, email, config])
+    encoded_email =
+      email
+      |> :erlang.term_to_binary()
+      |> Base.encode64()
+
+    MailerWorker.enqueue("email", %{"encoded_email" => encoded_email, "config" => config})
   end
 
   @doc "callback to perform send email from queue"

lib/pleroma/emails/user_email.ex

@@ -7,21 +7,21 @@ defmodule Pleroma.Emails.UserEmail do
 
   use Phoenix.Swoosh, view: Pleroma.Web.EmailView, layout: {Pleroma.Web.LayoutView, :email}
 
+  alias Pleroma.Config
+  alias Pleroma.User
   alias Pleroma.Web.Endpoint
   alias Pleroma.Web.Router
 
-  defp instance_config, do: Pleroma.Config.get(:instance)
+  defp instance_name, do: Config.get([:instance, :name])
-
-  defp instance_name, do: instance_config()[:name]
 
   defp sender do
-    email = Keyword.get(instance_config(), :notify_email, instance_config()[:email])
+    email = Config.get([:instance, :notify_email]) || Config.get([:instance, :email])
     {instance_name(), email}
   end
 
   defp recipient(email, nil), do: email
   defp recipient(email, name), do: {name, email}
-  defp recipient(%Pleroma.User{} = user), do: recipient(user.email, user.name)
+  defp recipient(%User{} = user), do: recipient(user.email, user.name)
 
   def password_reset_email(user, token) when is_binary(token) do
     password_reset_url = Router.Helpers.reset_password_url(Endpoint, :reset, token)
@@ -72,7 +72,7 @@ defmodule Pleroma.Emails.UserEmail do
         Endpoint,
         :confirm_email,
         user.id,
-        to_string(user.info.confirmation_token)
+        to_string(user.confirmation_token)
       )
 
     html_body = """
@@ -93,67 +93,86 @@ defmodule Pleroma.Emails.UserEmail do
   Includes Mentions and New Followers data
   If there are no mentions (even when new followers exist), the function will return nil
   """
-  @spec digest_email(Pleroma.User.t()) :: Swoosh.Email.t() | nil
+  @spec digest_email(User.t()) :: Swoosh.Email.t() | nil
   def digest_email(user) do
-    new_notifications =
+    notifications = Pleroma.Notification.for_user_since(user, user.last_digest_emailed_at)
-      Pleroma.Notification.for_user_since(user, user.last_digest_emailed_at)
+
-      |> Enum.reduce(%{followers: [], mentions: []}, fn
+    mentions =
-        %{activity: %{data: %{"type" => "Create"}, actor: actor} = activity} = notification,
+      notifications
-        acc ->
+      |> Enum.filter(&(&1.activity.data["type"] == "Create"))
-          new_mention = %{
+      |> Enum.map(fn notification ->
+        object = Pleroma.Object.normalize(notification.activity)
+        object = update_in(object.data["content"], &format_links/1)
+
+        %{
           data: notification,
-            object: Pleroma.Object.normalize(activity),
+          object: object,
-            from: Pleroma.User.get_by_ap_id(actor)
+          from: User.get_by_ap_id(notification.activity.actor)
         }
-
-          %{acc | mentions: [new_mention | acc.mentions]}
-
-        %{activity: %{data: %{"type" => "Follow"}, actor: actor} = activity} = notification,
-        acc ->
-          new_follower = %{
-            data: notification,
-            object: Pleroma.Object.normalize(activity),
-            from: Pleroma.User.get_by_ap_id(actor)
-          }
-
-          %{acc | followers: [new_follower | acc.followers]}
-
-        _, acc ->
-          acc
       end)
 
-    with [_ | _] = mentions <- new_notifications.mentions do
+    followers =
+      notifications
+      |> Enum.filter(&(&1.activity.data["type"] == "Follow"))
+      |> Enum.map(fn notification ->
+        %{
+          data: notification,
+          object: Pleroma.Object.normalize(notification.activity),
+          from: User.get_by_ap_id(notification.activity.actor)
+        }
+      end)
+
+    unless Enum.empty?(mentions) do
+      styling = Config.get([__MODULE__, :styling])
+      logo = Config.get([__MODULE__, :logo])
+
       html_data = %{
         instance: instance_name(),
         user: user,
         mentions: mentions,
-        followers: new_notifications.followers,
+        followers: followers,
-        unsubscribe_link: unsubscribe_url(user, "digest")
+        unsubscribe_link: unsubscribe_url(user, "digest"),
+        styling: styling
       }
 
+      logo_path =
+        if is_nil(logo) do
+          Path.join(:code.priv_dir(:pleroma), "static/static/logo.png")
+        else
+          Path.join(Config.get([:instance, :static_dir]), logo)
+        end
+
       new()
       |> to(recipient(user))
       |> from(sender())
       |> subject("Your digest from #{instance_name()}")
+      |> put_layout(false)
       |> render_body("digest.html", html_data)
-    else
+      |> attachment(Swoosh.Attachment.new(logo_path, filename: "logo.png", type: :inline))
-      _ ->
-        nil
     end
   end
 
+  defp format_links(str) do
+    re = ~r/<a.+href=['"].*>/iU
+    %{link_color: color} = Config.get([__MODULE__, :styling])
+
+    Regex.replace(re, str, fn link ->
+      String.replace(link, "<a", "<a style=\"color: #{color};text-decoration: none;\"")
+    end)
+  end
+
   @doc """
   Generate unsubscribe link for given user and notifications type.
   The link contains JWT token with the data, and subscription can be modified without
   authorization.
   """
-  @spec unsubscribe_url(Pleroma.User.t(), String.t()) :: String.t()
+  @spec unsubscribe_url(User.t(), String.t()) :: String.t()
   def unsubscribe_url(user, notifications_type) do
     token =
       %{"sub" => user.id, "act" => %{"unsubscribe" => notifications_type}, "exp" => false}
       |> Pleroma.JWT.generate_and_sign!()
       |> Base.encode64()
 
-    Router.Helpers.subscription_url(Pleroma.Web.Endpoint, :unsubscribe, token)
+    Router.Helpers.subscription_url(Endpoint, :unsubscribe, token)
   end
 end

lib/pleroma/emoji.ex

@@ -4,24 +4,37 @@
 
 defmodule Pleroma.Emoji do
   @moduledoc """
-  The emojis are loaded from:
+  This GenServer stores in an ETS table the list of the loaded emojis,
-
+  and also allows to reload the list at runtime.
-    * emoji packs in INSTANCE-DIR/emoji
-    * the files: `config/emoji.txt` and `config/custom_emoji.txt`
-    * glob paths, nested folder is used as tag name for grouping e.g. priv/static/emoji/custom/nested_folder
-
-  This GenServer stores in an ETS table the list of the loaded emojis, and also allows to reload the list at runtime.
   """
   use GenServer
 
+  alias Pleroma.Emoji.Loader
+
   require Logger
 
-  @type pattern :: Regex.t() | module() | String.t()
-  @type patterns :: pattern() | [pattern()]
-  @type group_patterns :: keyword(patterns())
-
   @ets __MODULE__.Ets
-  @ets_options [:ordered_set, :protected, :named_table, {:read_concurrency, true}]
+  @ets_options [
+    :ordered_set,
+    :protected,
+    :named_table,
+    {:read_concurrency, true}
+  ]
+
+  defstruct [:code, :file, :tags, :safe_code, :safe_file]
+
+  @doc "Build emoji struct"
+  def build({code, file, tags}) do
+    %__MODULE__{
+      code: code,
+      file: file,
+      tags: tags,
+      safe_code: Pleroma.HTML.strip_tags(code),
+      safe_file: Pleroma.HTML.strip_tags(file)
+    }
+  end
+
+  def build({code, file}), do: build({code, file, []})
 
   @doc false
   def start_link(_) do
@@ -44,11 +57,14 @@ defmodule Pleroma.Emoji do
   end
 
   @doc "Returns all the emojos!!"
-  @spec get_all() :: [{String.t(), String.t()}, ...]
+  @spec get_all() :: list({String.t(), String.t(), String.t()})
   def get_all do
     :ets.tab2list(@ets)
   end
 
+  @doc "Clear out old emojis"
+  def clear_all, do: :ets.delete_all_objects(@ets)
+
   @doc false
   def init(_) do
     @ets = :ets.new(@ets, @ets_options)
@@ -58,13 +74,13 @@ defmodule Pleroma.Emoji do
 
   @doc false
   def handle_cast(:reload, state) do
-    load()
+    update_emojis(Loader.load())
     {:noreply, state}
   end
 
   @doc false
   def handle_call(:reload, _from, state) do
-    load()
+    update_emojis(Loader.load())
     {:reply, :ok, state}
   end
 
@@ -75,189 +91,11 @@ defmodule Pleroma.Emoji do
 
   @doc false
   def code_change(_old_vsn, state, _extra) do
-    load()
+    update_emojis(Loader.load())
     {:ok, state}
   end
 
-  defp load do
+  defp update_emojis(emojis) do
-    emoji_dir_path =
+    :ets.insert(@ets, emojis)
-      Path.join(
-        Pleroma.Config.get!([:instance, :static_dir]),
-        "emoji"
-      )
-
-    emoji_groups = Pleroma.Config.get([:emoji, :groups])
-
-    case File.ls(emoji_dir_path) do
-      {:error, :enoent} ->
-        # The custom emoji directory doesn't exist,
-        # don't do anything
-        nil
-
-      {:error, e} ->
-        # There was some other error
-        Logger.error("Could not access the custom emoji directory #{emoji_dir_path}: #{e}")
-
-      {:ok, results} ->
-        grouped =
-          Enum.group_by(results, fn file -> File.dir?(Path.join(emoji_dir_path, file)) end)
-
-        packs = grouped[true] || []
-        files = grouped[false] || []
-
-        # Print the packs we've found
-        Logger.info("Found emoji packs: #{Enum.join(packs, ", ")}")
-
-        if not Enum.empty?(files) do
-          Logger.warn(
-            "Found files in the emoji folder. These will be ignored, please move them to a subdirectory\nFound files: #{
-              Enum.join(files, ", ")
-            }"
-          )
-        end
-
-        emojis =
-          Enum.flat_map(
-            packs,
-            fn pack -> load_pack(Path.join(emoji_dir_path, pack), emoji_groups) end
-          )
-
-        true = :ets.insert(@ets, emojis)
-    end
-
-    # Compat thing for old custom emoji handling & default emoji,
-    # it should run even if there are no emoji packs
-    shortcode_globs = Pleroma.Config.get([:emoji, :shortcode_globs], [])
-
-    emojis =
-      (load_from_file("config/emoji.txt", emoji_groups) ++
-         load_from_file("config/custom_emoji.txt", emoji_groups) ++
-         load_from_globs(shortcode_globs, emoji_groups))
-      |> Enum.reject(fn value -> value == nil end)
-
-    true = :ets.insert(@ets, emojis)
-
-    :ok
-  end
-
-  defp load_pack(pack_dir, emoji_groups) do
-    pack_name = Path.basename(pack_dir)
-
-    emoji_txt = Path.join(pack_dir, "emoji.txt")
-
-    if File.exists?(emoji_txt) do
-      load_from_file(emoji_txt, emoji_groups)
-    else
-      extensions = Pleroma.Config.get([:emoji, :pack_extensions])
-
-      Logger.info(
-        "No emoji.txt found for pack \"#{pack_name}\", assuming all #{Enum.join(extensions, ", ")} files are emoji"
-      )
-
-      make_shortcode_to_file_map(pack_dir, extensions)
-      |> Enum.map(fn {shortcode, rel_file} ->
-        filename = Path.join("/emoji/#{pack_name}", rel_file)
-
-        {shortcode, filename, [to_string(match_extra(emoji_groups, filename))]}
-      end)
-    end
-  end
-
-  def make_shortcode_to_file_map(pack_dir, exts) do
-    find_all_emoji(pack_dir, exts)
-    |> Enum.map(&Path.relative_to(&1, pack_dir))
-    |> Enum.map(fn f -> {f |> Path.basename() |> Path.rootname(), f} end)
-    |> Enum.into(%{})
-  end
-
-  def find_all_emoji(dir, exts) do
-    Enum.reduce(
-      File.ls!(dir),
-      [],
-      fn f, acc ->
-        filepath = Path.join(dir, f)
-
-        if File.dir?(filepath) do
-          acc ++ find_all_emoji(filepath, exts)
-        else
-          acc ++ [filepath]
-        end
-      end
-    )
-    |> Enum.filter(fn f -> Path.extname(f) in exts end)
-  end
-
-  defp load_from_file(file, emoji_groups) do
-    if File.exists?(file) do
-      load_from_file_stream(File.stream!(file), emoji_groups)
-    else
-      []
-    end
-  end
-
-  defp load_from_file_stream(stream, emoji_groups) do
-    stream
-    |> Stream.map(&String.trim/1)
-    |> Stream.map(fn line ->
-      case String.split(line, ~r/,\s*/) do
-        [name, file] ->
-          {name, file, [to_string(match_extra(emoji_groups, file))]}
-
-        [name, file | tags] ->
-          {name, file, tags}
-
-        _ ->
-          nil
-      end
-    end)
-    |> Enum.to_list()
-  end
-
-  defp load_from_globs(globs, emoji_groups) do
-    static_path = Path.join(:code.priv_dir(:pleroma), "static")
-
-    paths =
-      Enum.map(globs, fn glob ->
-        Path.join(static_path, glob)
-        |> Path.wildcard()
-      end)
-      |> Enum.concat()
-
-    Enum.map(paths, fn path ->
-      tag = match_extra(emoji_groups, Path.join("/", Path.relative_to(path, static_path)))
-      shortcode = Path.basename(path, Path.extname(path))
-      external_path = Path.join("/", Path.relative_to(path, static_path))
-      {shortcode, external_path, [to_string(tag)]}
-    end)
-  end
-
-  @doc """
-  Finds a matching group for the given emoji filename
-  """
-  @spec match_extra(group_patterns(), String.t()) :: atom() | nil
-  def match_extra(group_patterns, filename) do
-    match_group_patterns(group_patterns, fn pattern ->
-      case pattern do
-        %Regex{} = regex -> Regex.match?(regex, filename)
-        string when is_binary(string) -> filename == string
-      end
-    end)
-  end
-
-  defp match_group_patterns(group_patterns, matcher) do
-    Enum.find_value(group_patterns, fn {group, patterns} ->
-      patterns =
-        patterns
-        |> List.wrap()
-        |> Enum.map(fn pattern ->
-          if String.contains?(pattern, "*") do
-            ~r(#{String.replace(pattern, "*", ".*")})
-          else
-            pattern
-          end
-        end)
-
-      Enum.any?(patterns, matcher) && group
-    end)
   end
 end

lib/pleroma/emoji/formatter.ex

@@ -0,0 +1,59 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Emoji.Formatter do
+  alias Pleroma.Emoji
+  alias Pleroma.HTML
+  alias Pleroma.Web.MediaProxy
+
+  def emojify(text) do
+    emojify(text, Emoji.get_all())
+  end
+
+  def emojify(text, nil), do: text
+
+  def emojify(text, emoji, strip \\ false) do
+    Enum.reduce(emoji, text, fn
+      {_, %Emoji{safe_code: emoji, safe_file: file}}, text ->
+        String.replace(text, ":#{emoji}:", prepare_emoji_html(emoji, file, strip))
+
+      {unsafe_emoji, unsafe_file}, text ->
+        emoji = HTML.strip_tags(unsafe_emoji)
+        file = HTML.strip_tags(unsafe_file)
+        String.replace(text, ":#{emoji}:", prepare_emoji_html(emoji, file, strip))
+    end)
+    |> HTML.filter_tags()
+  end
+
+  defp prepare_emoji_html(_emoji, _file, true), do: ""
+
+  defp prepare_emoji_html(emoji, file, _strip) do
+    "<img class='emoji' alt='#{emoji}' title='#{emoji}' src='#{MediaProxy.url(file)}' />"
+  end
+
+  def demojify(text) do
+    emojify(text, Emoji.get_all(), true)
+  end
+
+  def demojify(text, nil), do: text
+
+  @doc "Outputs a list of the emoji-shortcodes in a text"
+  def get_emoji(text) when is_binary(text) do
+    Enum.filter(Emoji.get_all(), fn {emoji, %Emoji{}} ->
+      String.contains?(text, ":#{emoji}:")
+    end)
+  end
+
+  def get_emoji(_), do: []
+
+  @doc "Outputs a list of the emoji-Maps in a text"
+  def get_emoji_map(text) when is_binary(text) do
+    get_emoji(text)
+    |> Enum.reduce(%{}, fn {name, %Emoji{file: file}}, acc ->
+      Map.put(acc, name, "#{Pleroma.Web.Endpoint.static_url()}#{file}")
+    end)
+  end
+
+  def get_emoji_map(_), do: []
+end

lib/pleroma/emoji/loader.ex

@@ -0,0 +1,224 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Emoji.Loader do
+  @moduledoc """
+  The Loader emoji from:
+
+    * emoji packs in INSTANCE-DIR/emoji
+    * the files: `config/emoji.txt` and `config/custom_emoji.txt`
+    * glob paths, nested folder is used as tag name for grouping e.g. priv/static/emoji/custom/nested_folder
+  """
+  alias Pleroma.Config
+  alias Pleroma.Emoji
+
+  require Logger
+
+  @type pattern :: Regex.t() | module() | String.t()
+  @type patterns :: pattern() | [pattern()]
+  @type group_patterns :: keyword(patterns())
+  @type emoji :: {String.t(), Emoji.t()}
+
+  @doc """
+  Loads emojis from files/packs.
+
+  returns list emojis in format:
+  `{"000", "/emoji/freespeechextremist.com/000.png", ["Custom"]}`
+  """
+  @spec load() :: list(emoji)
+  def load do
+    emoji_dir_path = Path.join(Config.get!([:instance, :static_dir]), "emoji")
+
+    emoji_groups = Config.get([:emoji, :groups])
+
+    emojis =
+      case File.ls(emoji_dir_path) do
+        {:error, :enoent} ->
+          # The custom emoji directory doesn't exist,
+          # don't do anything
+          []
+
+        {:error, e} ->
+          # There was some other error
+          Logger.error("Could not access the custom emoji directory #{emoji_dir_path}: #{e}")
+          []
+
+        {:ok, results} ->
+          grouped =
+            Enum.group_by(results, fn file ->
+              File.dir?(Path.join(emoji_dir_path, file))
+            end)
+
+          packs = grouped[true] || []
+          files = grouped[false] || []
+
+          # Print the packs we've found
+          Logger.info("Found emoji packs: #{Enum.join(packs, ", ")}")
+
+          if not Enum.empty?(files) do
+            Logger.warn(
+              "Found files in the emoji folder. These will be ignored, please move them to a subdirectory\nFound files: #{
+                Enum.join(files, ", ")
+              }"
+            )
+          end
+
+          emojis =
+            Enum.flat_map(packs, fn pack ->
+              load_pack(Path.join(emoji_dir_path, pack), emoji_groups)
+            end)
+
+          Emoji.clear_all()
+          emojis
+      end
+
+    # Compat thing for old custom emoji handling & default emoji,
+    # it should run even if there are no emoji packs
+    shortcode_globs = Config.get([:emoji, :shortcode_globs], [])
+
+    emojis_txt =
+      (load_from_file("config/emoji.txt", emoji_groups) ++
+         load_from_file("config/custom_emoji.txt", emoji_groups) ++
+         load_from_globs(shortcode_globs, emoji_groups))
+      |> Enum.reject(fn value -> value == nil end)
+
+    Enum.map(emojis ++ emojis_txt, &prepare_emoji/1)
+  end
+
+  defp prepare_emoji({code, _, _} = emoji), do: {code, Emoji.build(emoji)}
+
+  defp load_pack(pack_dir, emoji_groups) do
+    pack_name = Path.basename(pack_dir)
+
+    pack_file = Path.join(pack_dir, "pack.json")
+
+    if File.exists?(pack_file) do
+      contents = Jason.decode!(File.read!(pack_file))
+
+      contents["files"]
+      |> Enum.map(fn {name, rel_file} ->
+        filename = Path.join("/emoji/#{pack_name}", rel_file)
+        {name, filename, ["pack:#{pack_name}"]}
+      end)
+    else
+      # Load from emoji.txt / all files
+      emoji_txt = Path.join(pack_dir, "emoji.txt")
+
+      if File.exists?(emoji_txt) do
+        load_from_file(emoji_txt, emoji_groups)
+      else
+        extensions = Pleroma.Config.get([:emoji, :pack_extensions])
+
+        Logger.info(
+          "No emoji.txt found for pack \"#{pack_name}\", assuming all #{
+            Enum.join(extensions, ", ")
+          } files are emoji"
+        )
+
+        make_shortcode_to_file_map(pack_dir, extensions)
+        |> Enum.map(fn {shortcode, rel_file} ->
+          filename = Path.join("/emoji/#{pack_name}", rel_file)
+
+          {shortcode, filename, [to_string(match_extra(emoji_groups, filename))]}
+        end)
+      end
+    end
+  end
+
+  def make_shortcode_to_file_map(pack_dir, exts) do
+    find_all_emoji(pack_dir, exts)
+    |> Enum.map(&Path.relative_to(&1, pack_dir))
+    |> Enum.map(fn f -> {f |> Path.basename() |> Path.rootname(), f} end)
+    |> Enum.into(%{})
+  end
+
+  def find_all_emoji(dir, exts) do
+    dir
+    |> File.ls!()
+    |> Enum.flat_map(fn f ->
+      filepath = Path.join(dir, f)
+
+      if File.dir?(filepath) do
+        find_all_emoji(filepath, exts)
+      else
+        [filepath]
+      end
+    end)
+    |> Enum.filter(fn f -> Path.extname(f) in exts end)
+  end
+
+  defp load_from_file(file, emoji_groups) do
+    if File.exists?(file) do
+      load_from_file_stream(File.stream!(file), emoji_groups)
+    else
+      []
+    end
+  end
+
+  defp load_from_file_stream(stream, emoji_groups) do
+    stream
+    |> Stream.map(&String.trim/1)
+    |> Stream.map(fn line ->
+      case String.split(line, ~r/,\s*/) do
+        [name, file] ->
+          {name, file, [to_string(match_extra(emoji_groups, file))]}
+
+        [name, file | tags] ->
+          {name, file, tags}
+
+        _ ->
+          nil
+      end
+    end)
+    |> Enum.to_list()
+  end
+
+  defp load_from_globs(globs, emoji_groups) do
+    static_path = Path.join(:code.priv_dir(:pleroma), "static")
+
+    paths =
+      Enum.map(globs, fn glob ->
+        Path.join(static_path, glob)
+        |> Path.wildcard()
+      end)
+      |> Enum.concat()
+
+    Enum.map(paths, fn path ->
+      tag = match_extra(emoji_groups, Path.join("/", Path.relative_to(path, static_path)))
+      shortcode = Path.basename(path, Path.extname(path))
+      external_path = Path.join("/", Path.relative_to(path, static_path))
+      {shortcode, external_path, [to_string(tag)]}
+    end)
+  end
+
+  @doc """
+  Finds a matching group for the given emoji filename
+  """
+  @spec match_extra(group_patterns(), String.t()) :: atom() | nil
+  def match_extra(group_patterns, filename) do
+    match_group_patterns(group_patterns, fn pattern ->
+      case pattern do
+        %Regex{} = regex -> Regex.match?(regex, filename)
+        string when is_binary(string) -> filename == string
+      end
+    end)
+  end
+
+  defp match_group_patterns(group_patterns, matcher) do
+    Enum.find_value(group_patterns, fn {group, patterns} ->
+      patterns =
+        patterns
+        |> List.wrap()
+        |> Enum.map(fn pattern ->
+          if String.contains?(pattern, "*") do
+            ~r(#{String.replace(pattern, "*", ".*")})
+          else
+            pattern
+          end
+        end)
+
+      Enum.any?(patterns, matcher) && group
+    end)
+  end
+end

lib/pleroma/filter.ex

@@ -12,7 +12,7 @@ defmodule Pleroma.Filter do
   alias Pleroma.User
 
   schema "filters" do
-    belongs_to(:user, User, type: Pleroma.FlakeId)
+    belongs_to(:user, User, type: FlakeId.Ecto.CompatType)
     field(:filter_id, :integer)
     field(:hide, :boolean, default: false)
     field(:whole_word, :boolean, default: true)

lib/pleroma/flake_id.ex

@@ -1,182 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Pleroma.FlakeId do
-  @moduledoc """
-  Flake is a decentralized, k-ordered id generation service.
-
-  Adapted from:
-
-  * [flaky](https://github.com/nirvana/flaky), released under the terms of the Truly Free License,
-  * [Flake](https://github.com/boundary/flake), Copyright 2012, Boundary, Apache License, Version 2.0
-  """
-
-  @type t :: binary
-
-  @behaviour Ecto.Type
-  use GenServer
-  require Logger
-  alias __MODULE__
-  import Kernel, except: [to_string: 1]
-
-  defstruct node: nil, time: 0, sq: 0
-
-  @doc "Converts a binary Flake to a String"
-  def to_string(<<0::integer-size(64), id::integer-size(64)>>) do
-    Kernel.to_string(id)
-  end
-
-  def to_string(<<_::integer-size(64), _::integer-size(48), _::integer-size(16)>> = flake) do
-    encode_base62(flake)
-  end
-
-  def to_string(s), do: s
-
-  def from_string(int) when is_integer(int) do
-    from_string(Kernel.to_string(int))
-  end
-
-  for i <- [-1, 0] do
-    def from_string(unquote(i)), do: <<0::integer-size(128)>>
-    def from_string(unquote(Kernel.to_string(i))), do: <<0::integer-size(128)>>
-  end
-
-  def from_string(<<_::integer-size(128)>> = flake), do: flake
-
-  def from_string(string) when is_binary(string) and byte_size(string) < 18 do
-    case Integer.parse(string) do
-      {id, ""} -> <<0::integer-size(64), id::integer-size(64)>>
-      _ -> nil
-    end
-  end
-
-  def from_string(string) do
-    string |> decode_base62 |> from_integer
-  end
-
-  def to_integer(<<integer::integer-size(128)>>), do: integer
-
-  def from_integer(integer) do
-    <<_time::integer-size(64), _node::integer-size(48), _seq::integer-size(16)>> =
-      <<integer::integer-size(128)>>
-  end
-
-  @doc "Generates a Flake"
-  @spec get :: binary
-  def get, do: to_string(:gen_server.call(:flake, :get))
-
-  # checks that ID is is valid FlakeID
-  #
-  @spec is_flake_id?(String.t()) :: boolean
-  def is_flake_id?(id), do: is_flake_id?(String.to_charlist(id), true)
-  defp is_flake_id?([c | cs], true) when c >= ?0 and c <= ?9, do: is_flake_id?(cs, true)
-  defp is_flake_id?([c | cs], true) when c >= ?A and c <= ?Z, do: is_flake_id?(cs, true)
-  defp is_flake_id?([c | cs], true) when c >= ?a and c <= ?z, do: is_flake_id?(cs, true)
-  defp is_flake_id?([], true), do: true
-  defp is_flake_id?(_, _), do: false
-
-  # -- Ecto.Type API
-  @impl Ecto.Type
-  def type, do: :uuid
-
-  @impl Ecto.Type
-  def cast(value) do
-    {:ok, FlakeId.to_string(value)}
-  end
-
-  @impl Ecto.Type
-  def load(value) do
-    {:ok, FlakeId.to_string(value)}
-  end
-
-  @impl Ecto.Type
-  def dump(value) do
-    {:ok, FlakeId.from_string(value)}
-  end
-
-  def autogenerate, do: get()
-
-  # -- GenServer API
-  def start_link(_) do
-    :gen_server.start_link({:local, :flake}, __MODULE__, [], [])
-  end
-
-  @impl GenServer
-  def init([]) do
-    {:ok, %FlakeId{node: worker_id(), time: time()}}
-  end
-
-  @impl GenServer
-  def handle_call(:get, _from, state) do
-    {flake, new_state} = get(time(), state)
-    {:reply, flake, new_state}
-  end
-
-  # Matches when the calling time is the same as the state time. Incr. sq
-  defp get(time, %FlakeId{time: time, node: node, sq: seq}) do
-    new_state = %FlakeId{time: time, node: node, sq: seq + 1}
-    {gen_flake(new_state), new_state}
-  end
-
-  # Matches when the times are different, reset sq
-  defp get(newtime, %FlakeId{time: time, node: node}) when newtime > time do
-    new_state = %FlakeId{time: newtime, node: node, sq: 0}
-    {gen_flake(new_state), new_state}
-  end
-
-  # Error when clock is running backwards
-  defp get(newtime, %FlakeId{time: time}) when newtime < time do
-    {:error, :clock_running_backwards}
-  end
-
-  defp gen_flake(%FlakeId{time: time, node: node, sq: seq}) do
-    <<time::integer-size(64), node::integer-size(48), seq::integer-size(16)>>
-  end
-
-  defp nthchar_base62(n) when n <= 9, do: ?0 + n
-  defp nthchar_base62(n) when n <= 35, do: ?A + n - 10
-  defp nthchar_base62(n), do: ?a + n - 36
-
-  defp encode_base62(<<integer::integer-size(128)>>) do
-    integer
-    |> encode_base62([])
-    |> List.to_string()
-  end
-
-  defp encode_base62(int, acc) when int < 0, do: encode_base62(-int, acc)
-  defp encode_base62(int, []) when int == 0, do: '0'
-  defp encode_base62(int, acc) when int == 0, do: acc
-
-  defp encode_base62(int, acc) do
-    r = rem(int, 62)
-    id = div(int, 62)
-    acc = [nthchar_base62(r) | acc]
-    encode_base62(id, acc)
-  end
-
-  defp decode_base62(s) do
-    decode_base62(String.to_charlist(s), 0)
-  end
-
-  defp decode_base62([c | cs], acc) when c >= ?0 and c <= ?9,
-    do: decode_base62(cs, 62 * acc + (c - ?0))
-
-  defp decode_base62([c | cs], acc) when c >= ?A and c <= ?Z,
-    do: decode_base62(cs, 62 * acc + (c - ?A + 10))
-
-  defp decode_base62([c | cs], acc) when c >= ?a and c <= ?z,
-    do: decode_base62(cs, 62 * acc + (c - ?a + 36))
-
-  defp decode_base62([], acc), do: acc
-
-  defp time do
-    {mega_seconds, seconds, micro_seconds} = :erlang.timestamp()
-    1_000_000_000 * mega_seconds + seconds * 1000 + :erlang.trunc(micro_seconds / 1000)
-  end
-
-  defp worker_id do
-    <<worker::integer-size(48)>> = :crypto.strong_rand_bytes(6)
-    worker
-  end
-end

lib/pleroma/following_relationship.ex

@@ -0,0 +1,110 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.FollowingRelationship do
+  use Ecto.Schema
+
+  import Ecto.Changeset
+  import Ecto.Query
+
+  alias FlakeId.Ecto.CompatType
+  alias Pleroma.Repo
+  alias Pleroma.User
+
+  schema "following_relationships" do
+    field(:state, :string, default: "accept")
+
+    belongs_to(:follower, User, type: CompatType)
+    belongs_to(:following, User, type: CompatType)
+
+    timestamps()
+  end
+
+  def changeset(%__MODULE__{} = following_relationship, attrs) do
+    following_relationship
+    |> cast(attrs, [:state])
+    |> put_assoc(:follower, attrs.follower)
+    |> put_assoc(:following, attrs.following)
+    |> validate_required([:state, :follower, :following])
+  end
+
+  def get(%User{} = follower, %User{} = following) do
+    __MODULE__
+    |> where(follower_id: ^follower.id, following_id: ^following.id)
+    |> Repo.one()
+  end
+
+  def update(follower, following, "reject"), do: unfollow(follower, following)
+
+  def update(%User{} = follower, %User{} = following, state) do
+    case get(follower, following) do
+      nil ->
+        follow(follower, following, state)
+
+      following_relationship ->
+        following_relationship
+        |> cast(%{state: state}, [:state])
+        |> validate_required([:state])
+        |> Repo.update()
+    end
+  end
+
+  def follow(%User{} = follower, %User{} = following, state \\ "accept") do
+    %__MODULE__{}
+    |> changeset(%{follower: follower, following: following, state: state})
+    |> Repo.insert(on_conflict: :nothing)
+  end
+
+  def unfollow(%User{} = follower, %User{} = following) do
+    case get(follower, following) do
+      nil -> {:ok, nil}
+      %__MODULE__{} = following_relationship -> Repo.delete(following_relationship)
+    end
+  end
+
+  def follower_count(%User{} = user) do
+    %{followers: user, deactivated: false}
+    |> User.Query.build()
+    |> Repo.aggregate(:count, :id)
+  end
+
+  def following_count(%User{id: nil}), do: 0
+
+  def following_count(%User{} = user) do
+    %{friends: user, deactivated: false}
+    |> User.Query.build()
+    |> Repo.aggregate(:count, :id)
+  end
+
+  def get_follow_requests(%User{id: id}) do
+    __MODULE__
+    |> join(:inner, [r], f in assoc(r, :follower))
+    |> where([r], r.state == "pending")
+    |> where([r], r.following_id == ^id)
+    |> select([r, f], f)
+    |> Repo.all()
+  end
+
+  def following?(%User{id: follower_id}, %User{id: followed_id}) do
+    __MODULE__
+    |> where(follower_id: ^follower_id, following_id: ^followed_id, state: "accept")
+    |> Repo.exists?()
+  end
+
+  def following(%User{} = user) do
+    following =
+      __MODULE__
+      |> join(:inner, [r], u in User, on: r.following_id == u.id)
+      |> where([r], r.follower_id == ^user.id)
+      |> where([r], r.state == "accept")
+      |> select([r, u], u.follower_address)
+      |> Repo.all()
+
+    if not user.local or user.nickname in [nil, "internal.fetch"] do
+      following
+    else
+      [user.follower_address | following]
+    end
+  end
+end