Merge remote-tracking branch 'origin/feature/bounded-retention' into shigusegubu-new

This reverts commit 573134ac6c.

Dockerfile

@@ -42,7 +42,7 @@ ARG HOME=/opt/pleroma
 ARG DATA=/var/lib/pleroma
 
 RUN apk update &&\
-	apk add exiftool ffmpeg vips libmagic ncurses postgresql-client &&\
+	apk add ffmpeg vips libmagic ncurses postgresql-client &&\
 	adduser --system --shell /bin/false --home ${HOME} pleroma &&\
 	mkdir -p ${DATA}/uploads &&\
 	mkdir -p ${DATA}/static &&\

Dockerfile-e2e

@@ -41,7 +41,7 @@ ARG HOME=/opt/pleroma
 ARG DATA=/var/lib/pleroma
 
 RUN apk update &&\
-	apk add exiftool ffmpeg vips libmagic ncurses postgresql-client &&\
+	apk add ffmpeg vips libmagic ncurses postgresql-client &&\
 	adduser --system --shell /bin/false --home ${HOME} pleroma &&\
 	mkdir -p ${DATA}/uploads &&\
 	mkdir -p ${DATA}/static &&\

changelog.d/advisory-bumps.security

@@ -0,0 +1 @@
+Update ssl_verify_fun to 1.1.9 (EEF-CVE-2026-117666) and phoenix_html to 3.3.5 (EEF-CVE-2026-117645)

changelog.d/bandit-1.14.1.security

@@ -0,0 +1 @@
+Update Bandit to version 1.14.1, which fixes an HTTP/2 request body memory amplification

changelog.d/c2s-rate-limit.security

@@ -0,0 +1 @@
+Rate limit ActivityPub C2S outbox posts and emoji reactions

changelog.d/card-skip-quote-link.fix

@@ -0,0 +1 @@
+Link preview cards skip the fallback link of quote posts (`RE: <url>`), which pointed at the quoted post and showed it twice.

changelog.d/cast-and-validate-cast-opts.fix

@@ -0,0 +1 @@
+Fix per-status and per-account rate limits when `Pleroma.Web.ApiSpec.CastAndValidate` `strict` is off (the production default)

changelog.d/cast-and-validate-many-params.fix

@@ -0,0 +1 @@
+Drop all unexpected query params, not just the first one, when `Pleroma.Web.ApiSpec.CastAndValidate` `strict` is off

changelog.d/config-db-unknown-module.fix

@@ -0,0 +1 @@
+Do not crash at boot or in the AdminAPI config listing when a ConfigDB setting refers to a module that no longer exists

changelog.d/context-local-only.fix

@@ -0,0 +1 @@
+Show local-only replies in status context to local users who don't follow the replier

changelog.d/current_token.add

@@ -0,0 +1 @@
+OAuth token list API now includes property to signify which token is being used to access the API

changelog.d/emoji-react-single-tag.fix

@@ -0,0 +1 @@
+Accept incoming emoji reactions whose tag is a single object instead of an array

changelog.d/empty-proxy-url.fix

@@ -0,0 +1 @@
+HTTP: treat an empty proxy_url as no proxy; with Hackney 4 it made every outgoing request fail

changelog.d/exif-heic-gps.security

@@ -0,0 +1 @@
+Strip GPS data from WebP and HEIC/AVIF uploads; the Exiftool.StripLocation filter used to skip them; GPS data in XMP (including DJI drone positions) is stripped too

changelog.d/exif-stripper.change

@@ -0,0 +1 @@
+Exiftool upload filters are replaced by the built-in Exif.StripLocation and Exif.ReadDescription filters; exiftool is no longer required, old filter names still work but are deprecated

changelog.d/gin-fuzzy-search-limit.fix

@@ -0,0 +1 @@
+gin_fuzzy_search_limit is now only set for search queries (configurable under Pleroma.Search.DatabaseSearch), so other GIN index scans such as recipient lookups and Oban job queries are exact again

changelog.d/gun-ipv6-only-hosts.fix

@@ -0,0 +1 @@
+Gun adapter: connect to hosts that only have IPv6 (AAAA) addresses

changelog.d/http-backoff.add

@@ -0,0 +1 @@
+Back off from hosts that answer 429 or 503 to object fetches and WebFinger, until Retry-After or X-RateLimit-Reset (default 5 minutes, max 1 hour)

changelog.d/jose-1.11.13.security

@@ -0,0 +1 @@
+Update jose to version 1.11.13, which fixes JWE, key parsing and signature validation vulnerabilities

changelog.d/marker-updated-at-format.fix

@@ -0,0 +1 @@
+Mastodon API: Use the Mastodon date format for marker updated_at

changelog.d/media-proxy-data-urls.fix

@@ -0,0 +1 @@
+MediaProxy: Do not proxy data: URLs

changelog.d/media-proxy-static-redirect.fix

@@ -0,0 +1 @@
+Media proxy: serve a preview for static=true on non-GIF files instead of redirecting to the main endpoint host

changelog.d/move-refetch-target.fix

@@ -0,0 +1 @@
+Accept a Move when the cached remote target does not list the alias yet, by refetching the target once

changelog.d/mrf-domain-regex-escape.security

@@ -0,0 +1 @@
+MRF domain rules now match dots literally, so a rule for `good.example` no longer also matches `goodxexample`, and wildcard rules no longer backtrack on hostile host names

changelog.d/personal-inbox-addressing.fix

@@ -0,0 +1 @@
+Accept activities delivered to a personal inbox that do not address the inbox owner (fixes Bridgy Fed follows)

changelog.d/question-note-fixups.fix

@@ -0,0 +1 @@
+Apply the Note fixups to incoming Questions, so polls with a single non-array tag or attachment are accepted

changelog.d/retention-mode.add

@@ -0,0 +1 @@
+Add a bounded retention mode (`config :pleroma, :retention`): a cron worker walks old activities in id order and evicts remote threads nobody local interacted with, in bounded batches, so the database plateaus instead of growing forever. Adds a `retention_cursors` table. `prune_objects --keep-threads` now also keeps threads that are addressed to a local user, still have a notification for one, contain a post quoted by a local post, or contain a reported post.

changelog.d/search-remote-fetch.security

@@ -0,0 +1 @@
+Status search only fetches remote URLs with resolve=true, on the first page, and for anonymous users only if limit_to_local_content allows it

changelog.d/shared-inbox.change

@@ -0,0 +1 @@
+Deliver to shared inboxes whenever possible, including followers-only posts and DMs, so each server gets one copy

changelog.d/signature-freshness.security

@@ -0,0 +1 @@
+Reject incoming HTTP signatures that do not sign the date, are older than 2 hours (or more than 1 hour in the future), or do not sign the digest of a request body, so captured requests cannot be replayed. Set `config :pleroma, :activitypub, signature_policy: :log` to only log these

changelog.d/stats-peers-from-instances.change

@@ -0,0 +1 @@
+Read the peer list from the instances table (hosts of remote actors, without our own host) and update instance stats every 5 minutes; the old peers query scanned all remote users every minute

changelog.d/status-id-action-rate-limit.fix

@@ -0,0 +1 @@
+Rate limit favourites and reblogs per status, not across all statuses

changelog.d/undo-block-outgoing-blocks.security

@@ -0,0 +1 @@
+Do not federate the Undo of a block when `outgoing_blocks` is disabled

config/config.exs

@@ -373,6 +373,9 @@ config :pleroma, :activitypub,
   note_replies_output_limit: 5,
   sign_object_fetches: true,
   authorized_fetch_mode: false,
+  signature_policy: :enforce,
+  signature_max_age: 7200,
+  signature_max_clock_skew: 3600,
   client_api_enabled: false,
   anonymize_reporter: false,
   anonymize_reporter_local_nickname: ""
@@ -619,7 +622,8 @@ config :pleroma, Oban,
   crontab: [
     {"0 0 * * 0", Pleroma.Workers.Cron.DigestEmailsWorker},
     {"0 0 * * *", Pleroma.Workers.Cron.NewUsersDigestWorker},
-    {"*/10 * * * *", Pleroma.Workers.Cron.AppCleanupWorker}
+    {"*/10 * * * *", Pleroma.Workers.Cron.AppCleanupWorker},
+    {"0 * * * *", Pleroma.Workers.Cron.RetentionWorker}
   ]
 
 config :pleroma, Pleroma.Formatter,
@@ -843,7 +847,7 @@ config :pleroma, :modules, runtime_dir: "instance/modules"
 config :pleroma, configurable_from_database: false
 
 config :pleroma, Pleroma.Repo,
-  parameters: [gin_fuzzy_search_limit: "500", jit: "off"],
+  parameters: [jit: "off"],
   prepare: :unnamed
 
 config :pleroma, :connections_pool,
@@ -928,6 +932,12 @@ config :web_push_encryption, http_client: Pleroma.HTTP.WebPush
 
 config :pleroma, :instances_favicons, enabled: false
 
+config :pleroma, :retention,
+  enabled: false,
+  max_objects: nil,
+  batch_size: 50_000,
+  keep_non_public: false
+
 config :floki, :html_parser, Floki.HTMLParser.FastHtml
 
 config :pleroma, Pleroma.Web.Auth.Authenticator, Pleroma.Web.Auth.PleromaAuthenticator
@@ -948,6 +958,8 @@ config :pleroma, Pleroma.Web.WebFinger, domain: nil, update_nickname_on_user_fet
 
 config :pleroma, Pleroma.Search, module: Pleroma.Search.DatabaseSearch
 
+config :pleroma, Pleroma.Search.DatabaseSearch, gin_fuzzy_search_limit: 500
+
 config :pleroma, Pleroma.Search.Meilisearch,
   url: "http://127.0.0.1:7700/",
   private_key: nil,

config/description.exs

@@ -1788,6 +1788,27 @@ config :pleroma, :config_description, [
         description:
           "List of IPs (CIDR format accepted) to exempt from HTTP Signatures requirement (for example to allow debugging, you shouldn't otherwise need this)"
       },
+      %{
+        key: :signature_policy,
+        type: {:dropdown, :atom},
+        description:
+          "What to do with incoming HTTP signatures that do not sign the date (or `(created)`), are outside the allowed time window, or do not sign the digest of a request body. `:enforce` treats them as invalid, `:log` only logs them",
+        suggestions: [:enforce, :log]
+      },
+      %{
+        key: :signature_max_age,
+        type: :integer,
+        description:
+          "How old, in seconds, the signed date of an incoming HTTP signature can be. Default: 7200 (2 hours)",
+        suggestions: [7200]
+      },
+      %{
+        key: :signature_max_clock_skew,
+        type: :integer,
+        description:
+          "How far, in seconds, the signed date of an incoming HTTP signature can be in the future. Default: 3600 (1 hour). A `(created)` time in the future is always rejected",
+        suggestions: [3600]
+      },
       %{
         key: :note_replies_output_limit,
         type: :integer,
@@ -3310,6 +3331,42 @@ config :pleroma, :config_description, [
       }
     ]
   },
+  %{
+    group: :pleroma,
+    key: :retention,
+    type: :group,
+    description:
+      "Bounded retention: continuously evict old remote threads nobody local interacted with." <>
+        " Local posts and threads with local interactions are always kept.",
+    children: [
+      %{
+        key: :enabled,
+        type: :boolean,
+        description: "Run the retention worker from the Oban crontab"
+      },
+      %{
+        key: :max_objects,
+        type: :integer,
+        description:
+          "Optional watermark. When the objects table is estimated to hold more rows than this," <>
+            " the oldest unpinned remote threads quiet for more than a day are evicted, whatever the retention period.",
+        suggestions: [nil, 1_000_000]
+      },
+      %{
+        key: :batch_size,
+        type: :integer,
+        description:
+          "Number of activities the worker walks per run; their threads are checked and evicted." <>
+            " Default: 50000.",
+        suggestions: [50_000]
+      },
+      %{
+        key: :keep_non_public,
+        type: :boolean,
+        description: "Also keep threads that contain non-public posts"
+      }
+    ]
+  },
   %{
     group: :ex_aws,
     key: :s3,
@@ -3565,6 +3622,21 @@ config :pleroma, :config_description, [
       }
     ]
   },
+  %{
+    group: :pleroma,
+    key: Pleroma.Search.DatabaseSearch,
+    type: :group,
+    description: "Built-in database search settings.",
+    children: [
+      %{
+        key: :gin_fuzzy_search_limit,
+        type: :integer,
+        description:
+          "Limits how many matches PostgreSQL considers in GIN full-text search queries (see `gin_fuzzy_search_limit` in the PostgreSQL docs). Only set for search queries. Keeps search fast on large databases, but results become incomplete and change between requests. Set to `nil` for exact results, which is fine on small instances. Default: 500.",
+        suggestions: [500, nil]
+      }
+    ]
+  },
   %{
     group: :pleroma,
     key: Pleroma.Search.Meilisearch,

docs/administration/CLI_tasks/database.md

@@ -28,6 +28,9 @@ Replaces embedded objects with references to them in the `objects` table. Only n
 
 This will prune remote posts older than 90 days (configurable with [`config :pleroma, :instance, remote_post_retention_days`](../../configuration/cheatsheet.md#instance)) from the database. Pruned posts may be refetched in some cases.
 
+!!! tip
+    To keep the database bounded continuously instead of pruning in one go, see [`:retention`](../../configuration/cheatsheet.md#retention). It uses the same rules as `--keep-threads` but runs in small batches from a cron worker.
+
 !!! note
     The disk space will only be reclaimed after a proper vacuum. By default Postgresql does this for you on a regular basis, but if your instance has been running for a long time and there are many rows deleted, it may be advantageous to use `VACUUM FULL` (e.g. by using the `--vacuum` option).
 

docs/administration/CLI_tasks/instance.md

@@ -37,8 +37,8 @@ If any of the options are left unspecified, you will be prompted interactively.
 - `--static-dir <path>` - the directory custom public files should be read from (custom emojis, frontend bundle overrides, robots.txt, etc.)
 - `--listen-ip <ip>` - the ip the app should listen to, defaults to 127.0.0.1
 - `--listen-port <port>` - the port the app should listen to, defaults to 4000
-- `--strip-uploads-location <Y|N>` - use ExifTool to strip uploads of sensitive location data
-- `--read-uploads-description <Y|N>` - use ExifTool to read image descriptions from uploads
+- `--strip-uploads-location <Y|N>` - strip sensitive location data from uploaded images
+- `--read-uploads-description <Y|N>` - read image descriptions from upload metadata
 - `--anonymize-uploads <Y|N>` - randomize uploaded filenames
 - `--dedupe-uploads <Y|N>` - store files based on their hash to reduce data storage requirements if duplicates are uploaded with different filenames
 - `--skip-release-env` - skip generation the release environment file

docs/configuration/cheatsheet.md

@@ -286,6 +286,9 @@ Notes:
 * `sign_object_fetches`: Sign object fetches with HTTP signatures
 * `authorized_fetch_mode`: Require HTTP signatures for AP fetches
 * `authorized_fetch_mode_exceptions`: List of IPs (CIDR format accepted) to exempt from HTTP Signatures requirement (for example to allow debugging, you shouldn't otherwise need this)
+* `signature_policy`: What to do with incoming HTTP signatures that do not sign the `date` header (or `(created)`), whose signed date is outside the window below, or that do not sign the `digest` header of a request with a body (or whose body Pleroma did not hash). `:enforce` (default) treats them as invalid, `:log` only logs a warning.
+* `signature_max_age`: How old, in seconds, the signed date of an incoming HTTP signature can be. Default: `7200` (2 hours).
+* `signature_max_clock_skew`: How far, in seconds, the signed `date` of an incoming HTTP signature can be in the future. Default: `3600` (1 hour). A `(created)` time in the future is always rejected.
 
 ## Pleroma.User
 
@@ -702,15 +705,19 @@ This filter replaces the filename (not the path) of an upload. For complete obfu
 
 No specific configuration.
 
-#### Pleroma.Upload.Filter.Exiftool.StripLocation
+#### Pleroma.Upload.Filter.Exif.StripLocation
 
-This filter only strips the GPS and location metadata with Exiftool leaving color profiles and attributes intact.
+This filter strips GPS and location metadata from JPEG, WebP, HEIC, AVIF and TIFF images, leaving color profiles and attributes intact. For PNG images it removes all text and EXIF metadata chunks. Images are not re-encoded, and other formats are left unchanged.
+
+It replaces the deprecated `Pleroma.Upload.Filter.Exiftool.StripLocation`, which no longer needs exiftool.
 
 No specific configuration.
 
-#### Pleroma.Upload.Filter.Exiftool.ReadDescription
+#### Pleroma.Upload.Filter.Exif.ReadDescription
 
-This filter reads the ImageDescription and iptc:Caption-Abstract fields with Exiftool so clients can prefill the media description field.
+This filter reads the ImageDescription and iptc:Caption-Abstract fields so clients can prefill the media description field.
+
+It replaces the deprecated `Pleroma.Upload.Filter.Exiftool.ReadDescription`, which no longer needs exiftool.
 
 No specific configuration.
 
@@ -1196,6 +1203,20 @@ Control favicons for instances.
 
 * `enabled`: Allow/disallow displaying and getting instances favicons
 
+## :retention
+
+Bounded retention of remote content. Local posts are your data; remote posts are a cache of somebody else's, and this treats them that way. When enabled, a cron worker (`Pleroma.Workers.Cron.RetentionWorker`, hourly by default) evicts a batch of remote threads that nobody local has interacted with and that have been quiet for longer than `remote_post_retention_days` (see [`:instance`](#instance)). The database then plateaus instead of growing without limit.
+
+A thread is never evicted if it contains a local post, reply, favourite, repeat or reaction, a post bookmarked by a local user, a post addressed to a local user (direct messages and mentions), a post quoted by a local post, a post a local user still has a notification for, or a post that was reported. Relationship activities (follows, blocks) and chat messages are never touched. Evicted posts are refetched on demand when somebody opens them again, as with `prune_objects`.
+
+* `enabled`: Run the worker. Defaults to `false`.
+* `max_objects`: Optional watermark. When the `objects` table is estimated to hold more rows than this, the oldest unpinned remote threads that have been quiet for more than a day are evicted, whatever `remote_post_retention_days` says. Defaults to `nil` (age only).
+* `batch_size`: Number of activities walked per run. Their threads are checked and, if unpinned, evicted. Defaults to `50000`.
+* `keep_non_public`: Also keep threads that contain a non-public post. Defaults to `false`.
+
+!!! note
+    The worker walks the `activities` table in id order, `batch_size` activities per run, up to the id that corresponds to the retention deadline, and remembers its position in the `retention_cursors` table. Each activity is visited once, when it becomes old enough, and only the threads it belongs to are checked through the context index. Over the `max_objects` watermark a second walk with its own cursor runs up to one day ago instead. The first run starts from the oldest activity; on an instance that never pruned this backlog takes many runs, so run `prune_objects --keep-threads --prune-orphaned-activities` once for the initial cut. Threads that were kept and later lost their pin (removed bookmarks, cleared notifications of other kinds) are only revisited on a fresh walk: delete the row from `retention_cursors` to restart from the beginning. Space freed by eviction is reused by new rows; it is not handed back to the filesystem unless you `VACUUM FULL`.
+
 ## Pleroma.User.Backup
 
 !!! note

docs/configuration/search.md

@@ -10,6 +10,14 @@ To use built-in search that has no external dependencies, set the search module
 
 While it has no external dependencies, it has problems with performance and relevancy.
 
+With a GIN index (the default), search queries limit how many matches PostgreSQL considers (`gin_fuzzy_search_limit`, see [PostgreSQL's docs](https://www.postgresql.org/docs/current/gin.html#GIN-TIPS)). This keeps searches for common words fast on large databases, but the results are incomplete and can change between requests. On smaller instances you can turn it off for exact results:
+
+> config :pleroma, Pleroma.Search.DatabaseSearch, gin_fuzzy_search_limit: nil
+
+Without a limit, searches for common words can take longer than the database timeout on large databases, and then return nothing.
+
+Earlier versions of Pleroma set this limit for every database connection, which also made other GIN index scans (for example recipient lookups) incomplete. It is now only set for search queries.
+
 ## QdrantSearch
 
 This uses the vector search engine [Qdrant](https://qdrant.tech) to search the posts in a vector space. This needs a way to generate embeddings and uses the [OpenAI API](https://platform.openai.com/docs/guides/embeddings/what-are-embeddings). This is implemented by several project besides OpenAI itself, including the python-based fastembed-server found in `supplemental/search/fastembed-api`.

docs/installation/alpine_linux_en.md

@@ -68,7 +68,7 @@ sudo rc-update add postgresql
 ### Install media / graphics packages (optional, see [`docs/installation/optional/media_graphics_packages.md`](../installation/optional/media_graphics_packages.md))
 
 ```shell
-sudo apk add ffmpeg imagemagick exiftool
+sudo apk add ffmpeg imagemagick
 ```
 
 ### Install PleromaBE

docs/installation/debian_based_en.md

@@ -36,7 +36,7 @@ sudo apt install elixir erlang-dev erlang-nox
 ### Optional packages: [`docs/installation/optional/media_graphics_packages.md`](../installation/optional/media_graphics_packages.md)
 
 ```shell
-sudo apt install imagemagick ffmpeg libimage-exiftool-perl
+sudo apt install imagemagick ffmpeg
 ```
 
 ### Install PleromaBE

docs/installation/debian_based_jp.md

@@ -28,7 +28,6 @@ Note: This article is potentially outdated because at this time we may not have
 - `certbot` (または何らかのLet's Encrypt向けACMEクライアント)
 - `ImageMagick`
 - `ffmpeg`
-- `exiftool`
 
 ### システムを準備する
 
@@ -60,7 +59,7 @@ sudo apt install elixir erlang-dev erlang-nox
 ### オプションパッケージ: [`docs/installation/optional/media_graphics_packages.md`](../installation/optional/media_graphics_packages.md)
 
 ```shell
-sudo apt install imagemagick ffmpeg libimage-exiftool-perl
+sudo apt install imagemagick ffmpeg
 ```
 
 ### Pleroma BE (バックエンド) をインストールします

docs/installation/freebsd_en.md

@@ -31,7 +31,7 @@ Setup the required services to automatically start at boot, using `sysrc(8)`.
 ### Install media / graphics packages (optional, see [`docs/installation/optional/media_graphics_packages.md`](../installation/optional/media_graphics_packages.md))
 
 ```shell
-# pkg install imagemagick ffmpeg p5-Image-ExifTool vips
+# pkg install imagemagick ffmpeg vips
 ```
 
 ## Configuring Pleroma

docs/installation/generic_dependencies.include

@@ -12,7 +12,6 @@
 
 * ImageMagick
 * FFmpeg
-* exiftool
 
 !!!info
     **Source installs only**  

docs/installation/gentoo_en.md

@@ -41,7 +41,6 @@ Gentoo quite pointedly does not come with a cron daemon installed, and as such i
 * `app-crypt/certbot-nginx` (nginx certbot plugin that allows use of the all-powerful `--nginx` flag on certbot)
 * `media-gfx/imagemagick`
 * `media-video/ffmpeg`
-* `media-libs/exiftool`
 
 ### Prepare the system
 
@@ -97,7 +96,7 @@ Not only does this make it much easier to deploy changes you make, as you can co
 ### Install media / graphics packages (optional, see [`docs/installation/optional/media_graphics_packages.md`](docs/installation/optional/media_graphics_packages.md))
 
 ```shell
-# emerge --ask media-video/ffmpeg media-gfx/imagemagick media-libs/exiftool
+# emerge --ask media-video/ffmpeg media-gfx/imagemagick
 ```
 
 ### Install PleromaBE

docs/installation/gentoo_otp_en.md

@@ -25,7 +25,6 @@ Gentoo quite pointedly does not come with a cron daemon installed, and as such i
 * `app-crypt/certbot-nginx` (nginx certbot plugin that allows use of the all-powerful `--nginx` flag on certbot)
 * `media-gfx/imagemagick`
 * `media-video/ffmpeg`
-* `media-libs/exiftool`
 
 ### Prepare the system
 
@@ -68,7 +67,7 @@ The output from emerging postgresql should give you a command for initializing t
 See [Optional software packages needed for specific functionality](optional/media_graphics_packages.md) for details.
 
 ```shell
-# emerge --ask media-video/ffmpeg media-gfx/imagemagick media-libs/exiftool
+# emerge --ask media-video/ffmpeg media-gfx/imagemagick
 ```
 
 ### Setup PleromaBE

docs/installation/openbsd_en.md

@@ -26,12 +26,11 @@ Independently of the reverse proxy, [acme-client(1)](https://man.openbsd.org/acm
 
   * ImageMagick
   * ffmpeg
-  * exiftool
 
 To install the above:
 
 ```
-# pkg_add ImageMagick ffmpeg p5-Image-ExifTool
+# pkg_add ImageMagick ffmpeg
 ```
 
 For more information read [`docs/installation/optional/media_graphics_packages.md`](../installation/optional/media_graphics_packages.md):

docs/installation/openbsd_fi.md

@@ -25,11 +25,10 @@ Asenna tarvittava ohjelmisto:
 [`docs/installation/optional/media_graphics_packages.md`](../installation/optional/media_graphics_packages.md):
   * ImageMagick
   * ffmpeg
-  * exiftool
 
 Asenna tarvittava ohjelmisto:
 
-`# pkg_add ImageMagick ffmpeg p5-Image-ExifTool`
+`# pkg_add ImageMagick ffmpeg`
 
 Luo postgresql-tietokanta:
 

docs/installation/optional/media_graphics_packages.md

@@ -2,9 +2,8 @@
 
 For specific Pleroma functionality (which is disabled by default) some or all of the below packages are required:
 
-* `ImageMagic`
+* `ImageMagick`
 * `ffmpeg`
-* `exiftool`
   
 Please refer to documentation in `docs/installation` on how to install them on specific OS.
   
@@ -26,12 +25,3 @@ It is required for the following Pleroma features:
 It is required for the following Pleroma features:
 
 * Media preview proxy for videos (related config: `media_preview_proxy/enabled` in `config/config.exs`)
-
-## `exiftool`
-
-`exiftool` is media files metadata reader/writer.
-
-It is required for the following Pleroma features:
-
-* `Pleroma.Upload.Filters.Exiftool.StripLocation` upload filter (related config: `Pleroma.Upload/filters` in `config/config.exs`)
-* `Pleroma.Upload.Filters.Exiftool.ReadDescription` upload filter (related config: `Pleroma.Upload/filters` in `config/config.exs`)

docs/installation/otp_en.md

@@ -61,17 +61,16 @@ Other than things bundled in the OTP release Pleroma depends on:
 Per [`docs/installation/optional/media_graphics_packages.md`](optional/media_graphics_packages.md):
   * ImageMagick
   * ffmpeg
-  * exiftool
 
 === "Alpine"
     ```
     apk update
-    apk add imagemagick ffmpeg exiftool
+    apk add imagemagick ffmpeg
     ```
 
 === "Debian/Ubuntu"
     ```
-    apt install imagemagick ffmpeg libimage-exiftool-perl
+    apt install imagemagick ffmpeg
     ```
 
 ## Setup

installation/release-to-docker/Dockerfile

@@ -12,7 +12,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     openssl libssl3 \
     libmagic1t64 file \
     postgresql-client \
-    ffmpeg imagemagick libimage-exiftool-perl \
+    ffmpeg imagemagick \
     libvips42t64 \
     unzip \
     curl \

lib/mix/pleroma.ex

@@ -17,7 +17,7 @@ defmodule Mix.Pleroma do
     :oban,
     :logger_backends
   ]
-  @cachex_children ["object", "user", "scrubber", "web_resp"]
+  @cachex_children ["object", "user", "scrubber", "web_resp", "http_backoff"]
   @doc "Common functions to be reused in mix tasks"
   def start_pleroma do
     Pleroma.Config.Holder.save_default()

lib/mix/tasks/pleroma/database.ex

@@ -113,36 +113,13 @@ defmodule Mix.Tasks.Pleroma.Database do
     Logger.info(log_message)
 
     if Keyword.get(options, :keep_threads) do
-      # We want to delete objects from threads where
-      # 1. the newest post is still old
-      # 2. none of the activities is local
-      # 3. none of the activities is bookmarked
-      # 4. optionally none of the posts is non-public
+      # Delete objects from threads that nobody local interacted with and that
+      # have been quiet for longer than the retention period, see Pleroma.Retention.
       deletable_context =
-        if Keyword.get(options, :keep_non_public) do
-          Pleroma.Activity
-          |> join(:left, [a], b in Pleroma.Bookmark, on: a.id == b.activity_id)
-          |> group_by([a], fragment("? ->> 'context'::text", a.data))
-          |> having(
-            [a],
-            not fragment(
-              # Posts (checked on Create Activity) is non-public
-              "bool_or((not(?->'to' \\? ? OR ?->'cc' \\? ?)) and ? ->> 'type' = 'Create')",
-              a.data,
-              ^Pleroma.Constants.as_public(),
-              a.data,
-              ^Pleroma.Constants.as_public(),
-              a.data
-            )
-          )
-        else
-          Pleroma.Activity
-          |> join(:left, [a], b in Pleroma.Bookmark, on: a.id == b.activity_id)
-          |> group_by([a], fragment("? ->> 'context'::text", a.data))
-        end
-        |> having([a], max(a.updated_at) < ^time_deadline)
-        |> having([a], not fragment("bool_or(?)", a.local))
-        |> having([_, b], fragment("max(?::text) is null", b.id))
+        time_deadline
+        |> Pleroma.Retention.evictable_contexts_query(
+          keep_non_public: Keyword.get(options, :keep_non_public)
+        )
         |> select([a], fragment("? ->> 'context'::text", a.data))
 
       Pleroma.Object
@@ -222,18 +199,7 @@ defmodule Mix.Tasks.Pleroma.Database do
       |> Repo.query([], timeout: :infinity)
     end
 
-    """
-    DELETE FROM hashtags AS ht
-    WHERE NOT EXISTS (
-      SELECT 1 FROM hashtags_objects hto
-      WHERE ht.id = hto.hashtag_id
-    )
-    AND NOT EXISTS (
-      SELECT 1 FROM user_follows_hashtag ufh
-      WHERE ht.id = ufh.hashtag_id
-    )
-    """
-    |> Repo.query()
+    Pleroma.Retention.delete_unused_hashtags()
 
     if Keyword.get(options, :vacuum) do
       Maintenance.vacuum("full")

lib/mix/tasks/pleroma/instance.ex

@@ -162,38 +162,20 @@ defmodule Mix.Tasks.Pleroma.Instance do
         )
         |> Path.expand()
 
-      {strip_uploads_location_message, strip_uploads_location_default} =
-        if Pleroma.Utils.command_available?("exiftool") do
-          {"Do you want to strip location (GPS) data from uploaded images? This requires exiftool, it was detected as installed. (y/n)",
-           "y"}
-        else
-          {"Do you want to strip location (GPS) data from uploaded images? This requires exiftool, it was detected as not installed, please install it if you answer yes. (y/n)",
-           "n"}
-        end
-
       strip_uploads_location =
         get_option(
           options,
           :strip_uploads_location,
-          strip_uploads_location_message,
-          strip_uploads_location_default
+          "Do you want to strip location (GPS) data from uploaded images? (y/n)",
+          "y"
         ) === "y"
 
-      {read_uploads_description_message, read_uploads_description_default} =
-        if Pleroma.Utils.command_available?("exiftool") do
-          {"Do you want to read data from uploaded files so clients can use it to prefill fields like image description? This requires exiftool, it was detected as installed. (y/n)",
-           "y"}
-        else
-          {"Do you want to read data from uploaded files so clients can use it to prefill fields like image description? This requires exiftool, it was detected as not installed, please install it if you answer yes. (y/n)",
-           "n"}
-        end
-
       read_uploads_description =
         get_option(
           options,
           :read_uploads_description,
-          read_uploads_description_message,
-          read_uploads_description_default
+          "Do you want to read metadata from uploaded images so clients can prefill the media description field? (y/n)",
+          "y"
         ) === "y"
 
       anonymize_uploads =
@@ -324,14 +306,14 @@ defmodule Mix.Tasks.Pleroma.Instance do
   defp upload_filters(filters) when is_map(filters) do
     enabled_filters =
       if filters.strip_location do
-        [Pleroma.Upload.Filter.Exiftool.StripLocation]
+        [Pleroma.Upload.Filter.Exif.StripLocation]
       else
         []
       end
 
     enabled_filters =
       if filters.read_description do
-        enabled_filters ++ [Pleroma.Upload.Filter.Exiftool.ReadDescription]
+        enabled_filters ++ [Pleroma.Upload.Filter.Exif.ReadDescription]
       else
         enabled_filters
       end

lib/pleroma/application.ex

@@ -198,6 +198,7 @@ defmodule Pleroma.Application do
       ),
       build_cachex("rel_me", limit: 2500),
       build_cachex("host_meta", default_ttl: :timer.minutes(120), limit: 5_000),
+      build_cachex("http_backoff", limit: 10_000),
       build_cachex("translations", default_ttl: :timer.hours(24), limit: 5_000)
     ]
   end

lib/pleroma/application_requirements.ex

@@ -168,8 +168,6 @@ defmodule Pleroma.ApplicationRequirements do
 
   defp check_system_commands!(:ok) do
     filter_commands_statuses = [
-      check_filter(Pleroma.Upload.Filter.Exiftool.StripLocation, "exiftool"),
-      check_filter(Pleroma.Upload.Filter.Exiftool.ReadDescription, "exiftool"),
       check_filter(Pleroma.Upload.Filter.Mogrify, "mogrify"),
       check_filter(Pleroma.Upload.Filter.Mogrifun, "mogrify"),
       check_filter(Pleroma.Upload.Filter.AnalyzeMetadata, "ffprobe")

lib/pleroma/config/deprecation_warnings.ex

@@ -22,40 +22,44 @@ defmodule Pleroma.Config.DeprecationWarnings do
 
   @logger_formatter_config_knobs [:colors, :format, :metadata, :truncate, :utc_log]
 
+  @exiftool_filter_replacements %{
+    Pleroma.Upload.Filter.Exiftool => Pleroma.Upload.Filter.Exif.StripLocation,
+    Pleroma.Upload.Filter.Exiftool.StripLocation => Pleroma.Upload.Filter.Exif.StripLocation,
+    Pleroma.Upload.Filter.Exiftool.ReadDescription => Pleroma.Upload.Filter.Exif.ReadDescription
+  }
+
   def check_exiftool_filter do
     filters = Config.get([Pleroma.Upload]) |> Keyword.get(:filters, [])
 
-    if Pleroma.Upload.Filter.Exiftool in filters do
-      Logger.warning("""
-      !!!DEPRECATION WARNING!!!
-      Your config is using Exiftool as a filter instead of Exiftool.StripLocation. This should work for now, but you are advised to change to the new configuration to prevent possible issues later:
+    deprecated_filters =
+      filters
+      |> Enum.filter(&Map.has_key?(@exiftool_filter_replacements, &1))
+      |> Enum.uniq()
 
-      ```
-      config :pleroma, Pleroma.Upload,
-        filters: [Pleroma.Upload.Filter.Exiftool]
-      ```
+    case deprecated_filters do
+      [] ->
+        :ok
 
-      Is now
+      deprecated_filters ->
+        replacements =
+          Enum.map_join(deprecated_filters, fn filter ->
+            "\n* `#{inspect(filter)}` is now `#{inspect(@exiftool_filter_replacements[filter])}`"
+          end)
 
+        Logger.warning("""
+        !!!DEPRECATION WARNING!!!
+        Your config is using Exiftool upload filters. Pleroma no longer needs exiftool, the filters were replaced by built-in ones. The old names should work for now, but you are advised to change to the new configuration to prevent possible issues later:
+        #{replacements}
+        """)
 
-      ```
-      config :pleroma, Pleroma.Upload,
-        filters: [Pleroma.Upload.Filter.Exiftool.StripLocation]
-      ```
-      """)
+        new_config =
+          filters
+          |> Enum.map(&Map.get(@exiftool_filter_replacements, &1, &1))
+          |> Enum.uniq()
 
-      new_config =
-        filters
-        |> Enum.map(fn
-          Pleroma.Upload.Filter.Exiftool -> Pleroma.Upload.Filter.Exiftool.StripLocation
-          filter -> filter
-        end)
+        Config.put([Pleroma.Upload, :filters], new_config)
 
-      Config.put([Pleroma.Upload, :filters], new_config)
-
-      :error
-    else
-      :ok
+        :error
     end
   end
 

lib/pleroma/config/transfer_task.ex

@@ -48,6 +48,7 @@ defmodule Pleroma.Config.TransferTask do
       settings =
         (Repo.all(ConfigDB) ++ deleted_settings)
         |> Enum.reject(&ConfigDB.static?/1)
+        |> Enum.reject(&(&1.group == :invalid_atom or &1.key == :invalid_atom))
         |> Enum.map(&merge_with_default/1)
 
       started_applications = Application.started_applications()

lib/pleroma/ecto_type/config/atom.ex

@@ -5,6 +5,8 @@
 defmodule Pleroma.EctoType.Config.Atom do
   use Ecto.Type
 
+  require Logger
+
   def type, do: :atom
 
   def cast(key) when is_atom(key) do
@@ -17,8 +19,17 @@ defmodule Pleroma.EctoType.Config.Atom do
 
   def cast(_), do: :error
 
+  # A key or group naming a module that no longer exists (e.g. a removed uploader)
+  # must not crash loading
   def load(key) do
     {:ok, Pleroma.ConfigDB.string_to_elixir_types(key)}
+  rescue
+    ArgumentError ->
+      Logger.warning(
+        "ConfigDB: unknown module #{key} in a setting key or group, the setting is not applied"
+      )
+
+      {:ok, :invalid_atom}
   end
 
   def dump(key) when is_atom(key), do: {:ok, inspect(key)}

lib/pleroma/gun/conn.ex

@@ -14,7 +14,9 @@ defmodule Pleroma.Gun.Conn do
       opts
       |> Enum.into(%{})
       |> Map.put_new(:connect_timeout, pool_opts[:connect_timeout] || 5_000)
-      |> Map.put_new(:supervise, false)
+      # Under gun's supervisor, a failed connect does not kill the caller
+      # through the link. Gun still stops when its owner goes down.
+      |> Map.put_new(:supervise, true)
       |> maybe_add_tls_opts(uri)
 
     do_open(uri, opts)
@@ -117,8 +119,7 @@ defmodule Pleroma.Gun.Conn do
     host = Pleroma.HTTP.AdapterHelper.parse_host(host)
     opts = Map.put(opts, :transport, transport(uri.scheme))
 
-    with {:ok, conn} <- Gun.open(host, port, opts),
-         {:ok, protocol} <- await_up(conn, opts[:connect_timeout]) do
+    with {:ok, conn, protocol} <- open_direct(host, port, opts) do
       {:ok, conn, protocol, nil}
     else
       error ->
@@ -130,6 +131,40 @@ defmodule Pleroma.Gun.Conn do
     end
   end
 
+  defp open_direct(host, port, opts) do
+    result =
+      with {:ok, conn} <- Gun.open(host, port, opts),
+           {:ok, protocol} <- await_up(conn, opts[:connect_timeout]) do
+        {:ok, conn, protocol}
+      end
+
+    case inet6_fallback(result, host, opts) do
+      {:ok, inet6_opts} -> open_direct(host, port, inet6_opts)
+      :error -> result
+    end
+  end
+
+  # Gun's default when :tcp_opts is not set.
+  @default_tcp_opts [send_timeout: 15_000, send_timeout_close: true]
+
+  @doc """
+  Gun resolves hostnames with `inet_tcp` (A records only) unless `:inet6` is
+  in `:tcp_opts`, for both the :tcp and :tls transports. When the A lookup
+  finds nothing, returns the options for one more attempt over IPv6.
+  """
+  @spec inet6_fallback(term(), Pleroma.HTTP.AdapterHelper.host(), map()) :: {:ok, map()} | :error
+  def inet6_fallback({:error, {:down, {:shutdown, :nxdomain}}}, host, opts) when is_list(host) do
+    tcp_opts = Map.get(opts, :tcp_opts, @default_tcp_opts)
+
+    if :inet6 in tcp_opts do
+      :error
+    else
+      {:ok, Map.put(opts, :tcp_opts, [:inet6 | tcp_opts])}
+    end
+  end
+
+  def inet6_fallback(_result, _host, _opts), do: :error
+
   defp destination_opts(%URI{host: host, port: port}) do
     host = Pleroma.HTTP.AdapterHelper.parse_host(host)
     %{host: host, port: port}

lib/pleroma/html.ex

@@ -66,14 +66,16 @@ defmodule Pleroma.HTML do
   end
 
   @spec extract_first_external_url_from_object(Pleroma.Object.t()) :: String.t() | nil
-  def extract_first_external_url_from_object(%{data: %{"content" => content}})
+  def extract_first_external_url_from_object(%{data: %{"content" => content} = data})
       when is_binary(content) do
+    # A quote post's fallback link ("RE: <url>", in a .quote-inline element or
+    # bare) points at the quoted post, which is shown already.
     content
     |> Floki.parse_fragment!()
+    |> Floki.filter_out(".quote-inline")
     |> Floki.find("a:not(.mention,.hashtag,.attachment,[rel~=\"tag\"])")
-    |> Enum.take(1)
     |> Floki.attribute("href")
-    |> Enum.at(0)
+    |> Enum.find(&(&1 != data["quoteUrl"]))
   end
 
   def extract_first_external_url_from_object(_), do: nil

lib/pleroma/http/adapter_helper.ex

@@ -20,8 +20,9 @@ defmodule Pleroma.HTTP.AdapterHelper do
 
   @callback options(keyword(), URI.t()) :: keyword()
 
+  # AdminFE stores an empty proxy field as "", which means no proxy.
   @spec format_proxy(String.t() | tuple() | nil) :: proxy() | nil
-  def format_proxy(nil), do: nil
+  def format_proxy(proxy) when proxy in [nil, ""], do: nil
 
   def format_proxy(proxy_url) do
     case parse_proxy(proxy_url) do
@@ -32,7 +33,7 @@ defmodule Pleroma.HTTP.AdapterHelper do
   end
 
   @spec maybe_add_proxy(keyword(), proxy() | nil) :: keyword()
-  def maybe_add_proxy(opts, nil), do: opts
+  def maybe_add_proxy(opts, proxy) when proxy in [nil, ""], do: opts
   def maybe_add_proxy(opts, proxy), do: Keyword.put_new(opts, :proxy, proxy)
 
   @doc """

lib/pleroma/http/backoff.ex

@@ -0,0 +1,153 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.HTTP.Backoff do
+  @moduledoc """
+  GET wrapper that stops sending requests to a host after it answers
+  429 or 503, until the time given by `Retry-After` or `X-RateLimit-Reset`.
+
+  Based on Akkoma's `Pleroma.HTTP.Backoff`.
+  """
+
+  alias Pleroma.HTTP
+
+  require Logger
+
+  @cachex Pleroma.Config.get([:cachex, :provider], Cachex)
+  @cache :http_backoff_cache
+
+  @default_seconds 5 * 60
+  @max_seconds 60 * 60
+  @max_job_age 2 * 60 * 60
+
+  @spec get(String.t(), list(), keyword()) :: {:ok, Tesla.Env.t()} | {:error, any()}
+  def get(url, headers \\ [], options \\ []) when is_binary(url) do
+    host = url |> URI.parse() |> Map.get(:host) |> downcase()
+
+    case @cachex.get(@cache, host) do
+      {:ok, true} ->
+        {:error, :ratelimit}
+
+      _ ->
+        url
+        |> HTTP.get(headers, options)
+        |> check_backoff(host)
+    end
+  end
+
+  defp downcase(nil), do: nil
+  defp downcase(host), do: String.downcase(host)
+
+  defp check_backoff({:ok, %{status: status, headers: headers} = env}, host)
+       when status in [429, 503] do
+    host = response_host(env, host)
+    seconds = backoff_seconds(headers, DateTime.utc_now())
+    Logger.warning("#{host} answered #{status}, backing off for #{seconds} seconds")
+    @cachex.put(@cache, host, true, ttl: :timer.seconds(seconds))
+    {:error, :ratelimit}
+  end
+
+  defp check_backoff(response, _host), do: response
+
+  # After a redirect, the 429/503 came from the last host, not the requested one.
+  defp response_host(%{url: url}, host) when is_binary(url) do
+    URI.parse(url).host |> downcase() || host
+  end
+
+  defp response_host(_env, host), do: host
+
+  @doc """
+  Turns an Oban job's `{:error, :ratelimit}` result into a snooze for the
+  default backoff, so the job's attempts aren't all spent while the host is
+  backed off. Jobs inserted more than #{@max_job_age} seconds ago get the
+  error back and fail as usual.
+  """
+  @spec snooze_ratelimited(any(), Oban.Job.t()) :: any()
+  def snooze_ratelimited({:error, :ratelimit} = error, %Oban.Job{inserted_at: inserted_at}) do
+    if is_nil(inserted_at) or DateTime.diff(DateTime.utc_now(), inserted_at) < @max_job_age do
+      {:snooze, @default_seconds}
+    else
+      error
+    end
+  end
+
+  def snooze_ratelimited(result, _job), do: result
+
+  @doc """
+  Seconds to back off for, from the first valid `Retry-After` or
+  `X-RateLimit-Reset` header. Missing, invalid or past values give the
+  default of #{@default_seconds} seconds; values are capped at #{@max_seconds}.
+  """
+  @spec backoff_seconds(list(), DateTime.t()) :: pos_integer()
+  def backoff_seconds(headers, now) do
+    [
+      parse_retry_after(header(headers, "retry-after"), now),
+      parse_ratelimit_reset(header(headers, "x-ratelimit-reset"), now)
+    ]
+    |> Enum.find(@default_seconds, &is_integer/1)
+    |> min(@max_seconds)
+  end
+
+  defp header(headers, name) do
+    Enum.find_value(headers, fn
+      {key, value} when is_binary(key) and is_binary(value) ->
+        if String.downcase(key) == name, do: String.trim(value)
+
+      _ ->
+        nil
+    end)
+  end
+
+  # Retry-After is either delta-seconds or an HTTP-date (RFC 9110, 10.2.3)
+  defp parse_retry_after(nil, _now), do: nil
+
+  defp parse_retry_after(value, now) do
+    case Integer.parse(value) do
+      {seconds, ""} when seconds > 0 -> seconds
+      {_, ""} -> nil
+      _ -> parse_timestamp(value, now)
+    end
+  end
+
+  # Servers send X-RateLimit-Reset as a timestamp (ISO 8601 like Mastodon, or
+  # Unix seconds) or as seconds until the reset.
+  @unix_timestamp_min 1_000_000_000
+
+  defp parse_ratelimit_reset(nil, _now), do: nil
+
+  defp parse_ratelimit_reset(value, now) do
+    case Integer.parse(value) do
+      {timestamp, ""} when timestamp >= @unix_timestamp_min ->
+        timestamp - DateTime.to_unix(now)
+
+      {seconds, ""} ->
+        seconds
+
+      _ ->
+        parse_timestamp(value, now)
+    end
+    |> positive()
+  end
+
+  defp positive(seconds) when is_integer(seconds) and seconds > 0, do: seconds
+  defp positive(_), do: nil
+
+  defp parse_timestamp(nil, _now), do: nil
+
+  defp parse_timestamp(value, now) do
+    with {:ok, %DateTime{} = datetime} <- parse_datetime(value),
+         seconds when seconds > 0 <- DateTime.diff(datetime, now) do
+      seconds
+    else
+      _ -> nil
+    end
+  end
+
+  defp parse_datetime(value) do
+    with {:error, _} <- Timex.parse(value, "{RFC1123}"),
+         {:ok, datetime, _offset} <- DateTime.from_iso8601(value) do
+      {:ok, datetime}
+    end
+  end
+end

lib/pleroma/instances/instance.ex

@@ -91,6 +91,13 @@ defmodule Pleroma.Instances.Instance do
 
   def reachable?(url_or_host) when is_binary(url_or_host), do: true
 
+  @doc "Creates a record for the instance unless one exists. Existing records are kept."
+  def ensure_exists(url_or_host) when is_binary(url_or_host) do
+    %Instance{host: host(url_or_host)}
+    |> changeset(%{})
+    |> Repo.insert(on_conflict: :nothing, conflict_target: :host)
+  end
+
   def set_reachable(url_or_host) when is_binary(url_or_host) do
     host = host(url_or_host)
 

lib/pleroma/object/fetcher.ex

@@ -216,7 +216,7 @@ defmodule Pleroma.Object.Fetcher do
       |> maybe_date_fetch(date)
       |> sign_fetch(id, date)
 
-    case HTTP.get(id, headers) do
+    case HTTP.Backoff.get(id, headers) do
       {:ok, %{body: body, status: code, headers: headers, url: final_url}}
       when code in 200..299 ->
         remote_host = if final_url, do: URI.parse(final_url).host, else: nil

lib/pleroma/retention.ex

@@ -0,0 +1,473 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Retention do
+  @moduledoc """
+  Bounded retention of remote content.
+
+  Local content is data; remote content is a cache of somebody else's data.
+  This module treats it that way: a remote thread is evictable when nobody
+  local has interacted with it and it has been quiet for longer than
+  `remote_post_retention_days`, or for a day when the objects table has grown
+  past the configured watermark. Local posts are never evicted.
+
+  A thread is pinned (never evicted) if any activity in it
+
+    * is local (posts, replies, favourites, repeats and reactions by local
+      users all carry the thread's context),
+    * is bookmarked by a local user,
+    * is addressed to a local user (direct messages and mentions),
+    * is a post quoted by a local post,
+    * still has a notification for a local user,
+    * is a report (Flag),
+
+  or if any object in it is referenced by a report.
+
+  Only activities that carry a context take part; relationship activities
+  such as Follow or Block and chat messages have none and are never touched.
+
+  ## How a run works
+
+  Grouping the whole activities table by thread is far too expensive to do
+  every hour on a large instance, so `run/1` walks the table incrementally
+  instead. Activity ids are ordered by creation time, so a run scans the next
+  `batch_size` activities after a persisted cursor (`Pleroma.Retention.Cursor`)
+  up to the id that corresponds to the retention deadline, collects the
+  threads they belong to, and verifies just those threads through the
+  `(type, context)` index. Threads that pass are evicted; the cursor moves on.
+  Every activity is thus visited exactly once, when it becomes old enough,
+  and a thread is re-checked whenever another of its activities ages out.
+  A thread that was kept because an activity's `updated_at` was bumped
+  after insertion is only re-checked on the next full walk, i.e. after a
+  cursor reset. Over the object watermark a second walk, with its own
+  cursor, runs up to a day ago instead of the deadline.
+
+  The first run starts from the oldest activity, which doubles as the
+  initial cut on an instance that never pruned. `prune_objects --keep-threads`
+  does the same job in one go and is faster for that purpose.
+  """
+
+  import Ecto.Query
+
+  alias Pleroma.Activity
+  alias Pleroma.Bookmark
+  alias Pleroma.Config
+  alias Pleroma.Notification
+  alias Pleroma.Object
+  alias Pleroma.Repo
+  alias Pleroma.Retention.Cursor
+
+  require Logger
+  require Pleroma.Constants
+
+  @type stats :: %{
+          contexts: non_neg_integer(),
+          objects: non_neg_integer(),
+          activities: non_neg_integer()
+        }
+
+  @empty_stats %{contexts: 0, objects: 0, activities: 0}
+
+  @default_batch_size 50_000
+  # Over the watermark a thread must still have been quiet this long, so
+  # posts people are reading right now are not evicted from under them.
+  @watermark_min_age 86_400
+  # Activities per verification chunk; the cursor is persisted after each.
+  @chunk 5_000
+
+  # Activity types that may carry a thread's context. Restricting the
+  # per-thread lookups to these lets them use the (type, context) index.
+  #
+  # Invariant: every type that *adopts* another activity's context must be
+  # listed, or its thread's local pins become invisible to the walk and the
+  # thread gets evicted. Today only Create, Announce, Like and EmojiReact do
+  # (see Pleroma.Web.ActivityPub.Builder), and EmojiReaction, the name
+  # EmojiReact had in older data. The others carry a context of their own (or
+  # did in older data) and are listed so they get evicted rather than left
+  # behind. The regression test "keeps threads pinned by each local
+  # interaction type" guards this.
+  @context_types ~w(Create Announce Like EmojiReact EmojiReaction Update Delete Undo Flag Listen Move)
+
+  @doc """
+  Runs one eviction batch using the `:retention` config and returns what was removed.
+  """
+  @spec run(keyword()) :: stats()
+  def run(config \\ Config.get(:retention, [])) do
+    budget = config[:batch_size] || @default_batch_size
+
+    # Over the watermark every unpinned thread older than a day is fair game,
+    # so a separate walk goes up to that point. It must not share the cursor
+    # with the age-based walk, or that one would be left behind the deadline.
+    {cursor, deadline} =
+      if over_watermark?(config[:max_objects]),
+        do: {"activities_watermark", watermark_deadline()},
+        else: {"activities", deadline()}
+
+    boundary = id_at(deadline)
+
+    opts = [
+      keep_non_public: config[:keep_non_public],
+      skip_reported: reported_contexts()
+    ]
+
+    from = Cursor.get(cursor)
+    rows = candidates(from, boundary, budget)
+
+    stats =
+      rows
+      |> Enum.chunk_every(@chunk)
+      |> Enum.reduce(@empty_stats, fn chunk, acc ->
+        stats = chunk |> contexts_of() |> evict_threads(deadline, opts)
+        # Persist progress per chunk, so a killed or crashed run resumes
+        # where it stopped instead of re-walking the whole range.
+        Cursor.put(cursor, chunk |> List.last() |> elem(0))
+        Map.merge(acc, stats, fn _key, x, y -> x + y end)
+      end)
+
+    # Fewer rows than asked for: everything up to the boundary is done.
+    if length(rows) < budget and after?(boundary, from), do: Cursor.put(cursor, boundary)
+
+    stats
+  end
+
+  # The next `budget` activities after `from`, up to `boundary`.
+  defp candidates(from, boundary, budget) do
+    Activity
+    |> where([a], a.id <= ^boundary)
+    |> maybe_after(from)
+    |> order_by([a], asc: a.id)
+    |> limit(^budget)
+    |> select([a], {a.id, a.local, fragment("? ->> 'context'", a.data)})
+    |> Repo.all(timeout: :infinity)
+  end
+
+  defp maybe_after(query, nil), do: query
+  defp maybe_after(query, from), do: where(query, [a], a.id > ^from)
+
+  defp contexts_of(rows) do
+    for {_id, false, context} when is_binary(context) <- rows, uniq: true, do: context
+  end
+
+  defp after?(_id, nil), do: true
+  defp after?(id, other), do: FlakeId.from_string(id) > FlakeId.from_string(other)
+
+  # Verifies the given threads against the pin rules and evicts those that
+  # pass. The transaction makes a chunk all-or-nothing but does not lock the
+  # threads: a local interaction landing between check and delete survives,
+  # pins the thread from then on, and its object is refetched when needed.
+  defp evict_threads([], _deadline, _opts), do: @empty_stats
+
+  defp evict_threads(contexts, deadline, opts) do
+    query =
+      deadline
+      |> evictable_contexts_query(Keyword.put(opts, :contexts, contexts))
+      |> select([a], %{
+        context: fragment("? ->> 'context'", a.data),
+        activity_ids: type(fragment("array_agg(?)", a.id), {:array, FlakeId.Ecto.CompatType}),
+        object_ids: fragment("array_agg(associated_object_id(?))", a.data)
+      })
+
+    {:ok, stats} =
+      Repo.transaction(
+        fn ->
+          query
+          |> Repo.all(timeout: :infinity)
+          |> evict()
+        end,
+        timeout: :infinity
+      )
+
+    stats
+  end
+
+  # The activity id a flake would have received at the given time. Ids at or
+  # below it belong to activities created before then.
+  defp id_at(%NaiveDateTime{} = time) do
+    ms = time |> DateTime.from_naive!("Etc/UTC") |> DateTime.to_unix(:millisecond) |> max(0)
+    FlakeId.to_string(<<ms::integer-size(64), 0::integer-size(64)>>)
+  end
+
+  @doc """
+  The point in time before which a quiet remote thread becomes evictable.
+  """
+  @spec deadline() :: NaiveDateTime.t()
+  def deadline do
+    days = Config.get([:instance, :remote_post_retention_days])
+    NaiveDateTime.add(NaiveDateTime.utc_now(), -days * 86_400)
+  end
+
+  # The watermark deadline is never further back than the normal one.
+  defp watermark_deadline do
+    min_age = NaiveDateTime.add(NaiveDateTime.utc_now(), -@watermark_min_age)
+    Enum.max([deadline(), min_age], NaiveDateTime)
+  end
+
+  @doc """
+  Activities grouped by thread context, restricted to threads that may be evicted.
+
+  The query has no `select`; callers pick what they need from the group (the
+  context alone for `IN (subquery)`, or aggregates of ids for deletion).
+
+  Options:
+
+    * `:keep_non_public` - also pin threads containing a non-public post
+    * `:contexts` - only look at these threads, using the `(type, context)`
+      index; without it the whole table is grouped
+    * `:skip_reported` - a `MapSet` of reported contexts already removed from
+      `:contexts` by the caller, so the subquery for them can be skipped
+  """
+  @spec evictable_contexts_query(NaiveDateTime.t(), keyword()) :: Ecto.Query.t()
+  def evictable_contexts_query(deadline, opts \\ []) do
+    contexts =
+      case {Keyword.get(opts, :contexts), Keyword.get(opts, :skip_reported)} do
+        {nil, _} -> nil
+        {contexts, nil} -> contexts
+        {contexts, reported} -> Enum.reject(contexts, &MapSet.member?(reported, &1))
+      end
+
+    Activity
+    |> join(:left, [a], b in Bookmark, on: a.id == b.activity_id)
+    |> join(:left, [a], n in Notification, on: a.id == n.activity_id)
+    |> where([a], not is_nil(fragment("? ->> 'context'", a.data)))
+    |> maybe_exclude_reported(is_nil(Keyword.get(opts, :skip_reported)))
+    |> maybe_only_contexts(contexts)
+    |> group_by([a], fragment("? ->> 'context'", a.data))
+    |> having([a], max(a.updated_at) < ^deadline)
+    |> having([a], not fragment("bool_or(?)", a.local))
+    |> having([_, b], fragment("max(?::text) is null", b.id))
+    |> having([_, _, n], fragment("max(?) is null", n.id))
+    |> having(
+      [a],
+      not fragment(
+        "bool_or(EXISTS (SELECT 1 FROM unnest(?) AS r WHERE left(r, ?) = ?))",
+        a.recipients,
+        ^local_prefix_length(),
+        ^local_prefix()
+      )
+    )
+    # A local quote has a context of its own, so look up quotes of the posts
+    # here; the objects_quote_url index is on the jsonb value.
+    |> having(
+      [a],
+      not fragment(
+        "bool_or(? ->> 'type' = 'Create' AND EXISTS (SELECT 1 FROM objects q WHERE q.data -> 'quoteUrl' = to_jsonb(associated_object_id(?)) AND left(q.data ->> 'actor', ?) = ?))",
+        a.data,
+        a.data,
+        ^local_prefix_length(),
+        ^local_prefix()
+      )
+    )
+    |> maybe_keep_non_public(Keyword.get(opts, :keep_non_public, false) == true)
+  end
+
+  defp maybe_exclude_reported(query, false), do: query
+
+  defp maybe_exclude_reported(query, true) do
+    where(
+      query,
+      [a],
+      fragment("? ->> 'context'", a.data) not in subquery(reported_contexts_query())
+    )
+  end
+
+  @doc "Contexts of reported objects and of the reports themselves."
+  @spec reported_contexts() :: MapSet.t(String.t())
+  def reported_contexts do
+    reported_contexts_query() |> Repo.all(timeout: :infinity) |> MapSet.new()
+  end
+
+  defp maybe_only_contexts(query, nil), do: query
+
+  defp maybe_only_contexts(query, contexts) do
+    query
+    |> where([a], fragment("? ->> 'type'", a.data) in ^@context_types)
+    |> where([a], fragment("? ->> 'context'", a.data) in ^contexts)
+  end
+
+  defp maybe_keep_non_public(query, false), do: query
+
+  defp maybe_keep_non_public(query, true) do
+    having(
+      query,
+      [a],
+      not fragment(
+        # Posts (checked on Create Activity) is non-public
+        "bool_or((not(?->'to' \\? ? OR ?->'cc' \\? ?)) and ? ->> 'type' = 'Create')",
+        a.data,
+        ^Pleroma.Constants.as_public(),
+        a.data,
+        ^Pleroma.Constants.as_public(),
+        a.data
+      )
+    )
+  end
+
+  # Contexts touched by reports: the contexts of the reported objects, plus the
+  # Flag activities' own contexts (a report gets a fresh context of its own).
+  # Flag objects are a list mixing actor ids, object ids and embedded copies
+  # of objects with an "id"; a single object instead of a list is tolerated.
+  defp reported_contexts_query do
+    flags = where(Activity, [f], fragment("? ->> 'type' = 'Flag'", f.data))
+
+    reported_objects =
+      flags
+      |> join(
+        :inner,
+        [f],
+        fo in fragment(
+          "jsonb_array_elements(CASE WHEN jsonb_typeof(? -> 'object') = 'array' THEN ? -> 'object' ELSE jsonb_build_array(? -> 'object') END)",
+          f.data,
+          f.data,
+          f.data
+        ),
+        on: true
+      )
+      |> join(:inner, [_, fo], o in Object,
+        on: fragment("? ->> 'id' = coalesce(? ->> 'id', ? #>> '{}')", o.data, fo.value, fo.value)
+      )
+      |> where([_, _, o], not is_nil(fragment("? ->> 'context'", o.data)))
+      |> select([_, _, o], fragment("? ->> 'context'", o.data))
+
+    flags
+    |> where([f], not is_nil(fragment("? ->> 'context'", f.data)))
+    |> select([f], fragment("? ->> 'context'", f.data))
+    |> union(^reported_objects)
+  end
+
+  @doc """
+  Deletes the given threads. Each entry carries the context, activity ids and
+  object ids collected per thread, so deletion uses primary key and ap_id
+  indexes only.
+
+  An object is only deleted if it belongs to one of the threads itself: a
+  Like or Announce may carry a context other than its object's, and that
+  object's own thread can be pinned.
+  """
+  @spec evict([
+          %{context: String.t(), activity_ids: [String.t()], object_ids: [String.t() | nil]}
+        ]) :: stats()
+  def evict([]), do: @empty_stats
+
+  def evict(threads) do
+    activity_ids = Enum.flat_map(threads, & &1.activity_ids)
+    object_ids = threads |> Enum.flat_map(& &1.object_ids) |> Enum.reject(&is_nil/1)
+    contexts = Enum.map(threads, & &1.context)
+
+    {:ok, stats} =
+      Repo.transaction(
+        fn ->
+          # hashtags_objects rows go with the objects, so collect them first
+          hashtag_ids =
+            from(ho in "hashtags_objects",
+              join: o in Object,
+              on: o.id == ho.object_id,
+              where: fragment("? ->> 'id'", o.data) in ^object_ids,
+              where: fragment("? ->> 'context'", o.data) in ^contexts,
+              distinct: true,
+              select: ho.hashtag_id
+            )
+            |> Repo.all(timeout: :infinity)
+
+          {objects, deleted_ap_ids} =
+            Object
+            |> where([o], fragment("? ->> 'id'", o.data) in ^object_ids)
+            |> where([o], fragment("? ->> 'context'", o.data) in ^contexts)
+            |> where(
+              [o],
+              fragment(
+                "left(? ->> 'actor', ?) != ?",
+                o.data,
+                ^local_prefix_length(),
+                ^local_prefix()
+              )
+            )
+            |> select([o], fragment("? ->> 'id'", o.data))
+            |> Repo.delete_all(timeout: :infinity)
+
+          {activities, _} =
+            Activity
+            |> where([a], a.id in ^activity_ids)
+            |> where([a], a.local == false)
+            |> where([a], fragment("? ->> 'type' != 'Flag'", a.data))
+            |> Repo.delete_all(timeout: :infinity)
+
+          # Remote activities outside the thread that point at objects we just
+          # removed (e.g. Deletes carry no context). Flags are kept as they
+          # may have report notes attached.
+          {referencing, _} =
+            Activity
+            |> where([a], a.local == false)
+            |> where([a], fragment("associated_object_id(?)", a.data) in ^deleted_ap_ids)
+            |> where([a], fragment("? ->> 'type' != 'Flag'", a.data))
+            |> Repo.delete_all(timeout: :infinity)
+
+          delete_unused_hashtags(hashtag_ids)
+
+          %{
+            contexts: length(threads),
+            objects: objects,
+            activities: activities + referencing
+          }
+        end,
+        timeout: :infinity
+      )
+
+    stats
+  end
+
+  @doc """
+  Whether the objects table is estimated to hold more rows than `max_objects`.
+  """
+  @spec over_watermark?(non_neg_integer() | nil) :: boolean()
+  def over_watermark?(nil), do: false
+  def over_watermark?(max_objects), do: estimated_object_count() > max_objects
+
+  @doc """
+  Row count of the objects table from planner statistics, falling back to an
+  exact count when the table has never been analyzed.
+  """
+  @spec estimated_object_count() :: non_neg_integer()
+  def estimated_object_count do
+    %{rows: [[estimate]]} =
+      Repo.query!("SELECT reltuples::bigint FROM pg_class WHERE oid = 'objects'::regclass")
+
+    if estimate > 0, do: estimate, else: Repo.aggregate(Object, :count, timeout: :infinity)
+  end
+
+  @unused_hashtags """
+  DELETE FROM hashtags AS ht
+  WHERE NOT EXISTS (
+    SELECT 1 FROM hashtags_objects hto
+    WHERE ht.id = hto.hashtag_id
+  )
+  AND NOT EXISTS (
+    SELECT 1 FROM user_follows_hashtag ufh
+    WHERE ht.id = ufh.hashtag_id
+  )
+  """
+
+  @doc """
+  Removes hashtags no longer attached to any object and not followed by anyone,
+  either all of them or only those among the given ids.
+  """
+  @spec delete_unused_hashtags([integer()] | nil) :: :ok
+  def delete_unused_hashtags(ids \\ nil)
+
+  def delete_unused_hashtags([]), do: :ok
+
+  def delete_unused_hashtags(nil) do
+    Repo.query!(@unused_hashtags, [], timeout: :infinity)
+    :ok
+  end
+
+  def delete_unused_hashtags(ids) do
+    Repo.query!(@unused_hashtags <> "AND ht.id = ANY($1)", [ids], timeout: :infinity)
+    :ok
+  end
+
+  # Actor ids of local users start with the instance url; an exact prefix
+  # comparison avoids LIKE wildcards and host/port ambiguity.
+  defp local_prefix, do: Pleroma.Web.Endpoint.url() <> "/"
+  defp local_prefix_length, do: String.length(local_prefix())
+end

lib/pleroma/retention/cursor.ex

@@ -0,0 +1,50 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Retention.Cursor do
+  @moduledoc """
+  Where a retention walk over the activities table got to, persisted across
+  runs and restarts. Positions are activity (flake) ids, which are ordered by
+  creation time.
+  """
+
+  use Ecto.Schema
+
+  import Ecto.Query
+
+  alias Pleroma.Repo
+
+  @primary_key {:name, :string, autogenerate: false}
+
+  schema "retention_cursors" do
+    field(:position, FlakeId.Ecto.CompatType)
+
+    timestamps()
+  end
+
+  @spec get(String.t()) :: String.t() | nil
+  def get(name) do
+    Repo.one(from(c in __MODULE__, where: c.name == ^name, select: c.position))
+  end
+
+  @spec put(String.t(), String.t()) :: :ok
+  def put(name, position) do
+    now = NaiveDateTime.truncate(NaiveDateTime.utc_now(), :second)
+
+    %__MODULE__{name: name, position: position}
+    |> Repo.insert!(
+      on_conflict: [set: [position: position, updated_at: now]],
+      conflict_target: :name
+    )
+
+    :ok
+  end
+
+  @doc "Forgets the position, so the next run starts from the oldest activity."
+  @spec reset(String.t()) :: :ok
+  def reset(name) do
+    Repo.delete_all(from(c in __MODULE__, where: c.name == ^name))
+    :ok
+  end
+end

lib/pleroma/search/database_search.ex

@@ -7,6 +7,7 @@ defmodule Pleroma.Search.DatabaseSearch do
   alias Pleroma.Config
   alias Pleroma.Object.Fetcher
   alias Pleroma.Pagination
+  alias Pleroma.Repo
   alias Pleroma.User
   alias Pleroma.Web.ActivityPub.Visibility
 
@@ -23,22 +24,49 @@ defmodule Pleroma.Search.DatabaseSearch do
     offset = Keyword.get(options, :offset, 0)
     author = Keyword.get(options, :author)
 
-    try do
-      Activity
-      |> Activity.with_preloaded_object()
-      |> Activity.restrict_deactivated_users()
-      |> restrict_public(user)
-      |> query_with(index_type, search_query)
-      |> maybe_restrict_local(user)
-      |> maybe_restrict_author(author)
-      |> maybe_restrict_blocked(user)
-      |> Pagination.fetch_paginated(
-        %{"offset" => offset, "limit" => limit, "skip_order" => index_type == :rum},
-        :offset
-      )
-      |> maybe_fetch(user, search_query)
-    rescue
-      _ -> maybe_fetch([], user, search_query)
+    activities =
+      try do
+        with_fuzzy_limit(fn ->
+          Activity
+          |> Activity.with_preloaded_object()
+          |> Activity.restrict_deactivated_users()
+          |> restrict_public(user)
+          |> query_with(index_type, search_query)
+          |> maybe_restrict_local(user)
+          |> maybe_restrict_author(author)
+          |> maybe_restrict_blocked(user)
+          |> Pagination.fetch_paginated(
+            %{"offset" => offset, "limit" => limit, "skip_order" => index_type == :rum},
+            :offset
+          )
+        end)
+      rescue
+        _ -> []
+      end
+
+    # Outside the query transaction, as it may fetch over the network.
+    maybe_fetch(activities, user, search_query, options)
+  end
+
+  @doc """
+  Runs `fun` with `gin_fuzzy_search_limit` set, if configured under
+  `Pleroma.Search.DatabaseSearch`. The limit makes GIN scans faster but their
+  results incomplete and random, so it is only set for search queries.
+  """
+  def with_fuzzy_limit(fun) do
+    case Config.get([__MODULE__, :gin_fuzzy_search_limit]) do
+      limit when is_integer(limit) and limit > 0 ->
+        {:ok, result} =
+          Repo.transaction(fn ->
+            # SET LOCAL takes no parameters; limit is an integer.
+            Repo.query!("SET LOCAL gin_fuzzy_search_limit TO #{limit}")
+            fun.()
+          end)
+
+        result
+
+      _ ->
+        fun.()
     end
   end
 
@@ -132,7 +160,27 @@ defmodule Pleroma.Search.DatabaseSearch do
 
   defp restrict_local(q), do: where(q, local: true)
 
-  def maybe_fetch(activities, user, search_query) do
+  @doc """
+  Adds the status a URL query points to, fetching it if needed. As in Mastodon,
+  this only happens with `resolve: true` and on the first page, and for
+  anonymous users only if `[:instance, :limit_to_local_content]` is `false`.
+  """
+  def maybe_fetch(activities, user, search_query, options) do
+    if fetch_allowed?(user, options),
+      do: fetch(activities, user, search_query),
+      else: activities
+  end
+
+  defp fetch_allowed?(user, options) do
+    options[:resolve] == true and (options[:offset] || 0) == 0 and
+      case Config.get([:instance, :limit_to_local_content], :unauthenticated) do
+        :all -> false
+        :unauthenticated -> match?(%User{}, user)
+        false -> true
+      end
+  end
+
+  defp fetch(activities, user, search_query) do
     with true <- Regex.match?(~r/https?:/, search_query),
          {:ok, object} <- Fetcher.fetch_object_from_id(search_query),
          %Activity{} = activity <- Activity.get_create_by_object_ap_id(object.data["id"]),

lib/pleroma/search/meilisearch.ex

@@ -111,11 +111,11 @@ defmodule Pleroma.Search.Meilisearch do
         |> maybe_restrict_local(user)
         |> maybe_restrict_author(author)
         |> maybe_restrict_blocked(user)
-        |> maybe_fetch(user, query)
+        |> maybe_fetch(user, query, options)
         |> order_by([object: obj], desc: obj.data["published"])
         |> Pleroma.Repo.all()
       rescue
-        _ -> maybe_fetch([], user, query)
+        _ -> maybe_fetch([], user, query, options)
       end
     end
   end

lib/pleroma/search/parade_db.ex

@@ -12,7 +12,7 @@ defmodule Pleroma.Search.ParadeDB do
 
   import Pleroma.Search.DatabaseSearch,
     only: [
-      maybe_fetch: 3,
+      maybe_fetch: 4,
       maybe_restrict_author: 2,
       maybe_restrict_blocked: 2,
       maybe_restrict_local: 2
@@ -149,19 +149,19 @@ defmodule Pleroma.Search.ParadeDB do
 
     case search_ids(query, limit, offset, author) do
       {:ok, []} ->
-        maybe_fetch([], user, query)
+        maybe_fetch([], user, query, options)
 
       {:ok, ids} ->
         ids
-        |> fetch_activities(user, query, author)
+        |> fetch_activities(user, query, author, options)
 
       {:error, error} ->
         Logger.error("ParadeDB search failed: #{inspect(error)}")
-        maybe_fetch([], user, query)
+        maybe_fetch([], user, query, options)
     end
   end
 
-  defp fetch_activities(ids, user, query, author) do
+  defp fetch_activities(ids, user, query, author, options) do
     dumped_ids =
       ids
       |> Enum.map(&dump_activity_id/1)
@@ -183,7 +183,7 @@ defmodule Pleroma.Search.ParadeDB do
       )
     )
     |> Pleroma.Repo.all()
-    |> maybe_fetch(user, query)
+    |> maybe_fetch(user, query, options)
   end
 
   defp maybe_restrict_public(query, %Pleroma.User{}) do

lib/pleroma/search/qdrant_search.ex

@@ -10,7 +10,7 @@ defmodule Pleroma.Search.QdrantSearch do
   alias __MODULE__.OpenAIClient
   alias __MODULE__.QdrantClient
 
-  import Pleroma.Search.DatabaseSearch, only: [maybe_fetch: 3]
+  import Pleroma.Search.DatabaseSearch, only: [maybe_fetch: 4]
 
   @impl true
   def create_index do
@@ -131,7 +131,7 @@ defmodule Pleroma.Search.QdrantSearch do
       |> Activity.restrict_deactivated_users()
       |> Ecto.Query.order_by([a], fragment("array_position(?, ?)", ^ids, a.id))
       |> Pleroma.Repo.all()
-      |> maybe_fetch(user, original_query)
+      |> maybe_fetch(user, original_query, options)
     else
       _ ->
         []

lib/pleroma/signature.ex

@@ -95,6 +95,84 @@ defmodule Pleroma.Signature do
     Timex.format!(date, "{WDshort}, {0D} {Mshort} {YYYY} {h24}:{m}:{s} GMT")
   end
 
+  @doc """
+  Checks the parts of an incoming signature that the cryptographic check does
+  not cover, so that a captured request cannot be replayed later or with a
+  different body.
+
+  The signature must cover `date` or `(created)`, and the signed time must be
+  within the configured window. Requests other than GET and HEAD must also
+  sign `digest`, and DigestPlug must have hashed their body. Based on Akkoma's http_signatures fork and Mastodon's
+  SignedRequest.
+  """
+  @spec check_signature_policy(Plug.Conn.t(), DateTime.t()) :: :ok | {:error, atom()}
+  def check_signature_policy(%Plug.Conn{} = conn, now \\ DateTime.utc_now()) do
+    signature = HTTPSignatures.signature_for_conn(conn)
+    signed = Map.get(signature, "headers", [])
+
+    cond do
+      "date" not in signed and "(created)" not in signed ->
+        {:error, :date_not_signed}
+
+      conn.method not in ["GET", "HEAD"] and "digest" not in signed ->
+        {:error, :digest_not_signed}
+
+      # The plug puts the digest computed by DigestPlug into the header. With
+      # no computed digest (e.g. multipart), nothing checked the body.
+      conn.method not in ["GET", "HEAD"] and is_nil(conn.assigns[:digest]) ->
+        {:error, :body_not_hashed}
+
+      true ->
+        with :ok <- check_signed_date(conn, signed, now) do
+          check_signed_created(signature, signed, now)
+        end
+    end
+  end
+
+  defp check_signed_date(conn, signed, now) do
+    if "date" in signed do
+      # Exactly one header: a second one could make this check and the
+      # signature check see different dates.
+      with [date] <- Plug.Conn.get_req_header(conn, "date"),
+           {:ok, %DateTime{} = date} <- Timex.parse(date, "{RFC1123}") do
+        check_time_window(date, now)
+      else
+        _ -> {:error, :invalid_date}
+      end
+    else
+      :ok
+    end
+  end
+
+  defp check_signed_created(signature, signed, now) do
+    if "(created)" in signed do
+      with created when is_binary(created) <- signature["created"],
+           {created, ""} <- Integer.parse(created),
+           {:ok, created} <- DateTime.from_unix(created) do
+        check_time_window(created, now)
+      else
+        _ -> {:error, :invalid_date}
+      end
+    else
+      :ok
+    end
+  end
+
+  defp check_time_window(signed_at, now) do
+    age = DateTime.diff(now, signed_at)
+
+    cond do
+      age > Pleroma.Config.get([:activitypub, :signature_max_age], 7200) ->
+        {:error, :date_too_old}
+
+      -age > Pleroma.Config.get([:activitypub, :signature_max_clock_skew], 3600) ->
+        {:error, :date_in_future}
+
+      true ->
+        :ok
+    end
+  end
+
   @spec validate_signature(Plug.Conn.t(), String.t()) :: boolean()
   def validate_signature(%Plug.Conn{} = conn, request_target) do
     # Newer drafts for HTTP signatures now use @request-target instead of the

lib/pleroma/stats.ex

@@ -8,10 +8,11 @@ defmodule Pleroma.Stats do
   import Ecto.Query
 
   alias Pleroma.CounterCache
+  alias Pleroma.Instances.Instance
   alias Pleroma.Repo
   alias Pleroma.User
 
-  @interval :timer.seconds(60)
+  @interval :timer.seconds(300)
 
   def start_link(_) do
     GenServer.start_link(
@@ -64,14 +65,12 @@ defmodule Pleroma.Stats do
           }
         }
   def calculate_stat_data do
+    # Hosts are unique in instances, and remote users add theirs. Scanning
+    # all remote users for their domains was one of the costliest queries.
+    # Favicon lookups for local accounts can add our own host, so skip it.
     peers =
-      from(
-        u in User,
-        select: fragment("distinct split_part(?, '@', 2)", u.nickname),
-        where: u.local != ^true
-      )
+      from(i in Instance, where: i.host != ^local_host(), select: i.host)
       |> Repo.all()
-      |> Enum.filter(& &1)
 
     domain_count = Enum.count(peers)
 
@@ -97,6 +96,9 @@ defmodule Pleroma.Stats do
     }
   end
 
+  # Read from the config: this also runs before the endpoint has started.
+  defp local_host, do: Pleroma.Config.get([Pleroma.Web.Endpoint, :url, :host])
+
   @spec get_status_visibility_count(String.t() | nil) :: map()
   def get_status_visibility_count(instance \\ nil) do
     if is_nil(instance) do

lib/pleroma/upload/filter/exif/read_description.ex

@@ -0,0 +1,32 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2026 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Upload.Filter.Exif.ReadDescription do
+  @moduledoc """
+  Reads a description from image metadata when the upload has none yet.
+
+  Checks EXIF ImageDescription, then IPTC Caption-Abstract (and XMP for GIF). Blank
+  values and values longer than `[:instance, :description_limit]` are ignored.
+  """
+
+  @behaviour Pleroma.Upload.Filter
+
+  @impl true
+  def filter(%Pleroma.Upload{description: description}) when is_binary(description),
+    do: {:ok, :noop}
+
+  def filter(%Pleroma.Upload{tempfile: file, content_type: "image" <> _} = upload) do
+    max_length = Pleroma.Config.get([:instance, :description_limit])
+
+    case ExifStripper.Description.read_file(file, max_length: max_length) do
+      description when is_binary(description) ->
+        {:ok, :filtered, %{upload | description: description}}
+
+      nil ->
+        {:ok, :noop}
+    end
+  end
+
+  def filter(_), do: {:ok, :noop}
+end

lib/pleroma/upload/filter/exif/strip_location.ex

@@ -0,0 +1,25 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2026 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Upload.Filter.Exif.StripLocation do
+  @moduledoc """
+  Strips GPS metadata from uploaded images and overwrites the file in place.
+
+  Supports JPEG, PNG, WebP, TIFF and ISO-BMFF images (HEIC, AVIF). This is lossless:
+  the image data is not re-encoded. Other image types are left untouched.
+  """
+
+  @behaviour Pleroma.Upload.Filter
+
+  @impl true
+  def filter(%Pleroma.Upload{tempfile: file, content_type: "image" <> _}) do
+    case ExifStripper.GpsStripper.strip_gps_data(file, file) do
+      :ok -> {:ok, :filtered}
+      :noop -> {:ok, :noop}
+      {:error, _} = error -> error
+    end
+  end
+
+  def filter(_), do: {:ok, :noop}
+end

lib/pleroma/upload/filter/exiftool.ex

@@ -0,0 +1,14 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2026 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Upload.Filter.Exiftool do
+  @moduledoc """
+  Deprecated: use `Pleroma.Upload.Filter.Exif.StripLocation`. This module delegates to it.
+  """
+
+  @behaviour Pleroma.Upload.Filter
+
+  @impl true
+  defdelegate filter(upload), to: Pleroma.Upload.Filter.Exif.StripLocation
+end

lib/pleroma/upload/filter/exiftool/read_description.ex

@@ -4,51 +4,11 @@
 
 defmodule Pleroma.Upload.Filter.Exiftool.ReadDescription do
   @moduledoc """
-  Gets a valid description from the related EXIF tags and provides them in the response if no description is provided yet.
-  It will first check ImageDescription, when that doesn't probide a valid description, it will check iptc:Caption-Abstract.
-  A valid description means the fields are filled in and not too long (see `:instance, :description_limit`).
+  Deprecated: use `Pleroma.Upload.Filter.Exif.ReadDescription`. This module delegates to it.
   """
+
   @behaviour Pleroma.Upload.Filter
 
-  def filter(%Pleroma.Upload{description: description})
-      when is_binary(description),
-      do: {:ok, :noop}
-
-  def filter(%Pleroma.Upload{tempfile: file} = upload),
-    do: {:ok, :filtered, upload |> Map.put(:description, read_description_from_exif_data(file))}
-
-  def filter(_, _), do: {:ok, :noop}
-
-  defp read_description_from_exif_data(file) do
-    nil
-    |> read_when_empty(file, "-ImageDescription")
-    |> read_when_empty(file, "-iptc:Caption-Abstract")
-  end
-
-  defp read_when_empty(current_description, _, _) when is_binary(current_description),
-    do: current_description
-
-  defp read_when_empty(_, file, tag) do
-    if File.exists?(file) do
-      try do
-        {tag_content, 0} =
-          System.cmd("exiftool", ["-m", "-b", "-s3", tag, file],
-            stderr_to_stdout: false,
-            parallelism: true
-          )
-
-        tag_content = String.trim(tag_content)
-
-        if tag_content != "" and
-             String.length(tag_content) <=
-               Pleroma.Config.get([:instance, :description_limit]),
-           do: tag_content,
-           else: nil
-      rescue
-        _ in ErlangError -> nil
-      end
-    else
-      nil
-    end
-  end
+  @impl true
+  defdelegate filter(upload), to: Pleroma.Upload.Filter.Exif.ReadDescription
 end

lib/pleroma/upload/filter/exiftool/strip_location.ex

@@ -4,30 +4,11 @@
 
 defmodule Pleroma.Upload.Filter.Exiftool.StripLocation do
   @moduledoc """
-  Strips GPS related EXIF tags and overwrites the file in place.
-  Also strips or replaces filesystem metadata e.g., timestamps.
+  Deprecated: use `Pleroma.Upload.Filter.Exif.StripLocation`. This module delegates to it.
   """
+
   @behaviour Pleroma.Upload.Filter
 
-  # Formats not compatible with exiftool at this time
-  def filter(%Pleroma.Upload{content_type: "image/heic"}), do: {:ok, :noop}
-  def filter(%Pleroma.Upload{content_type: "image/webp"}), do: {:ok, :noop}
-  def filter(%Pleroma.Upload{content_type: "image/svg" <> _}), do: {:ok, :noop}
-
-  def filter(%Pleroma.Upload{tempfile: file, content_type: "image" <> _}) do
-    try do
-      case System.cmd("exiftool", ["-m", "-overwrite_original", "-gps:all=", "-png:all=", file],
-             stderr_to_stdout: true,
-             parallelism: true
-           ) do
-        {_response, 0} -> {:ok, :filtered}
-        {error, _} -> {:error, error}
-      end
-    rescue
-      e in ErlangError ->
-        {:error, "#{__MODULE__}: #{inspect(e)}"}
-    end
-  end
-
-  def filter(_), do: {:ok, :noop}
+  @impl true
+  defdelegate filter(upload), to: Pleroma.Upload.Filter.Exif.StripLocation
 end

lib/pleroma/web/activity_pub/activity_pub.ex

@@ -11,6 +11,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
   alias Pleroma.Conversation.Participation
   alias Pleroma.Filter
   alias Pleroma.Hashtag
+  alias Pleroma.Instances.Instance
   alias Pleroma.Maps
   alias Pleroma.Notification
   alias Pleroma.Object
@@ -465,21 +466,28 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
   end
 
   def fetch_activities_for_context_query(context, opts) do
-    public = [Constants.as_public()]
+    user = opts[:user]
 
     recipients =
-      if opts[:user],
-        do: [opts[:user].ap_id | User.following(opts[:user])] ++ public,
-        else: public
+      cond do
+        user && user.local ->
+          [Constants.as_public(), as_local_public(), user.ap_id | User.following(user)]
+
+        user ->
+          [Constants.as_public(), user.ap_id | User.following(user)]
+
+        true ->
+          [Constants.as_public()]
+      end
 
     from(activity in Activity)
     |> maybe_preload_objects(opts)
     |> maybe_preload_bookmarks(opts)
     |> maybe_set_thread_muted_field(opts)
-    |> restrict_unauthenticated(opts[:user])
+    |> restrict_unauthenticated(user)
     |> restrict_blocked(opts)
     |> restrict_blockers_visibility(opts)
-    |> restrict_recipients(recipients, opts[:user])
+    |> restrict_recipients(recipients, user)
     |> restrict_filtered(opts)
     |> where(
       [activity],
@@ -1948,6 +1956,10 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
         Logger.info("Rejected user #{ap_id}: #{inspect(reason)}")
         {:error, e}
 
+      {:error, :ratelimit = e} ->
+        Logger.debug("Backing off from fetching user #{ap_id}")
+        {:error, e}
+
       {:error, e} ->
         Logger.error("Could not decode user at fetch #{ap_id}, #{inspect(e)}")
         {:error, e}
@@ -2057,6 +2069,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
   def make_user_from_ap_id(ap_id, additional \\ []) do
     with {:ok, data} <- fetch_and_prepare_user_from_ap_id(ap_id, additional) do
       enqueue_pin_fetches(data)
+      # Keeps the instances table complete, so it can list our peers.
+      Instance.ensure_exists(data.ap_id)
 
       # the fetched actor's id differs from ap_id when it was fetched from a display URL
       user = User.get_cached_by_ap_id(data.ap_id)

lib/pleroma/web/activity_pub/activity_pub_controller.ex

@@ -26,6 +26,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
   alias Pleroma.Web.Plugs.EnsureAuthenticatedPlug
   alias Pleroma.Web.Plugs.FederatingPlug
   alias Pleroma.Web.Plugs.OAuthScopesPlug
+  alias Pleroma.Web.Plugs.RateLimiter
 
   require Logger
 
@@ -66,6 +67,9 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
     when action in [:followers, :following]
   )
 
+  # Shares the per-user bucket with Mastodon API status actions and emoji reactions.
+  plug(RateLimiter, [name: :statuses_actions] when action == :update_outbox)
+
   plug(Majic.Plug, [pool: Pleroma.MajicPool] when action in [:upload_media])
 
   plug(
@@ -341,7 +345,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
            {:sender_exists, User.get_or_fetch_by_ap_id(params["actor"])},
          {:recipient_active, true} <- {:recipient_active, recipient.is_active},
          {:sender_active, true} <- {:sender_active, actor.is_active},
-         true <- Utils.recipient_in_message(recipient, actor, params),
          params <- Utils.maybe_splice_recipient(recipient.ap_id, params) do
       Federator.incoming_ap_doc(params)
       json(conn, "ok")
@@ -386,18 +389,25 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
   end
 
   def inbox(%{assigns: %{valid_signature: false}} = conn, params) do
-    if unsupported_http_message_signature?(conn) do
-      reject_unsupported_http_message_signature(conn)
-    else
-      Federator.incoming_failed_signature_ap_doc(%{
-        method: conn.method,
-        req_headers: conn.req_headers,
-        request_path: conn.request_path,
-        params: params,
-        query_string: conn.query_string
-      })
+    cond do
+      unsupported_http_message_signature?(conn) ->
+        reject_unsupported_http_message_signature(conn)
 
-      json(conn, "ok")
+      reason = conn.assigns[:signature_error] ->
+        conn
+        |> put_status(:unauthorized)
+        |> json("error, invalid HTTP signature (#{reason})")
+
+      true ->
+        Federator.incoming_failed_signature_ap_doc(%{
+          method: conn.method,
+          req_headers: conn.req_headers,
+          request_path: conn.request_path,
+          params: params,
+          query_string: conn.query_string
+        })
+
+        json(conn, "ok")
     end
   end
 

lib/pleroma/web/activity_pub/mrf.ex

@@ -148,14 +148,13 @@ defmodule Pleroma.Web.ActivityPub.MRF do
   @spec subdomains_regex([String.t()]) :: [Regex.t()]
   def subdomains_regex(domains) when is_list(domains) do
     for domain <- domains do
-      try do
-        target = String.replace(domain, "*.", "(.*\\.)*")
-        ~r<^#{target}$>i
-      rescue
-        e ->
-          Logger.error("MRF: Invalid subdomain Regex: #{domain}")
-          reraise e, __STACKTRACE__
-      end
+      {prefix, rest} =
+        case domain do
+          "*." <> rest -> {"(.*\\.)?", rest}
+          _ -> {"", domain}
+        end
+
+      ~r<^#{prefix}#{Regex.escape(rest)}$>i
     end
   end
 

lib/pleroma/web/activity_pub/object_validator.ex

@@ -78,10 +78,16 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do
            |> Ecto.Changeset.apply_action(:insert) do
       object = stringify_keys(object)
       undone_object = Activity.get_by_ap_id(object["object"])
+      outgoing_blocks = Pleroma.Config.get([:activitypub, :outgoing_blocks])
+      # if we're undoing a block, and do not permit federating that:
+      do_not_federate =
+        Keyword.get(meta, :do_not_federate) ||
+          (Map.get(undone_object.data, "type") == "Block" && !outgoing_blocks)
 
       meta =
         meta
         |> Keyword.put(:object_data, undone_object.data)
+        |> Keyword.put(:do_not_federate, do_not_federate)
 
       {:ok, object, meta}
     end

lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex

@@ -213,7 +213,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNotePageValidator do
 
   defp fix_misskey_content(object), do: object
 
-  defp fix(data) do
+  def fix(data) do
     data
     |> CommonFixes.fix_actor()
     |> CommonFixes.fix_object_defaults()

lib/pleroma/web/activity_pub/object_validators/emoji_react_validator.ex

@@ -57,7 +57,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.EmojiReactValidator do
       |> CommonFixes.fix_actor()
       |> CommonFixes.fix_activity_addressing()
 
-    data = Map.put_new(data, "tag", [])
+    data = fix_tag(data)
 
     case Object.normalize(data["object"]) do
       %Object{} = object ->
@@ -70,6 +70,10 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.EmojiReactValidator do
     end
   end
 
+  # JSON-LD compaction turns a single-element array into a bare object
+  defp fix_tag(%{"tag" => %{} = tag} = data), do: Map.put(data, "tag", [tag])
+  defp fix_tag(data), do: Map.put_new(data, "tag", [])
+
   defp fix_emoji_qualification(%{"content" => emoji} = data) do
     new_emoji = Pleroma.Emoji.fully_qualify_emoji(emoji)
 

lib/pleroma/web/activity_pub/object_validators/question_validator.ex

@@ -6,10 +6,9 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.QuestionValidator do
   use Ecto.Schema
 
   alias Pleroma.EctoType.ActivityPub.ObjectValidators
-  alias Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes
+  alias Pleroma.Web.ActivityPub.ObjectValidators.ArticleNotePageValidator
   alias Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations
   alias Pleroma.Web.ActivityPub.ObjectValidators.QuestionOptionsValidator
-  alias Pleroma.Web.ActivityPub.Transmogrifier
 
   import Ecto.Changeset
 
@@ -62,11 +61,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.QuestionValidator do
 
   defp fix(data) do
     data
-    |> CommonFixes.fix_actor()
-    |> CommonFixes.fix_object_defaults()
-    |> CommonFixes.fix_quote_url()
-    |> CommonFixes.fix_likes()
-    |> Transmogrifier.fix_emoji()
+    |> ArticleNotePageValidator.fix()
     |> fix_closed()
   end
 

lib/pleroma/web/activity_pub/publisher.ex

@@ -281,36 +281,18 @@ defmodule Pleroma.Web.ActivityPub.Publisher do
   defp maybe_use_sharedinbox(%User{shared_inbox: shared_inbox}), do: shared_inbox
 
   @doc """
-  Determine a user inbox to use based on heuristics.  These heuristics
-  are based on an approximation of the ``sharedInbox`` rules in the
-  [ActivityPub specification][ap-sharedinbox].
+  Determines the inbox to deliver an activity to.
 
-  Please do not edit this function (or its children) without reading
-  the spec, as editing the code is likely to introduce some breakage
-  without some familiarity.
-
-     [ap-sharedinbox]: https://www.w3.org/TR/activitypub/#shared-inbox-delivery
+  Like Mastodon, GoToSocial and Akkoma, this always prefers the shared inbox,
+  which saves duplicate deliveries to the same server. Only activities with
+  `bcc` or `bto` go to personal inboxes, because the receiver rebuilds that
+  addressing from the inbox the activity arrived at.
   """
-  def determine_inbox(
-        %Activity{data: activity_data},
-        %User{inbox: inbox} = user
-      ) do
-    to = activity_data["to"] || []
-    cc = activity_data["cc"] || []
-    type = activity_data["type"]
-
-    cond do
-      type == "Delete" ->
-        maybe_use_sharedinbox(user)
-
-      Pleroma.Constants.as_public() in to || Pleroma.Constants.as_public() in cc ->
-        maybe_use_sharedinbox(user)
-
-      length(to) + length(cc) > 1 ->
-        maybe_use_sharedinbox(user)
-
-      true ->
-        inbox
+  def determine_inbox(%Activity{data: activity_data}, %User{inbox: inbox} = user) do
+    if Enum.empty?(activity_data["bcc"] || []) and Enum.empty?(activity_data["bto"] || []) do
+      maybe_use_sharedinbox(user)
+    else
+      inbox
     end
   end
 

lib/pleroma/web/activity_pub/transmogrifier.ex

@@ -799,7 +799,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
        ) do
     with %User{} = origin_user <- User.get_cached_by_ap_id(origin_actor),
          {:ok, %User{} = target_user} <- User.get_or_fetch_by_ap_id(target_actor),
-         true <- origin_actor in target_user.also_known_as do
+         {:ok, %User{} = target_user} <- refetch_unless_aliased(target_user, origin_actor) do
       ActivityPub.move(origin_user, target_user, false)
     else
       _e -> :error
@@ -1204,4 +1204,20 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
   def maybe_fix_user_url(data), do: data
 
   def maybe_fix_user_object(data), do: maybe_fix_user_url(data)
+
+  # A Move usually follows right after the alias was added, which the cached
+  # remote target may not have yet.
+  defp refetch_unless_aliased(%User{} = target, origin_ap_id, refetch? \\ true) do
+    cond do
+      origin_ap_id in target.also_known_as ->
+        {:ok, target}
+
+      refetch? and not target.local ->
+        with {:ok, %User{} = target} <- User.fetch_by_ap_id(target.ap_id),
+             do: refetch_unless_aliased(target, origin_ap_id, false)
+
+      true ->
+        :error
+    end
+  end
 end

lib/pleroma/web/activity_pub/utils.ex

@@ -78,22 +78,6 @@ defmodule Pleroma.Web.ActivityPub.Utils do
       [params["to"], params["cc"], params["bto"], params["bcc"]]
       |> Enum.any?(&label_in_collection?(label, &1))
 
-  @spec unaddressed_message?(map()) :: boolean()
-  def unaddressed_message?(params),
-    do:
-      [params["to"], params["cc"], params["bto"], params["bcc"]]
-      |> Enum.all?(fn
-        nil -> true
-        [] -> true
-        _ -> false
-      end)
-
-  @spec recipient_in_message(User.t(), User.t(), map()) :: boolean()
-  def recipient_in_message(%User{ap_id: ap_id} = recipient, %User{} = actor, params),
-    do:
-      label_in_message?(ap_id, params) || unaddressed_message?(params) ||
-        User.following?(recipient, actor)
-
   defp extract_list(target) when is_binary(target), do: [target]
   defp extract_list(lst) when is_list(lst), do: lst
   defp extract_list(_), do: []

lib/pleroma/web/api_spec/cast_and_validate.ex

@@ -20,6 +20,9 @@ defmodule Pleroma.Web.ApiSpec.CastAndValidate do
 
   require Logger
 
+  # How often to drop offending query params and retry in permissive mode
+  @max_retries 20
+
   @impl Plug
   def init(opts) do
     opts
@@ -114,34 +117,49 @@ defmodule Pleroma.Web.ApiSpec.CastAndValidate do
          false = _strict,
          cast_opts
        ) do
-    case OpenApiSpex.cast_and_validate(spec, operation, conn, content_type) do
-      {:ok, conn} ->
-        {:ok, conn}
+    permissive_cast_and_validate(spec, operation, conn, content_type, cast_opts, @max_retries)
+  end
 
-      # Remove unexpected query params and cast/validate again
-      {:error, errors} ->
-        query_params =
-          Enum.reduce(errors, conn.query_params, fn
-            %{reason: :unexpected_field, name: name, path: [name]}, params ->
-              Map.delete(params, name)
+  # OpenApiSpex reports one offending param at a time, so drop it and cast/validate again
+  defp permissive_cast_and_validate(
+         spec,
+         operation,
+         %Conn{} = conn,
+         content_type,
+         cast_opts,
+         retries
+       ) do
+    with {:error, errors} = error <-
+           OpenApiSpex.cast_and_validate(spec, operation, conn, content_type, cast_opts) do
+      query_params = drop_invalid_query_params(errors, conn.query_params)
 
-            # Filter out empty params
-            %{reason: :invalid_type, path: [name_atom], value: ""}, params ->
-              Map.delete(params, to_string(name_atom))
-
-            %{reason: :invalid_enum, name: nil, path: path, value: value}, params ->
-              path = path |> Enum.reverse() |> tl() |> Enum.reverse() |> list_items_to_string()
-              update_in(params, path, &List.delete(&1, value))
-
-            _, params ->
-              params
-          end)
-
-        conn = %Conn{conn | query_params: query_params}
-        OpenApiSpex.cast_and_validate(spec, operation, conn, content_type, cast_opts)
+      if retries > 0 and query_params != conn.query_params do
+        conn = %{conn | query_params: query_params}
+        permissive_cast_and_validate(spec, operation, conn, content_type, cast_opts, retries - 1)
+      else
+        error
+      end
     end
   end
 
+  defp drop_invalid_query_params(errors, query_params) do
+    Enum.reduce(errors, query_params, fn
+      %{reason: :unexpected_field, name: name, path: [name]}, params ->
+        Map.delete(params, name)
+
+      # Filter out empty params
+      %{reason: :invalid_type, path: [name_atom], value: ""}, params ->
+        Map.delete(params, to_string(name_atom))
+
+      %{reason: :invalid_enum, name: nil, path: path, value: value}, params ->
+        path = path |> Enum.reverse() |> tl() |> Enum.reverse() |> list_items_to_string()
+        update_in(params, path, &List.delete(&1, value))
+
+      _, params ->
+        params
+    end)
+  end
+
   defp list_items_to_string(list) do
     Enum.map(list, fn
       i when is_atom(i) -> to_string(i)

lib/pleroma/web/mastodon_api/controllers/status_controller.ex

@@ -92,13 +92,13 @@ defmodule Pleroma.Web.MastodonAPI.StatusController do
 
   plug(
     RateLimiter,
-    [name: :status_id_action, bucket_name: "status_id_action:reblog_unreblog", params: [:id]]
+    [name: :status_id_action, bucket_name: "status_id_action:reblog_unreblog", params: ["id"]]
     when action in ~w(reblog unreblog)a
   )
 
   plug(
     RateLimiter,
-    [name: :status_id_action, bucket_name: "status_id_action:fav_unfav", params: [:id]]
+    [name: :status_id_action, bucket_name: "status_id_action:fav_unfav", params: ["id"]]
     when action in ~w(favourite unfavourite)a
   )
 

lib/pleroma/web/mastodon_api/views/marker_view.ex

@@ -5,13 +5,15 @@
 defmodule Pleroma.Web.MastodonAPI.MarkerView do
   use Pleroma.Web, :view
 
+  alias Pleroma.Web.CommonAPI.Utils
+
   def render("markers.json", %{markers: markers}) do
     Map.new(markers, fn m ->
       {m.timeline,
        %{
          last_read_id: m.last_read_id,
          version: m.lock_version,
-         updated_at: NaiveDateTime.to_iso8601(m.updated_at),
+         updated_at: Utils.to_masto_date(m.updated_at),
          pleroma: %{
            unread_count: m.unread_count
          }

lib/pleroma/web/media_proxy.ex

@@ -45,7 +45,7 @@ defmodule Pleroma.Web.MediaProxy do
   def url("/" <> _ = url), do: url
 
   def url(url) do
-    if enabled?() and url_proxiable?(url) do
+    if enabled?() and not data_url?(url) and url_proxiable?(url) do
       encode_url(url)
     else
       url
@@ -61,7 +61,7 @@ defmodule Pleroma.Web.MediaProxy do
   def preview_url("//" <> _ = url, pparams), do: preview_url("https:" <> url, pparams)
 
   def preview_url(url, preview_params) do
-    if preview_enabled?() do
+    if preview_enabled?() and not data_url?(url) do
       encode_preview_url(url, preview_params)
     else
       url(url)
@@ -74,6 +74,8 @@ defmodule Pleroma.Web.MediaProxy do
   #   non-local non-whitelisted URLs through it and be sure that body size constraint is preserved.
   def preview_enabled?, do: enabled?() and !!Config.get([:media_preview_proxy, :enabled])
 
+  defp data_url?(url), do: url |> String.slice(0, 5) |> String.downcase() == "data:"
+
   def local?(url), do: String.starts_with?(url, Endpoint.url())
 
   def whitelisted?(url) do

lib/pleroma/web/media_proxy/media_proxy_controller.ex

@@ -7,7 +7,6 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do
 
   alias Pleroma.Config
   alias Pleroma.Helpers.MediaHelper
-  alias Pleroma.Helpers.UriHelper
   alias Pleroma.ReverseProxy
   alias Pleroma.Web.MediaProxy
   alias Plug.Conn
@@ -67,9 +66,6 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do
         static and content_type == "image/gif" ->
           handle_jpeg_preview(conn, media_proxy_url)
 
-        static ->
-          drop_static_param_and_redirect(conn)
-
         content_type == "image/gif" ->
           conn
           |> put_status(301)
@@ -166,15 +162,6 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do
     end
   end
 
-  defp drop_static_param_and_redirect(conn) do
-    uri_without_static_param =
-      conn
-      |> current_url()
-      |> UriHelper.modify_uri_params(%{}, ["static"])
-
-    redirect(conn, external: uri_without_static_param)
-  end
-
   defp fallback_on_preview_error(conn, media_proxy_url) do
     redirect(conn, external: media_proxy_url)
   end

lib/pleroma/web/o_auth/token_controller.ex

@@ -26,11 +26,11 @@ defmodule Pleroma.Web.OAuth.TokenController do
     end
   end
 
-  def oauth_tokens(%{assigns: %{user: user}} = conn, _params) do
+  def oauth_tokens(%{assigns: %{user: user, token: current_token}} = conn, _params) do
     with oauth_tokens <- Token.get_user_tokens(user) do
       conn
       |> put_view(TokenView)
-      |> render("index.json", %{tokens: oauth_tokens})
+      |> render("index.json", %{tokens: oauth_tokens, current_token: current_token})
     end
   end
 

lib/pleroma/web/o_auth/token_view.ex

@@ -5,18 +5,21 @@
 defmodule Pleroma.Web.OAuth.TokenView do
   use Pleroma.Web, :view
 
-  def render("index.json", %{tokens: tokens}) do
+  def render("index.json", %{tokens: tokens, current_token: current_token}) do
     tokens
-    |> render_many(Pleroma.Web.OAuth.TokenView, "show.json")
+    |> render_many(Pleroma.Web.OAuth.TokenView, "show.json", %{
+      current_token: current_token
+    })
     |> Enum.filter(&Enum.any?/1)
   end
 
-  def render("show.json", %{token: token_entry}) do
+  def render("show.json", %{token: token_entry, current_token: current_token}) do
     %{
       id: token_entry.id,
       valid_until: token_entry.valid_until,
       app_name: token_entry.app.client_name,
-      scopes: token_entry.scopes
+      scopes: token_entry.scopes,
+      is_current_token: token_entry.id == current_token.id
     }
   end
 end

lib/pleroma/web/pleroma_api/controllers/emoji_reaction_controller.ex

@@ -12,6 +12,7 @@ defmodule Pleroma.Web.PleromaAPI.EmojiReactionController do
   alias Pleroma.Web.CommonAPI
   alias Pleroma.Web.MastodonAPI.StatusView
   alias Pleroma.Web.Plugs.OAuthScopesPlug
+  alias Pleroma.Web.Plugs.RateLimiter
 
   plug(Pleroma.Web.ApiSpec.CastAndValidate)
   plug(OAuthScopesPlug, %{scopes: ["write:statuses"]} when action in [:create, :delete])
@@ -22,6 +23,14 @@ defmodule Pleroma.Web.PleromaAPI.EmojiReactionController do
     when action == :index
   )
 
+  plug(
+    RateLimiter,
+    [name: :status_id_action, bucket_name: "status_id_action:emoji_react", params: [:id]]
+    when action in [:create, :delete]
+  )
+
+  plug(RateLimiter, [name: :statuses_actions] when action in [:create, :delete])
+
   defdelegate open_api_operation(action), to: Pleroma.Web.ApiSpec.EmojiReactionOperation
 
   action_fallback(Pleroma.Web.MastodonAPI.FallbackController)

lib/pleroma/web/plugs/http_signature_plug.ex

@@ -45,20 +45,52 @@ defmodule Pleroma.Web.Plugs.HTTPSignaturePlug do
 
   defp maybe_assign_valid_signature(conn) do
     if has_signature_header?(conn) do
-      # we replace the digest header with the one we computed in DigestPlug
-      conn =
-        case conn do
-          %{assigns: %{digest: digest}} = conn -> put_req_header(conn, "digest", digest)
-          conn -> conn
-        end
+      conn = put_computed_digest(conn)
 
-      assign(conn, :valid_signature, Signature.validate_signature(conn))
+      case check_signature_policy(conn) do
+        :ok ->
+          assign(conn, :valid_signature, Signature.validate_signature(conn))
+
+        {:error, reason} ->
+          reject_signature(conn, reason)
+      end
     else
       Logger.debug("No signature header!")
       conn
     end
   end
 
+  # we replace the digest header with the one we computed in DigestPlug
+  defp put_computed_digest(%{assigns: %{digest: digest}} = conn),
+    do: put_req_header(conn, "digest", digest)
+
+  defp put_computed_digest(conn), do: conn
+
+  defp check_signature_policy(conn) do
+    with {:error, reason} <- Signature.check_signature_policy(conn) do
+      if Pleroma.Config.get([:activitypub, :signature_policy], :enforce) == :log do
+        Logger.warning(
+          "HTTP signature policy violation (#{reason}) for #{conn.method} #{conn.request_path}, allowed by config"
+        )
+
+        :ok
+      else
+        {:error, reason}
+      end
+    end
+  end
+
+  # The request is treated as badly signed. The reason is kept so the inbox
+  # can refuse it instead of queueing a failed-signature retry, which only
+  # checks the cryptographic signature.
+  defp reject_signature(conn, reason) do
+    Logger.debug("Rejecting HTTP signature: #{reason}")
+
+    conn
+    |> assign(:valid_signature, false)
+    |> assign(:signature_error, reason)
+  end
+
   defp maybe_assign_actor_id(%{assigns: %{valid_signature: true}} = conn) do
     adapter = Application.get_env(:http_signatures, :adapter)
 

lib/pleroma/web/web_finger.ex

@@ -158,7 +158,11 @@ defmodule Pleroma.Web.WebFinger do
   @cachex Pleroma.Config.get([:cachex, :provider], Cachex)
   def find_lrdd_template(domain) do
     @cachex.fetch!(:host_meta_cache, domain, fn _ ->
-      {:commit, fetch_lrdd_template(domain)}
+      case fetch_lrdd_template(domain) do
+        # retry once the backoff expires instead of caching the miss
+        {:error, :ratelimit} = error -> {:ignore, error}
+        result -> {:commit, result}
+      end
     end)
   rescue
     e -> {:error, "Cachex error: #{inspect(e)}"}
@@ -168,9 +172,13 @@ defmodule Pleroma.Web.WebFinger do
     # WebFinger is restricted to HTTPS - https://tools.ietf.org/html/rfc7033#section-9.1
     meta_url = "https://#{domain}/.well-known/host-meta"
 
-    with {:ok, %{status: status, body: body}} when status in 200..299 <- HTTP.get(meta_url) do
+    with {:ok, %{status: status, body: body}} when status in 200..299 <-
+           HTTP.Backoff.get(meta_url) do
       get_template_from_xml(body)
     else
+      {:error, :ratelimit} = error ->
+        error
+
       error ->
         Logger.warning("Can't find LRDD template in #{inspect(meta_url)}: #{inspect(error)}")
         {:error, :lrdd_not_found}
@@ -210,7 +218,7 @@ defmodule Pleroma.Web.WebFinger do
 
     with address when is_binary(address) <- get_address_from_domain(domain, account),
          {:ok, %{status: status, body: body, headers: headers}} when status in 200..299 <-
-           HTTP.get(
+           HTTP.Backoff.get(
              address,
              [{"accept", "application/xrd+xml,application/jrd+json"}]
            ) do

lib/pleroma/workers/cron/retention_worker.ex

@@ -0,0 +1,40 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Workers.Cron.RetentionWorker do
+  @moduledoc """
+  Evicts one batch of old remote threads per run, see `Pleroma.Retention`.
+  """
+
+  use Oban.Worker,
+    queue: :background,
+    max_attempts: 1,
+    unique: [period: :infinity, states: [:available, :scheduled, :executing]]
+
+  alias Pleroma.Config
+  alias Pleroma.Retention
+
+  require Logger
+
+  @impl true
+  def perform(_job) do
+    config = Config.get(:retention, [])
+
+    if config[:enabled] do
+      stats = Retention.run(config)
+
+      if stats.contexts > 0 do
+        Logger.info(
+          "Retention: evicted #{stats.contexts} threads " <>
+            "(#{stats.objects} objects, #{stats.activities} activities)"
+        )
+      end
+    end
+
+    :ok
+  end
+
+  @impl true
+  def timeout(_job), do: :timer.minutes(30)
+end