hj/pleroma
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Merge remote-tracking branch 'origin/develop' into shigusegubu-new

Browse source
This commit is contained in:
Henry Jameson 2026-10-25 18:45:39 +02:00
commit cafc59bf4f
821 changed files with 18904 additions and 2466 deletions
Download patch file Download diff file Expand all files Collapse all files

85
.woodpecker/docker-armv7.yaml
View file

@ -1,85 +0,0 @@
when:
# Temporary PR validation while arm/v7 Docker builds are being brought up.
- event: pull_request
path: [ "**/*.ex", "**/*.eex", "**/*.exs", "mix.lock", ".woodpecker/**", "Dockerfile" ]
- event: push
branch: ${CI_REPO_DEFAULT_BRANCH}
path: [ "**/*.ex", "**/*.eex", "**/*.exs", "mix.lock", ".woodpecker/**", "Dockerfile" ]
- event: tag
- event: manual
branch: ${CI_REPO_DEFAULT_BRANCH}
- event: manual
branch: stable
# Target the CAX21 arm64 host with native arm32 userspace support, not generic arm64 runners.
labels:
platform: linux/arm
armv7: native
variables:
docker_variables: &docker_variables
repo: pleroma/pleroma
registry: git.pleroma.social
armv7_build_settings: &armv7_build_settings
<<: *docker_variables
build_args:
- ELIXIR_IMG=arm32v7/elixir
- ELIXIR_TAG=1.17.3-otp-26-alpine
- ALPINE_IMG=arm32v7/alpine
- ALPINE_VER=3.20
extra_opts: --custom-platform=linux/arm/v7
armv7_push_settings: &armv7_push_settings
<<: *armv7_build_settings
username:
from_secret: pleroma-ci-user
password:
from_secret: pleroma-ci-password
steps:
docker-armv7-pr:
image: woodpeckerci/plugin-kaniko:2.3.1
when:
- event: pull_request
settings:
<<: *armv7_build_settings
dry_run: true
tags:
- armv7-pr-check
docker-develop-armv7:
image: woodpeckerci/plugin-kaniko:2.3.1
when:
- event: push
branch: ${CI_REPO_DEFAULT_BRANCH}
- event: manual
branch: ${CI_REPO_DEFAULT_BRANCH}
settings:
<<: *armv7_push_settings
tags:
- latest-armv7
- develop-armv7
- ${CI_COMMIT_SHA:0:8}-armv7
docker-stable-armv7:
image: woodpeckerci/plugin-kaniko:2.3.1
when:
- evaluate: 'CI_PIPELINE_EVENT == "manual" && CI_COMMIT_BRANCH == "stable" && CI_COMMIT_TAG == ""'
settings:
<<: *armv7_push_settings
tags:
- latest-armv7
- stable-armv7
- ${CI_COMMIT_SHA:0:8}-armv7
docker-stable-tag-armv7:
image: woodpeckerci/plugin-kaniko:2.3.1
when:
- event: tag
- evaluate: 'CI_PIPELINE_EVENT == "manual" && CI_COMMIT_BRANCH == "stable" && CI_COMMIT_TAG != ""'
settings:
<<: *armv7_push_settings
tags:
- latest-armv7
- stable-armv7
- ${CI_COMMIT_SHA:0:8}-armv7
- ${CI_COMMIT_TAG}-armv7

14
.woodpecker/docker-combine.yaml
View file

@ -18,7 +18,7 @@ labels:
steps:
docker-develop-combine:
image: git.fluffytail.org/phnt/wpc-docker-tagger:latest
image: &docker_image git.pleroma.social/phnt/wpc-docker-tagger:0.9.1
when:
- event: push
branch: ${CI_REPO_DEFAULT_BRANCH}
@ -27,18 +27,18 @@ steps:
settings: &docker_settings
registry: "git.pleroma.social"
image: "pleroma/pleroma"
architectures: [amd64, arm64]
architectures: [amd64, arm64, armv7]
tags:
- latest
- develop
- ${CI_COMMIT_SHA:0:8}
- "${CI_COMMIT_SHA:0:8}"
username:
from_secret: pleroma-ci-user
password:
from_secret: pleroma-ci-password
docker-stable-combine:
image: git.fluffytail.org/phnt/wpc-docker-tagger:latest
image: *docker_image
when:
- evaluate: 'CI_PIPELINE_EVENT == "manual" && CI_COMMIT_BRANCH == "stable" && CI_COMMIT_TAG == ""'
settings:
@ -46,10 +46,10 @@ steps:
tags: &stable_docker_tags
- latest
- stable
- ${CI_COMMIT_SHA:0:8}
- "${CI_COMMIT_SHA:0:8}"
docker-stable-tag-combine:
image: git.fluffytail.org/phnt/wpc-docker-tagger:latest
image: *docker_image
when:
- event: tag
- evaluate: 'CI_PIPELINE_EVENT == "manual" && CI_COMMIT_BRANCH == "stable" && CI_COMMIT_TAG != ""'
@ -57,4 +57,4 @@ steps:
<<: *docker_settings
tags:
- <<: *stable_docker_tags
- ${CI_COMMIT_TAG}
- "${CI_COMMIT_TAG}"

48
.woodpecker/docker.yaml
View file

@ -12,6 +12,7 @@ matrix:
platform:
- linux/amd64
- linux/arm64
- linux/arm
# This is needed for the when clauses below.
labels:
@ -26,11 +27,19 @@ variables:
from_secret: pleroma-ci-user
password:
from_secret: pleroma-ci-password
kaniko_image: &kaniko_image woodpeckerci/plugin-kaniko:2.3.1
armv7_docker_variables: &armv7_docker_variables
<<: *docker_variables
build_args:
- ELIXIR_IMG=docker.io/arm32v7/elixir
- ELIXIR_TAG=1.17.3-otp-27-alpine
- ALPINE_IMG=docker.io/arm32v7/alpine
- ALPINE_VER=3.20
extra_opts: --custom-platform=linux/arm/v7
kaniko_image: &kaniko_image docker.io/woodpeckerci/plugin-kaniko:2.3.2
steps:
docker-develop-amd64:
image: woodpeckerci/plugin-kaniko:2.3.1
image: *kaniko_image
when:
- evaluate: 'platform == "linux/amd64" && CI_COMMIT_BRANCH == "${CI_REPO_DEFAULT_BRANCH}"'
settings:
@ -41,7 +50,7 @@ steps:
- ${CI_COMMIT_SHA:0:8}-amd64
docker-develop-arm64:
image: woodpeckerci/plugin-kaniko:2.3.1
image: *kaniko_image
when:
- evaluate: 'platform == "linux/arm64" && CI_COMMIT_BRANCH == "${CI_REPO_DEFAULT_BRANCH}"'
settings:
@ -51,6 +60,17 @@ steps:
- develop-arm64
- ${CI_COMMIT_SHA:0:8}-arm64
docker-develop-armv7:
image: *kaniko_image
when:
- evaluate: 'platform == "linux/arm" && CI_COMMIT_BRANCH == "${CI_REPO_DEFAULT_BRANCH}"'
settings:
<<: *armv7_docker_variables
tags:
- latest-armv7
- develop-armv7
- ${CI_COMMIT_SHA:0:8}-armv7
docker-stable-amd64:
image: *kaniko_image
when:
@ -106,3 +126,25 @@ steps:
tags:
- <<: *arm64_tags
- ${CI_COMMIT_TAG}-arm64
docker-stable-armv7:
image: *kaniko_image
when:
- evaluate: 'platform == "linux/arm" && CI_PIPELINE_EVENT == "manual" && CI_COMMIT_BRANCH == "stable" && CI_COMMIT_TAG == ""'
settings:
<<: *armv7_docker_variables
tags: &armv7_tags
- latest-armv7
- stable-armv7
- ${CI_COMMIT_SHA:0:8}-armv7
docker-stable-tag-armv7:
image: *kaniko_image
when:
- evaluate: 'platform == "linux/arm" && CI_PIPELINE_EVENT == "tag"'
- evaluate: 'platform == "linux/arm" && CI_PIPELINE_EVENT == "manual" && CI_COMMIT_BRANCH == "stable" && CI_COMMIT_TAG != ""'
settings:
<<: *armv7_docker_variables
tags:
- <<: *armv7_tags
- ${CI_COMMIT_TAG}-armv7

45
.woodpecker/docs.yaml Normal file
View file

@ -0,0 +1,45 @@
when:
- event: push
branch: ${CI_REPO_DEFAULT_BRANCH}
path: [ "docs/**" ]
- event: tag
- event: manual
evaluate: 'CI_COMMIT_BRANCH == "stable" || CI_COMMIT_BRANCH == CI_REPO_DEFAULT_BRANCH'
labels:
platform: linux/amd64
skip_clone: true
variables:
trigger_settings: &trigger_settings
deploy: 'docs'
last-successful: true
token:
from_secret: pleroma-ci-woodpecker-token
repositories:
- pleroma/docs@master
docker_image: &docker_image docker.io/woodpeckerci/plugin-trigger:1.1.1
steps:
trigger-deploy-stable:
when:
- event: tag
- event: manual
branch: stable
image: *docker_image
settings:
<<: *trigger_settings
params:
- BRANCH=stable
trigger-deploy-develop:
when:
- event: push
- event: manual
branch: ${CI_REPO_DEFAULT_BRANCH}
image: *docker_image
settings:
<<: *trigger_settings
params:
- BRANCH=develop

2
.woodpecker/lint.yaml
View file

@ -18,7 +18,7 @@ variables:
steps:
mix-format:
image: &elixir-image
docker.io/elixir:1.15-alpine
docker.io/elixir:1.17-otp-27-alpine
entrypoint: *script_file_entrypoint
failure: ignore
commands:

6
.woodpecker/otp-musl.yaml
View file

@ -33,10 +33,10 @@ variables:
- mkdir release
- export PLEROMA_BUILD_BRANCH=${CI_COMMIT_BRANCH}
- mix release --path release
build_image_amd64: &build_image_amd64 docker.io/hexpm/elixir-amd64:1.17.3-erlang-27.3.4.2-alpine-3.22.1
build_image_amd64: &build_image_amd64 docker.io/hexpm/elixir:1.17.3-erlang-27.3.4.17-alpine-3.22.5
build_image_arm: &build_image_arm docker.io/arm32v7/elixir:1.17.3-alpine
build_image_arm64: &build_image_arm64 docker.io/hexpm/elixir-arm64:1.17.3-erlang-27.3.4.2-alpine-3.22.1
artifacts_uploader_image: &artifacts_uploader_image docker.io/woodpeckercommunity/plugin-gitea-package:0.5.0
build_image_arm64: &build_image_arm64 docker.io/hexpm/elixir:1.17.3-erlang-27.3.4.17-alpine-3.22.5
artifacts_uploader_image: &artifacts_uploader_image docker.io/woodpeckercommunity/plugin-gitea-package:0.5.1
artifacts_uploader_settings: &artifacts_uploader_settings
user:
from_secret: pleroma-ci-user

6
.woodpecker/otp.yaml
View file

@ -33,10 +33,10 @@ variables:
- mkdir release
- export PLEROMA_BUILD_BRANCH=${CI_COMMIT_BRANCH}
- mix release --path release
build_image_amd64: &build_image_amd64 docker.io/hexpm/elixir-amd64:1.17.3-erlang-27.3.4.2-ubuntu-noble-20250716
build_image_amd64: &build_image_amd64 docker.io/hexpm/elixir:1.17.3-erlang-27.3.4.17-ubuntu-noble-20260810
build_image_arm: &build_image_arm docker.io/arm32v7/elixir:1.17.3
build_image_arm64: &build_image_arm64 docker.io/hexpm/elixir-arm64:1.17.3-erlang-27.3.4.2-ubuntu-noble-20250716
artifacts_uploader_image: &artifacts_uploader_image docker.io/woodpeckercommunity/plugin-gitea-package:0.5.0
build_image_arm64: &build_image_arm64 docker.io/hexpm/elixir:1.17.3-erlang-27.3.4.17-ubuntu-noble-20260810
artifacts_uploader_image: &artifacts_uploader_image docker.io/woodpeckercommunity/plugin-gitea-package:0.5.1
artifacts_uploader_settings: &artifacts_uploader_settings
user:
from_secret: pleroma-ci-user

66
.woodpecker/paradedb-integration.yaml Normal file
View file

@ -0,0 +1,66 @@
when:
- event: pull_request
path:
- ".woodpecker/**"
- "lib/pleroma/search/**"
- "test/pleroma/search/**"
- "mix.exs"
- "mix.lock"
evaluate: 'CI_COMMIT_SOURCE_BRANCH != "weblate"'
- event: push
branch: ${CI_REPO_DEFAULT_BRANCH}
path:
- ".woodpecker/**"
- "lib/pleroma/search/**"
- "test/pleroma/search/**"
- "mix.exs"
- "mix.lock"
depends_on:
- lint
labels:
platform: linux/amd64
variables:
script_file_entrypoint: &script_file_entrypoint
- /bin/sh
- -c
- 'printf "%s" "$CI_SCRIPT" | base64 -d > /tmp/ci-script.sh && /bin/sh -xe /tmp/ci-script.sh'
steps:
paradedb-integration:
image: docker.io/elixir:1.19-otp-28-alpine
entrypoint: *script_file_entrypoint
environment:
MIX_ENV: test
VIX_COMPILATION_MODE: PLATFORM_PROVIDED_LIBVIPS
DB_HOST: postgres
DB_PORT: 5432
PARADEDB_TEST_URL: postgres://postgres:postgres@paradedb:5432/paradedb
commands:
- apk add --no-cache build-base cmake exiftool ffmpeg vips-dev file-dev git openssl postgresql-client
- mix local.hex --force
- mix local.rebar --force
- mix deps.get
- timeout 60 sh -c 'until pg_isready -h postgres -U postgres -d pleroma_test; do sleep 1; done' || { echo "PostgreSQL did not become ready" >&2; exit 1; }
- timeout 60 sh -c 'until pg_isready -h paradedb -U postgres -d paradedb; do sleep 1; done' || { echo "ParadeDB did not become ready" >&2; exit 1; }
- mix ecto.create
- mix ecto.migrate
- mix test --include integration test/pleroma/search/parade_db_integration_test.exs
services:
postgres:
image: docker.io/postgres:13-alpine
failure: ignore
environment:
POSTGRES_DB: pleroma_test
POSTGRES_USER: postgres
POSTGRES_PASSWORD: postgres
paradedb:
image: docker.io/paradedb/paradedb:0.24.3-pg15@sha256:997d62f6c2605becb7c45878af3f8c8c698f355006b675b3ce212d4eb2d14f2e
failure: ignore
environment:
POSTGRES_DB: paradedb
POSTGRES_USER: postgres
POSTGRES_PASSWORD: postgres

103
.woodpecker/spec-build.yaml Normal file
View file

@ -0,0 +1,103 @@
when:
- event: push
branch: ${CI_REPO_DEFAULT_BRANCH}
path:
- ".woodpecker/spec-build.yaml"
- "mix.exs"
- "mix.lock"
- "lib/pleroma/web.ex"
- "lib/pleroma/web/**/*.ex"
- event: tag
- event: manual
evaluate: 'CI_COMMIT_BRANCH == "stable" || CI_COMMIT_BRANCH == CI_REPO_DEFAULT_BRANCH'
labels:
platform: linux/amd64
concurrency: 1
variables:
artifacts_uploader_image: &artifacts_uploader_image docker.io/woodpeckercommunity/plugin-gitea-package:0.5.1
artifacts_uploader_settings: &artifacts_uploader_settings
user:
from_secret: pleroma-ci-user
password:
from_secret: pleroma-ci-password
owner: 'pleroma'
package_name: api-docs-spec
file_source: ./spec.json
file_name: spec.json
update: 'true'
trigger_image: &trigger_image docker.io/woodpeckerci/plugin-trigger:1.1.1
trigger_settings: &trigger_settings
deploy: 'api-docs'
last-successful: true
token:
from_secret: pleroma-ci-woodpecker-token
repositories:
- pleroma/api-docs@master
steps:
build-spec:
image: docker.io/hexpm/elixir:1.17.3-erlang-27.3.4.17-alpine-3.22.5
environment:
MIX_ENV: prod
VIX_COMPILATION_MODE: PLATFORM_PROVIDED_LIBVIPS
commands:
- apk update --no-cache
- apk add git build-base cmake file-dev openssl vips-dev zip
- mix local.hex --force
- mix local.rebar --force
- mix deps.get --only prod
- mix pleroma.openapi_spec spec.json
- |
printf 'BRANCH=stable\nSPEC_VERSION=%s\n' "$${CI_COMMIT_SHA}" > trigger-deploy-stable.params
printf 'BRANCH=develop\nSPEC_VERSION=%s\n' "$${CI_COMMIT_SHA}" > trigger-deploy-develop.params
upload-spec:
image: *artifacts_uploader_image
settings:
<<: *artifacts_uploader_settings
package_version: ${CI_COMMIT_SHA}
upload-spec-stable:
when:
- event: tag
- event: manual
branch: stable
image: *artifacts_uploader_image
settings:
<<: *artifacts_uploader_settings
package_version: stable
upload-spec-develop:
when:
- event: push
- event: manual
branch: ${CI_REPO_DEFAULT_BRANCH}
image: *artifacts_uploader_image
settings:
<<: *artifacts_uploader_settings
package_version: develop
trigger-deploy-stable:
when:
- event: tag
- event: manual
branch: stable
image: *trigger_image
settings:
<<: *trigger_settings
params:
- trigger-deploy-stable.params
trigger-deploy-develop:
when:
- event: push
- event: manual
branch: ${CI_REPO_DEFAULT_BRANCH}
image: *trigger_image
settings:
<<: *trigger_settings
params:
- trigger-deploy-develop.params

8
.woodpecker/unit-testing-elixir-1.15.yaml → .woodpecker/unit-testing-elixir-1.17.yaml
View file

@ -19,15 +19,16 @@ variables:
- 'printf "%s" "$CI_SCRIPT" | base64 -d > /tmp/ci-script.sh && /bin/sh -xe /tmp/ci-script.sh'
steps:
unit-testing-elixir-1.15:
image: elixir:1.15-alpine
unit-testing-elixir-1.17:
image: elixir:1.17-otp-27-alpine
entrypoint: *script_file_entrypoint
environment:
MIX_ENV: test
VIX_COMPILATION_MODE: PLATFORM_PROVIDED_LIBVIPS
DB_HOST: postgres
DB_PORT: 5432
commands:
- apk add --no-cache build-base cmake exiftool ffmpeg file-dev git openssl
- apk add --no-cache build-base cmake exiftool ffmpeg vips-dev file-dev git openssl
- adduser -D -h /home/testuser testuser
- mkdir -p /home/testuser/.mix /home/testuser/.hex
- chown -R testuser:testuser . /home/testuser
@ -39,6 +40,7 @@ steps:
services:
postgres:
image: postgres:13-alpine
failure: ignore
environment:
POSTGRES_DB: pleroma_test
POSTGRES_USER: postgres

8
.woodpecker/unit-testing-elixir-1.18.yaml → .woodpecker/unit-testing-elixir-1.19.yaml
View file

@ -19,15 +19,16 @@ variables:
- 'printf "%s" "$CI_SCRIPT" | base64 -d > /tmp/ci-script.sh && /bin/sh -xe /tmp/ci-script.sh'
steps:
unit-testing-elixir-1.18:
image: elixir:1.18-otp-27-alpine
unit-testing-elixir-1.19:
image: elixir:1.19-otp-28-alpine
entrypoint: *script_file_entrypoint
environment:
MIX_ENV: test
VIX_COMPILATION_MODE: PLATFORM_PROVIDED_LIBVIPS
DB_HOST: postgres
DB_PORT: 5432
commands:
- apk add --no-cache build-base cmake exiftool ffmpeg file-dev git openssl
- apk add --no-cache build-base cmake exiftool ffmpeg vips-dev file-dev git openssl
- adduser -D -h /home/testuser testuser
- mkdir -p /home/testuser/.mix /home/testuser/.hex
- chown -R testuser:testuser . /home/testuser
@ -39,6 +40,7 @@ steps:
services:
postgres:
image: postgres:13-alpine
failure: ignore
environment:
POSTGRES_DB: pleroma_test
POSTGRES_USER: postgres

109
CHANGELOG.md
View file

@ -4,6 +4,115 @@ All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
## 2.11.0
### Security
- Update Bandit to address HTTP request handling vulnerabilities.
- Update bandit to version 1.12.4 to fix a DoS vulnerability
- Update bandit to version 1.12.5 to fix an HTTP/2 DoS vulnerability
- Update bandit to version 1.13.1 resolving a stuck process and unvalidated HTTP/1 headers vulnerabilities
- Updated CAStore to version 1.0.22 with updated Mozilla certificate bundle after eMudhra Intermediate CA Compromise
- Update certifi to version 2.18.0, which removes the distrusted emSign roots
- Update plug_crypto to version 2.2.1, so a malformed shout socket token no longer writes the secret key base to the error log
- Update Cowboy to address multipart header parsing denial-of-service vulnerabilities.
- Update cowboy to version 2.19.0 which fixes a memory exhaustion DoS
- Update Cowlib to address HTTP parsing resource exhaustion and injection vulnerabilities.
- Update cowlib to version 2.20.0 which fixes HTTP splitting and header smuggling/injection vulnerabilities
- Update gun to version 2.7.0 and cowlib to version 2.21.0 resolving a header smuggling vulnerability
- Update Decimal and compatible dependencies to address a rounding vulnerability.
- Update Decimal to version 3.2.0 resolving an unbounded allocation vulnerability
- Replace retired Earmark rendering with the maintained EarmarkParser and an attribute-escaping renderer.
- Update gun to version 2.6.0 which fixes an HTTP request/response smuggling vulnerability
- Update Hackney and its HTTPoison integration to address SSRF, timeout, and injection vulnerabilities.
- Hackney, Finch, Mua and the Swoosh HTTP mail adapters now trust only the CAStore certificate bundle, which rejects the compromised eMudhra intermediate CAs
- Ensure Host header is present and matches instance URI
- Update mint to version 1.10.0 which fixes multiple DoS vulnerabilities
- Update mint to version 1.11.0 resolving multiple vulnerabilities
- Update Phoenix LiveView to version 1.2.9 to fix an open redirect vulnerability
- Update Phoenix to version 1.8.11 resolving a process-exhaustion DoS vulnerability
- Update Phoenix to address long-poll NDJSON request denial-of-service vulnerabilities.
- Update Plug to address multipart header parsing denial-of-service vulnerabilities.
- Update Plug.Cowboy to address HTTP/2 scheme atom exhaustion denial-of-service vulnerabilities.
- Update Postgrex to address notification channel-name SQL injection vulnerabilities.
- Update postgrex to version 0.22.4 to fix an SQL injection vulnerability
- Remove unused QUIC lockfile entry with vulnerable TLS verification behavior.
- Update swoosh to version 1.28.0 which fixes a url injection vulnerability for the Microsoft Graph adapter
- Scrubbers: remove links and images whose scheme is hidden with tabs, newlines or leading control characters (for example java<TAB>script:), which browsers still run as javascript:. Custom scrubbers should use Pleroma.HTML.URIAttributes.allow_tag_with_uri_attributes instead of the FastSanitize macro
### Changed
- Update bundled Pleroma-FE to 2.11.4
- Allow disabling Chats feature
- Pleroma now requires Elixir 1.17 or newer and Erlang/OTP 27 or newer
- Update Debian and Ubuntu installation requirements and OTP dependencies
- Update Jason to support Decimal 3 dependency resolution.
- Changed how Logger is configured due to changes in Elixir and added deprecation warnings for the old configuration format
- Update majic to 1.2.0.
- Avoid a duplicate user query during OAuth authentication.
- Use the Hex package for oban_plugins_lazarus.
- Docs OpenBSD: Increased recommended `datasize` from 1536M to 4096M to avoid Pleroma crashes
- OpenRC: Automatically run migrations before starting pleroma
- Optimize orphaned OAuth apps cleanup by deleting records in batches instead of one by one.
- Switch patched Phoenix 1.7.14 back to upstream with Phoenix 1.8.0+
- Add optional migrations for rollback from Rebased
- Use upstream Hex releases for the `remote_ip` dependency and expose client IP ranges for remote IP resolution.
- Improve user search / autocompletion ordering.
- Gun is now the default HTTP client instead of Hackney
### Added
- Allow creating accounts without a password in the AdminAPI
- Implemented ConfigDB configuration blacklist and accompanying Mix task
- Accept more real-world ActivityPub object shapes collected by fedidev.fun
- Add Mastodon-compatible grouped notifications API endpoints under `/api/v2/notifications` for newly-created notifications
- Docs: Added documentation for installing newer version of Elixir/Erlang than what distro ships
- Allow attaching emojis to user lists
- Implement basics of Mastodon admin API
- Add backend support for Misskey Markdown (MFM) posts
- Add mix_audit for dependency vulnerability checks.
- Oban: Enabled the Reindexer plugin to reduce oban_jobs index DB bloat
- Add ParadeDB (pg_search) as an optional external search backend, indexing status content, content warnings, and attachment descriptions.
- Add support for Mastodon-compatible webhooks
### Fixed
- Fixed `/api/v1/pleroma/accounts/:id/favourites` ordering and pagination to use the favourite date.
- Reject unsupported or ambiguous inbox signatures instead of queueing a doomed failed-signature retry, allowing clients to fall back to legacy HTTP Signatures.
- Switch native captcha to the published pleroma_captcha Hex package.
- Updated the captcha package to fix compilation failure on BSD-based systems
- Dockerfile: update base images to fix problems with SSL handshakes
- Docs: Fixed OTP install links in documentation, broke up wall of install commands into smaller chunks, fixed sudo commands execution
- Fix notification and timeline queries for users with domain blocks choosing a plan that scans every active user per row; the followed-accounts exception is now an `ARRAY(SELECT ...)` InitPlan that the planner costs once.
- Fixed Elixir 1.19 warnings
- Support fetching featured collection items from the first collection page
- Fix preferred frontend not working due to InstanceStatic plug prematurely returning HTTP 404
- Fixed Gun HTTP/1 concurrency, forwarding and authenticated proxy handling, tunnel requests, and streamed connection cleanup. Updated Gun and Cowlib for HTTP client security fixes.
- Stop leaking a Hackney connection process, and the response bytes it read, for every unpooled request that ends without a reusable socket
- Fix hashtag timeline query for additional tag handling
- Fix compatibility with timestamped HTTP Signatures used by GoToSocial
- Handle reports with just actor ap id as the object
- Fix Swoosh HTTP mail adapters failing with `cacerts: :undefined` on OTP 26+
- Echo Mastodon-style `Sec-WebSocket-Protocol` tokens in streaming WebSocket handshakes.
- Enforce the max_body HTTP option again, which Hackney 4 silently stopped honouring because it always returns a fully read body
- Abort incomplete media proxy responses instead of caching partial bodies.
- Updated Pleroma-FE build URL after Forgejo migration
- Fix votersCount inflation when same voter picks multiple options
- Status: Fill all MastoAPI required values in preview cards
- Avoid creating atoms from serialized PublisherWorker parameters.
- Reject incoming reports when both the reporter and reported account are remote
- Fixed account status and media views exposing activities restricted from unauthenticated users.
- Stop leaking a Hackney connection process for every media proxy response that was streamed to the end
- Fix RSS feeds to be W3C compliant and render better in RSS readers
- Keep followed users visible in streams when their domain is blocked.
- Fix intermittent failure to compile syslog package
- Fix inconsistent User cache keys and invalidate numeric ID aliases.
- Include arm32v7 in multi-arch Docker images
- RichMedia: Fix backfill causing old posts to show up on timelines by disabling it in MastoAPI StatusView
- Fix Admin-FE installation by downloading builds from the package registry instead of GitLab job artifacts.
- Hackney: use HTTP/1.1 only, so closing one media proxy response no longer breaks other downloads from the same host
- Hackney: use the configured proxy when its host is an IP address; Hackney 4 ignored such proxies and connected directly
## 2.10.2
### Security

4
Dockerfile
View file

@ -1,8 +1,8 @@
# https://hub.docker.com/r/hexpm/elixir/tags
ARG ELIXIR_IMG=hexpm/elixir
ARG ELIXIR_VER=1.17.3
ARG ERLANG_VER=26.2.5.6
ARG ALPINE_VER=3.17.9
ARG ERLANG_VER=27.3.4.17
ARG ALPINE_VER=3.23.5
ARG ELIXIR_TAG=${ELIXIR_VER}-erlang-${ERLANG_VER}-alpine-${ALPINE_VER}
ARG ALPINE_IMG=alpine

2
README.md
View file

@ -1,4 +1,4 @@
<img src="https://git.pleroma.social/attachments/06a95f5a-7cac-42ad-8b1d-1483f1739f38" width="300px" />
<img src="https://git.pleroma.social/pleroma/pleroma/raw/branch/develop/logo-with-name.png" width="300px" />
## About

1
changelog.d/captcha-package.fix
View file

@ -1 +0,0 @@
Switch native captcha to the published pleroma_captcha Hex package.

1
changelog.d/context-cleanup.skip
View file

@ -1 +0,0 @@
litepub-0.1.jsonld cleanup

1
changelog.d/featured-collection-page.fix
View file

@ -1 +0,0 @@
Support fetching featured collection items from the first collection page

1
changelog.d/host-header-verification.security
View file

@ -1 +0,0 @@
Ensure Host header is present and matches instance URI

1
changelog.d/http-signatures-0.1.3.fix
View file

@ -1 +0,0 @@
Fix compatibility with timestamped HTTP Signatures used by GoToSocial

1
changelog.d/iceshrimpnet-reports-fix.fix
View file

@ -1 +0,0 @@
Handle reports with just actor ap id as the object

1
changelog.d/legacy-sequences.fix Normal file
View file

@ -0,0 +1 @@
Delete legacy Postgres sequences for activities, chats, and users which were converted to Flake IDs

1
changelog.d/majic-1.2.0.change
View file

@ -1 +0,0 @@
Update majic to 1.2.0.

1
changelog.d/mastodon-websocket-protocol.fix
View file

@ -1 +0,0 @@
Echo Mastodon-style `Sec-WebSocket-Protocol` tokens in streaming WebSocket handshakes.

1
changelog.d/mfm-backend.add
View file

@ -1 +0,0 @@
Add backend support for Misskey Markdown (MFM) posts

0
changelog.d/mfm-extend.skip
View file

1
changelog.d/oban-plugins-lazarus-hex.change
View file

@ -1 +0,0 @@
Use the Hex package for oban_plugins_lazarus.

1
changelog.d/phoenix-sec-websocket-headers.change
View file

@ -1 +0,0 @@
Switch patched Phoenix 1.7.14 back to upstream with Phoenix 1.8.0+

1
changelog.d/pleroma-fe-link.fix
View file

@ -1 +0,0 @@
Updated Pleroma-FE build URL after Forgejo migration

1
changelog.d/poll-voters-count-inflation.fix
View file

@ -1 +0,0 @@
Fix votersCount inflation when same voter picks multiple options

1
changelog.d/reject-third-party-reports.fix
View file

@ -1 +0,0 @@
Reject incoming reports when both the reporter and reported account are remote

0
changelog.d/docker-e2e-image.skip → changelog.d/release-2.11.0.skip
View file

1
changelog.d/remote-ip-hex.change
View file

@ -1 +0,0 @@
Use upstream Hex releases for the `remote_ip` dependency and expose client IP ranges for remote IP resolution.

1
changelog.d/user-search-sorting.change
View file

@ -1 +0,0 @@
Improve user search / autocompletion ordering.

0
changelog.d/weblate-ci.skip
View file

1
changelog.d/wss-necroposts.fix
View file

@ -1 +0,0 @@
RichMedia: Fix backfill causing old posts to show up on timelines by disabling it in MastoAPI StatusView

42
config/config.exs
View file

@ -133,13 +133,17 @@ config :pleroma, Pleroma.Web.Endpoint,
]
# Configures Elixir's Logger
config :logger, backends: [:console]
# Primary config
config :logger, level: :debug
config :logger, :console,
level: :debug,
# Console config
config :logger, :default_handler, level: :debug
config :logger, :default_formatter,
format: "\n$time $metadata[$level] $message\n",
metadata: [:actor, :path, :type, :user]
# Syslog config
config :logger, :ex_syslogger,
level: :debug,
ident: "pleroma",
@ -157,7 +161,11 @@ config :mime, :types, %{
"image/apng" => ["apng"]
}
config :tesla, adapter: Tesla.Adapter.Hackney
config :tesla, adapter: Tesla.Adapter.Gun
# Hackney 4 shares one HTTP/2 connection per host, and closing one response
# (as the media proxy does) stops it for every other stream on it.
config :hackney, default_protocols: [:http1]
# Configures http settings, upstream proxy etc.
config :pleroma, :http,
@ -603,7 +611,11 @@ config :pleroma, Oban,
search_indexing: [limit: 10, paused: true],
slow: 5
],
plugins: [Oban.Plugins.Lazarus, {Oban.Plugins.Pruner, max_age: 900}],
plugins: [
Oban.Plugins.Lazarus,
{Oban.Plugins.Pruner, max_age: 900},
{Oban.Plugins.Reindexer, schedule: "@weekly"}
],
crontab: [
{"0 0 * * 0", Pleroma.Workers.Cron.DigestEmailsWorker},
{"0 0 * * *", Pleroma.Workers.Cron.NewUsersDigestWorker},
@ -794,7 +806,7 @@ config :pleroma, :frontends,
"name" => "admin-fe",
"git" => "https://git.pleroma.social/pleroma/admin-fe",
"build_url" =>
"https://git.pleroma.social/pleroma/admin-fe/-/jobs/artifacts/${ref}/download?job=build",
"https://git.pleroma.social/api/packages/pleroma/generic/admin-fe-builds/${ref}/latest.zip",
"ref" => "develop"
},
"soapbox" => %{
@ -928,7 +940,8 @@ config :pleroma, Pleroma.User.Backup,
timeout: :timer.minutes(30)
config :pleroma, ConcurrentLimiter, [
{Pleroma.Search, [max_running: 30, max_waiting: 50]}
{Pleroma.Search, [max_running: 30, max_waiting: 50]},
{Pleroma.Webhook.Notify, [max_running: 5, max_waiting: 200]}
]
config :pleroma, Pleroma.Web.WebFinger, domain: nil, update_nickname_on_user_fetch: true
@ -940,6 +953,15 @@ config :pleroma, Pleroma.Search.Meilisearch,
private_key: nil,
initial_indexing_chunk_size: 100_000
config :pleroma, Pleroma.Search.ParadeDB,
url: nil,
table: "pleroma_search_documents",
fuzzy_distance: 0
config :pleroma, Pleroma.Search.ParadeDB.Repo,
pool_size: 2,
prepare: :unnamed
config :pleroma, Pleroma.Application,
background_migrators: true,
internal_fetch: true,
@ -962,6 +984,10 @@ config :pleroma, Pleroma.Search.QdrantSearch,
vectors: %{size: 384, distance: "Cosine"}
}
config :pleroma, :database_config_blacklist, [
{:pleroma, :logger}
]
config :pleroma, :database_config_whitelist, [
{:pleroma},
{:cors_plug},
@ -971,6 +997,8 @@ config :pleroma, :database_config_whitelist, [
{:web_push_encryption, :vapid_details}
]
config :pleroma, Pleroma.Chat, enabled: true
# Import environment specific config. This must remain at the bottom
# of this file so it overrides the configuration defined above.
import_config "#{Mix.env()}.exs"

77
config/description.exs
View file

@ -2726,6 +2726,19 @@ config :pleroma, :config_description, [
}
]
},
%{
group: :pleroma,
key: Pleroma.Chat,
type: :group,
description: "Pleroma Chat settings",
children: [
%{
key: :enabled,
type: :boolean,
description: "Enables the chats API."
}
]
},
%{
group: :pleroma,
key: :http,
@ -3085,9 +3098,16 @@ config :pleroma, :config_description, [
%{
key: :max_connections,
type: :integer,
description: "Maximum number of connections in the pool. Default: 250 connections.",
description:
"Maximum total number of connections in the pool. HTTP/1 origins may use multiple connections within this limit, while HTTP/2 connections are multiplexed. Default: 250 connections.",
suggestions: [250]
},
%{
key: :max_idle_time,
type: :integer,
description: "Time before an unused connection is closed. Default: 30000ms.",
suggestions: [30_000]
},
%{
key: :connect_timeout,
type: :integer,
@ -3503,6 +3523,26 @@ config :pleroma, :config_description, [
suggestion: [5]
}
]
},
%{
key: Pleroma.Webhook.Notify,
type: :keyword,
description: "Concurrent limits configuration for webhooks.",
suggestions: [max_running: 5, max_waiting: 200],
children: [
%{
key: :max_running,
type: :integer,
description: "Max running concurrently jobs.",
suggestion: [5]
},
%{
key: :max_waiting,
type: :integer,
description: "Max waiting jobs.",
suggestion: [200]
}
]
}
]
},
@ -3516,7 +3556,12 @@ config :pleroma, :config_description, [
key: :module,
type: :keyword,
description: "Selected search module.",
suggestion: [Pleroma.Search.DatabaseSearch, Pleroma.Search.Meilisearch]
suggestion: [
Pleroma.Search.DatabaseSearch,
Pleroma.Search.Meilisearch,
Pleroma.Search.QdrantSearch,
Pleroma.Search.ParadeDB
]
}
]
},
@ -3549,6 +3594,34 @@ config :pleroma, :config_description, [
}
]
},
%{
group: :pleroma,
key: Pleroma.Search.ParadeDB,
type: :group,
description: "ParadeDB settings.",
children: [
%{
key: :url,
type: :string,
description:
"ParadeDB database URL (separate Postgres instance). " <>
"Can also be set via `PARADEDB_DATABASE_URL`.",
suggestion: ["postgres://postgres:postgres@127.0.0.1:5432/paradedb"]
},
%{
key: :table,
type: :string,
description: "Table name used for search documents.",
suggestion: ["pleroma_search_documents"]
},
%{
key: :fuzzy_distance,
type: :integer,
description: "Edit-distance used for fuzzy token matching (0 disables fuzziness, max 2).",
suggestion: [0, 1, 2]
}
]
},
%{
group: :pleroma,
key: Pleroma.Language.LanguageDetector,

2
config/dev.exs
View file

@ -36,7 +36,7 @@ config :pleroma, Pleroma.Emails.Mailer, adapter: Swoosh.Adapters.Local
# different ports.
# Do not include timestamps in development logs
config :logger, Logger.Backends.Console, format: "$metadata[$level] $message\n"
config :logger, :default_formatter, format: "$metadata[$level] $message\n"
# Set a higher stacktrace during development. Avoid configuring such
# in production as building large stacktraces may be expensive.

3
config/prod.exs
View file

@ -20,8 +20,7 @@ config :pleroma, Pleroma.Web.Endpoint,
config :phoenix, serve_endpoints: true
# Do not print debug messages in production
config :logger, Logger.Backends.Console, level: :info
config :logger, :console, level: :info
config :logger, :default_handler, level: :info
config :logger, :ex_syslogger, level: :info
# ## SSL Support

8
config/test.exs
View file

@ -15,9 +15,11 @@ config :pleroma, Pleroma.Captcha,
method: Pleroma.Captcha.Mock
# Print only warnings and errors during test
config :logger, :console,
level: :warning,
format: "\n[$level] $message\n"
config :logger, :default_handler, level: :warning
config :logger, :default_formatter,
format: "\n[$level] $message\n",
colors: [enabled: false]
config :pleroma, :auth, oauth_consumer_strategies: []

16
docs/administration/CLI_tasks/frontend.md
View file

@ -76,6 +76,22 @@ The resulting frontend will always be installed into a folder of this template:
Careful: This folder will be completely replaced on installation.
## Installing Admin-FE
=== "OTP"
```sh
./bin/pleroma_ctl frontend install admin-fe --ref stable
```
=== "From Source"
```sh
mix pleroma.frontend install admin-fe --ref stable
```
Builds come from the package registry: `https://git.pleroma.social/api/packages/pleroma/generic/admin-fe-builds/${ref}/latest.zip`. `stable` and `develop` are published; `stable` is the better choice for production. A release tag works as `--ref` once a build for it is published.
## Example installation for an unknown frontend
The installation process is the same, but you will have to give all the needed options on the command line. For example:

2
docs/administration/frontends-management.md
View file

@ -56,7 +56,7 @@ Due to aforementioned bug, preferred way of installing frontends in AdminFE is b
![screenshot of admin-fe with instructions on how to install a frontend](../assets/way_to_install_frontends.png)
and filling in the fields. Unfortunately AdminFE does not provide the raw data necessary for you to fill those fields, so your best bet is to see what backend returns in browser's devtools or refer to the [source code](https://git.pleroma.social/pleroma/pleroma/-/blob/develop/config/config.exs?ref_type=heads#L742-791). For the most part, only **Name**, **Ref** (i.e. version) and **Build URL** fields are required, although some frontends might also require **Build Directory** to work.
For pleroma-fe you can use either `master` or `develop` refs, or potentially any ref in GitLab that has artifacts for `build` job, but that's outside scope of this document.
Pleroma-FE and Admin-FE builds come from the package registry, not from GitLab job artifacts. For Admin-FE, set **Name** to `admin-fe`, **Ref** to `stable` (or `develop`) and **Build URL** to `https://git.pleroma.social/api/packages/pleroma/generic/admin-fe-builds/${ref}/latest.zip`. Only refs with a published build can be installed: `stable`, `develop`, and release tags once they are published.
### Selecting default frontend

37
docs/configuration/cheatsheet.md
View file

@ -541,6 +541,15 @@ Available caches:
## HTTP client
Pleroma uses [Gun](https://github.com/ninenines/gun) for outgoing HTTP requests. To use Hackney instead:
```elixir
config :tesla, adapter: Tesla.Adapter.Hackney
```
!!! note
With Hackney, size limits for fetched responses (for example for link previews) are checked only after Hackney has read the whole response. They limit what Pleroma keeps, not how much it downloads or holds in memory. Gun stops reading at the limit.
### :http
* `proxy_url`: an upstream proxy to fetch posts and/or media with, (default: `nil`)
@ -550,6 +559,8 @@ Available caches:
### :hackney_pools
*For `hackney` adapter*
Advanced. Tweaks Hackney (http client) connections pools.
There's three pools used:
@ -573,7 +584,8 @@ Settings for HTTP connection pool.
* `:connection_acquisition_wait` - Timeout to acquire a connection from pool.The total max time is this value multiplied by the number of retries.
* `connection_acquisition_retries` - Number of attempts to acquire the connection from the pool if it is overloaded. Each attempt is timed `:connection_acquisition_wait` apart.
* `:max_connections` - Maximum number of connections in the pool.
* `:max_connections` - Maximum total number of connections in the pool. HTTP/1 origins may use multiple connections within this limit, while HTTP/2 connections are multiplexed.
* `:max_idle_time` - Time before an unused connection is closed.
* `:connect_timeout` - Timeout to connect to the host.
* `:reclaim_multiplier` - Multiplied by `:max_connections` this will be the maximum number of idle connections that will be reclaimed in case the pool is overloaded.
@ -858,21 +870,23 @@ Web Push Notifications configuration. You can use the mix task `mix web_push.gen
* ``private_key``: VAPID private key
## :logger
* `backends`: `:console` is used to send logs to stdout, `{ExSyslogger, :ex_syslogger}` to log to syslog
* Logging to console/stdout is done by default, use `{ExSyslogger, :ex_syslogger}` to log to syslog
An example to enable ONLY ExSyslogger (f/ex in ``prod.secret.exs``) with info and debug suppressed:
```elixir
config :logger,
config :pleroma, :logger,
backends: [{ExSyslogger, :ex_syslogger}]
config :logger, default_handler: false
config :logger, :ex_syslogger,
level: :warning
```
Another example, keeping console output and adding the pid to syslog output:
```elixir
config :logger,
backends: [:console, {ExSyslogger, :ex_syslogger}]
config :pleroma, :logger,
backends: [{ExSyslogger, :ex_syslogger}]
config :logger, :ex_syslogger,
level: :warning,
@ -883,23 +897,22 @@ See: [logger’s documentation](https://hexdocs.pm/logger/Logger.html) and [ex_s
An example of logging info to local syslog, but debug to console:
```elixir
config :logger,
backends: [ {ExSyslogger, :ex_syslogger}, :console ],
level: :info
config :pleroma, :logger,
backends: [{ExSyslogger, :ex_syslogger}]
config :logger, :ex_syslogger,
level: :info,
ident: "pleroma",
format: "$metadata[$level] $message"
config :logger, :console,
level: :debug,
config :logger, :default_handler,
level: :debug
config :logger, :default_formatter,
format: "\n$time $metadata[$level] $message\n",
metadata: [:request_id]
```
## Database options
### RUM indexing for full text search

3
docs/configuration/howto_proxy.md
View file

@ -10,3 +10,6 @@ The other way to do it, for example, with Tor you would most likely add somethin
```
config :pleroma, :http, proxy_url: {:socks5, :localhost, 9050}
```
The Gun HTTP adapter supports SOCKS5. SOCKS4 proxies require another HTTP adapter.
For regular HTTP proxies, Gun forwards plain HTTP requests directly and uses CONNECT tunnels for HTTPS requests.

154
docs/configuration/search.md
View file

@ -85,7 +85,7 @@ You will see a "Default Admin API Key", this is the key you actually put into yo
### Initial indexing
After setting up the configuration, you'll want to index all of your already existing posts. Only public posts are indexed. You'll only
After setting up the configuration, you'll want to index all of your already existing posts. Only public/unlisted posts are indexed. You'll only
have to do it one time, but it might take a while, depending on the amount of posts your instance has seen. This is also a fairly RAM
consuming process for `meilisearch`, and it will take a lot of RAM when running if you have a lot of posts (seems to be around 5G for ~1.2
million posts while idle and up to 7G while indexing initially, but your experience may be different).
@ -145,3 +145,155 @@ This will clear **all** the posts from the search index. Note, that deleted post
there is no need to actually clear the whole index, unless you want **all** of it gone. That said, the index does not hold any information
that cannot be re-created from the database, it should also generally be a lot smaller than the size of your database. Still, the size
depends on the amount of text in posts.
## ParadeDB
[ParadeDB](https://www.paradedb.com/) is a Postgres extension that provides BM25 full-text search. In Pleroma, it can be used as an
external search backend by pointing Pleroma at a separate Postgres instance with the `pg_search` extension installed.
Pleroma's integration suite tests `pg_search` 0.24.3 on PostgreSQL 15.
Pleroma will maintain a small `pleroma_search_documents` table in that database (via the existing search indexing queue) and run search
queries against it.
The indexed text includes status content, content warnings, and attachment descriptions.
### Configuration
Set the search module to `Pleroma.Search.ParadeDB` and configure the ParadeDB database URL. These settings must be in static/runtime config
(or `PARADEDB_DATABASE_URL`) so the dedicated ParadeDB Repo is started with the application. They cannot be migrated to or changed through
ConfigDB.
> config :pleroma, Pleroma.Search, module: Pleroma.Search.ParadeDB
>
> config :pleroma, Pleroma.Search.ParadeDB,
> url: System.get_env("PARADEDB_DATABASE_URL"),
> table: "pleroma_search_documents",
> fuzzy_distance: 0
>
> config :pleroma, Pleroma.Search.ParadeDB.Repo,
> pool_size: 2,
> prepare: :unnamed
When connecting over an untrusted network, enable certificate verification in the Repo configuration. For example:
> config :pleroma, Pleroma.Search.ParadeDB.Repo,
> ssl: [
> verify: :verify_peer,
> cacertfile: "/etc/pleroma/paradedb-ca.pem",
> server_name_indication: ~c"paradedb.example.com"
> ]
### Initial rollout
The `pg_search` extension must already be available in the database. Creating the extension normally requires an administrative database
role. The Pleroma runtime role requires `CONNECT` on the database, `USAGE` on the table's schema and the `pdb` schema, and `SELECT`, `INSERT`, `UPDATE`,
and `DELETE` on the generated table. For example, after creating the table as its owner:
```sql
GRANT CONNECT ON DATABASE paradedb TO pleroma_search;
GRANT USAGE ON SCHEMA public, pdb TO pleroma_search;
GRANT SELECT, INSERT, UPDATE, DELETE ON TABLE pleroma_search_documents TO pleroma_search;
```
Use this rollout order to prevent jobs from being handled by the previous backend:
1. Install `pg_search` 0.24.3 on PostgreSQL 15 or another deliberately tested combination.
2. Stop Pleroma so no search indexing jobs are running.
3. Put the ParadeDB backend in static configuration.
4. Create the table and BM25 index in a one-shot offline process using `PARADEDB_DATABASE_URL` with the table owner/setup credential.
5. Apply the runtime grants shown above, set the service's `PARADEDB_DATABASE_URL` to the restricted runtime credential, and start Pleroma.
6. Start the initial backfill and monitor the `search_indexing` queue.
Create the table and BM25 index once with:
=== "OTP"
```sh
PARADEDB_DATABASE_URL='OWNER_URL' PLEROMA_CTL_RPC_DISABLED=true \
./bin/pleroma_ctl search.indexer create_index
```
=== "From Source"
```sh
PARADEDB_DATABASE_URL='OWNER_URL' mix pleroma.search.indexer create_index
```
The task currently prints an error without returning a failing process status. Verify that it prints `Index created` before continuing.
### Backfill
Run an initial indexing pass to enqueue existing posts:
=== "OTP"
```sh
./bin/pleroma_ctl search.indexer index
```
=== "From Source"
```sh
PARADEDB_DATABASE_URL='RUNTIME_URL' mix pleroma.search.indexer index
```
The command enqueues up to 100,000 Create activities by default. It uses keyset pagination, so new activities arriving during the backfill
do not shift the remaining pages. The defaults and accepted options are:
* `--limit`: maximum activities to enqueue in this run; defaults to `100000` and must be positive
* `--step`: activities fetched from Pleroma's database per page; defaults to `1000` and must be positive
* `--chunk`: jobs inserted into Oban per batch; defaults to `100` and must be positive
* `--before`: resume before the checkpoint activity ID printed by an earlier run; the cursor is exclusive
The checkpoint is the last activity enqueued, not confirmation that its job succeeded. Repeat the command with its last `--before`
checkpoint only when it reports that more activities remain. Inspect failed or discarded Oban jobs as well as waiting jobs before declaring
the backfill complete.
The same operation can be run from an attached console while Pleroma is online:
```elixir
Pleroma.Search.Backfill.run(limit: 10_000)
```
The returned `next_cursor` can be supplied as `before` when `exhausted` is `false`. The attached API also accepts `limit: :infinity`, but this
can create an unbounded number of Oban rows; bounded runs are safer on large instances. An `on_page` callback may be supplied for progress
reporting.
Note: Like the other external search backends, only public/unlisted Notes are indexed.
Set `fuzzy_distance` to `1` or `2` to allow typo-tolerant matching (`2` is the maximum ParadeDB supports).
Indexing runs via the Oban `search_indexing` queue. Its stock configuration starts paused so the search health monitor can control it; verify
that the queue is running before backfilling. A manually managed queue can be enabled with `search_indexing: 10` in the Oban `queues`
configuration instead of a `paused: true` entry.
Monitor that queue until the backfill jobs have drained before evaluating search completeness.
### Rebuilding
Stop Pleroma, or otherwise ensure no search indexing jobs are running, before rebuilding. While ParadeDB is still the configured backend,
run `drop_index`, then `create_index` with the table owner/setup credential. Reapply runtime grants after recreating the table, restart
Pleroma, and begin a new backfill. The DDL tasks print failures without returning a failing process status, so verify the exact `Index
dropped` and `Index created` success messages. Dropping the table removes only derived search data; Pleroma activities remain in the primary
database.
### Rolling back
To retain the ParadeDB table for a later retry, configure `Pleroma.Search.DatabaseSearch` (or the previous external backend) in static config
and restart Pleroma.
Before switching backends, inspect the `search_indexing` queue. Drain or cancel pending backfill jobs, or explicitly accept that they will run
against the newly configured backend after restart.
To remove the table during rollback, first stop Pleroma and run `drop_index` with the table owner/setup credential while ParadeDB is still
configured:
=== "OTP"
```sh
PARADEDB_DATABASE_URL='OWNER_URL' PLEROMA_CTL_RPC_DISABLED=true \
./bin/pleroma_ctl search.indexer drop_index
```
=== "From Source"
```sh
PARADEDB_DATABASE_URL='OWNER_URL' mix pleroma.search.indexer drop_index
```
Verify that the task prints `Index dropped`. Then change the backend in static config and restart. The ParadeDB environment variable may be
removed after removing the ParadeDB stanza or switching to a backend that does not start its Repo. After switching away from ParadeDB, the
generic `drop_index` task targets the newly configured backend and will not remove the ParadeDB table; remove it directly with the database
owner instead.

107
docs/development/API/admin_api.md
View file

@ -80,10 +80,24 @@ The `/api/v1/pleroma/admin/*` path is backwards compatible with `/api/pleroma/ad
{
`nickname`,
`email`,
`password`
`password` (optional; when provided, must not be empty)
}
]
- Response: Array of user objects
- Response: Array of user objects. When a user is created without a `password`, `data`
object contains a `password_reset_link` the user can follow to set their own password.
```json
[
{
"data": {
"nickname": string,
"email": string,
"password_reset_link": string (optional)
},
"type": "success"
}
]
```
## `POST /api/v1/pleroma/admin/users/follow`
@ -1839,3 +1853,92 @@ Note that this differs from the Mastodon API variant: Mastodon API only returns
```json
{}
```
## `GET /api/v1/pleroma/admin/webhooks`
### List webhooks
- Method: `GET`
- Response:
```json
[
{
"enabled": true,
"id": "2",
"events": ["account.created"],
"url": "https://webhook.example/",
"secret": "eb85d4ccd8510e78f912743949dc354e8146987d",
"updated_at": "2022-10-29T17:44:16.000Z",
"created_at": "2022-10-29T17:44:13.000Z"
}
]
```
## `GET /api/v1/pleroma/admin/webhooks/:id`
### Get an individual webhook
- Method: `GET`
- Params:
- `id`: **string** Webhook ID
- Response: A webhook
## `POST /api/v1/pleroma/admin/webhooks`
### Create a webhook
- Method: `POST`
- Params:
- `url`: **string** Webhook URL
- `events`: **[string]** Types of events to trigger on (`account.created`, `report.created`)
- *optional* `enabled`: **boolean** Whether webhook is enabled
- Response: A webhook
## `PATCH /api/v1/pleroma/admin/webhooks/:id`
### Update a webhook
- Method: `PATCH`
- Params:
- `id`: **string** Webhook ID
- *optional* `url`: **string** Webhook URL
- *optional* `events`: **[string]** Types of events to trigger on (`account.created`, `report.created`)
- *optional* `enabled`: **boolean** Whether webhook is enabled
- Response: A webhook
## `DELETE /api/v1/pleroma/admin/webhooks/:id`
### Delete a webhook
- Method: `DELETE`
- Params:
- `id`: **string** Webhook ID
- Response: A webhook
## `POST /api/v1/pleroma/admin/webhooks/:id/enable`
### Activate a webhook
- Method: `POST`
- Params:
- `id`: **string** Webhook ID
- Response: A webhook
## `POST /api/v1/pleroma/admin/webhooks/:id/disable`
### Deactivate a webhook
- Method: `POST`
- Params:
- `id`: **string** Webhook ID
- Response: A webhook
## `POST /api/v1/pleroma/admin/webhooks/:id/rotate_secret`
### Rotate webhook signing secret
- Method: `POST`
- Params:
- `id`: **string** Webhook ID
- Response: A webhook

2
docs/installation/debian_based_en.md
View file

@ -4,7 +4,7 @@
## Installation
This guide will assume you are on Debian 12 (“bookworm”) or later. This guide should also work with Ubuntu 22.04 (“jammy”) and later. It also assumes that you have administrative rights, either as root or a user with [sudo permissions](https://www.digitalocean.com/community/tutorials/how-to-add-delete-and-grant-sudo-privileges-to-users-on-a-debian-vps). If you want to run this guide with root, ignore the `sudo` at the beginning of the lines, unless it calls a user like `sudo -Hu pleroma`; in this case, use `su <username> -s $SHELL -c 'command'` instead.
This guide will assume you are on Debian 13 (“Trixie”) or later. This guide should also work with Ubuntu 26.04 (“resolute”) and later. It also assumes that you have administrative rights, either as root or a user with [sudo permissions](https://www.digitalocean.com/community/tutorials/how-to-add-delete-and-grant-sudo-privileges-to-users-on-a-debian-vps). If you want to run this guide with root, ignore the `sudo` at the beginning of the lines, unless it calls a user like `sudo -Hu pleroma`; in this case, use `su <username> -s $SHELL -c 'command'` instead.
{! backend/installation/generic_dependencies.include !}

2
docs/installation/debian_based_jp.md
View file

@ -14,7 +14,7 @@ Note: This article is potentially outdated because at this time we may not have
- PostgreSQL 11.0以上 (Ubuntu16.04では9.5しか提供されていないので，[](https://www.postgresql.org/download/linux/ubuntu/)こちらから新しいバージョンを入手してください)
- `postgresql-contrib` 11.0以上 (同上)
- Elixir 1.15 以上 ([Debianのリポジトリからインストールしないこと！！！ ここからインストールすること！](https://elixir-lang.org/install.html#unix-and-unix-like)。または [asdf](https://github.com/asdf-vm/asdf) をpleromaユーザーでインストールしてください)
- Elixir 1.17 以上 (Erlang/OTP 27 以上) ([Debianのリポジトリからインストールしないこと！！！ ここからインストールすること！](https://elixir-lang.org/install.html#unix-and-unix-like)。または [asdf](https://github.com/asdf-vm/asdf) をpleromaユーザーでインストールしてください)
- `erlang-dev`
- `erlang-nox`
- `git`

2
docs/installation/erlang_elixir_requirements.include Normal file
View file

@ -0,0 +1,2 @@
* Elixir >=1.17.0, <1.20
* Erlang OTP >=27.0.0 (supported: <29)

7
docs/installation/generic_dependencies.include
View file

@ -1,8 +1,7 @@
## Required dependencies
{! backend/installation/erlang_elixir_requirements.include !}
* PostgreSQL >=11.0
* Elixir >=1.15.0 <1.19
* Erlang OTP >=23.0.0 (supported: <28)
* git
* file / libmagic
* gcc or clang
@ -14,3 +13,7 @@
* ImageMagick
* FFmpeg
* exiftool
!!!info
**Source installs only**
If your operating system repositories don't meet the Elixir/Erlang requirements, check [Installing newer versions of Elixir and Erlang](./installing_newer_elixir_erlang.md).

333
docs/installation/installing_newer_elixir_erlang.md Normal file
View file

@ -0,0 +1,333 @@
# Installing newer versions of Elixir and Erlang
This guide is for installing newer versions of Elixir and/or Erlang on your system when installing Pleroma from source and your operating system repositories don't meet the minimum requirements listed below.
This guide assumes you are using Debian or Ubuntu.
**Using the system repositories is recommended if possible.**
**If a newer release of your operating system would meet the requirements, consider upgrading instead.**
Shell commands prepended with `#` in this page are supposed to be ran with root privileges, commands prepended with `$` are supposed to be ran as the Pleroma user unless noted otherwise.
!!!warning
If you are, or will be, following the source install guide, make sure to switch to the Pleroma user before running any of the `mix` commands using `sudo -Hu pleroma /bin/bash`.
Not doing so will result in `command not found` errors.
!!!info
Most of this guide can be used to install Elixir and/or Erlang on operating systems that don't package either one.
Installing Pleroma on such systems is largely untested.
## Requirements
{! backend/installation/erlang_elixir_requirements.include !}
## Install methods
Choose one of the following install methods:
* User level:
* mise (Erlang and Elixir install methods are built-in[^1])
* asdf (plugin-based, plugins are shells scripts; builds Erlang from source)
* pkgsrc (for advanced users, harder to install and maintain; builds Elixir and Erlang from source)
* System-wide:
* pkgsrc (for advanced users, harder to install and maintain; builds Elixir and Erlang from source)
[^1]: On Ubuntu downloads binaries both for Elixir and Erlang. On Debian builds Erlang from source and downloads binaries for Elixir.
!!!warning
**Ignore if using mise on Ubuntu.**
You are about to compile Erlang on your server.
If you are planning to run Pleroma on a VPS virtual machine, make sure you have enough disk space and system resources available.
At least 2GB of RAM and 8GB of extra free disk space is required to continue in this guide.
If your system does not meet these requirements, use [OTP install](./otp_en.md).
**This will take a considerable amount of time.**
!!!warning
Before proceeding, stop Pleroma if already running and uninstall both Elixir and Erlang.
Keeping Elixir will likely result in a non-functional Elixir install and keeping Erlang can result in various failures to compile Pleroma and its dependencies.
## mise
To start, create the Pleroma user according to the [source install guide](./debian_based_en.md#install-pleromabe).
### Debian-only preparation
If you are using Ubuntu, skip to the [next section](./installing_newer_elixir_erlang.md#elixirerlang-installation).
Install dependencies:
```
# apt install build-essential libssl-dev libncurses-dev git unzip
```
Add Erlang autotools configure options to Pleroma user's shell config (`.bashrc` for bash):
```sh
export KERL_CONFIGURE_OPTIONS="--without-javac --without-odbc --without-wx --without-ssh"
```
### Elixir/Erlang installation
Install curl (for the mise installer):
```
# apt install curl
```
Switch to the Pleroma user using `sudo -Hu pleroma /bin/bash` and follow the install steps from mise documentation and install it for the user Pleroma: [https://mise.jdx.dev/getting-started.html](https://mise.jdx.dev/getting-started.html)
Add the following to activate mise in the Pleroma user's shell configuration (`.bashrc` for bash):
```sh
# Modify to match the mise installation location
eval "$($HOME/.local/bin/mise activate bash)"
```
Source new profile settings:
```sh
$ source ~/.bashrc
```
If installing Pleroma, follow the [Pleroma source installation instructions](./debian_based_en.md) up to the point before downloading mix dependencies for Pleroma, ignoring the instructions to download Elixir/Erlang.
Install Erlang (replace `<version>` with a supported release meeting Pleroma's requirements, OTP 27 is recommended):
```sh
$ mise install erlang@<version>
```
!!!info
To get a list of available versions, use `mise ls-remote <language/tool>`.
Then run the following in the Pleroma source code repository:
```sh
$ mise use erlang@<version>
```
Install Elixir (replace `<version>` with a supported release and `<OTP version>` with the mise-installed Erlang/OTP version, Elixir 1.18 is recommended):
```sh
$ mise install elixir@<version>-otp-<OTP version>
```
Then run the following in the Pleroma source code directory:
```sh
$ mise use elixir@<Elixir version>
```
If installing Pleroma, continue with the source install guide up to the point of installing the systemd Pleroma service.
Don't attempt to start the Pleroma service yet.
Create a `pleroma.sh` startup script owned by the Pleroma user in Pleroma user's home directory:
```sh
#!/bin/bash
# Modify to match the mise installation location
eval "$($HOME/.local/bin/mise activate bash)"
exec mix phx.server
```
Make the startup script executable:
```sh
chmod u+x pleroma.sh
```
Override the Pleroma service file to point to the new start up script, run `# systemctl edit pleroma` and add the following:
```ini
[Service]
ExecStart=
ExecStart=/var/lib/pleroma/pleroma.sh
```
You can now start the Pleroma service.
If you are installing Pleroma, continue with creating your first user according to the install guide.
## asdf
To start, create the Pleroma user according to the [source install guide](./debian_based_en.md#install-pleromabe) and switch to the user using `sudo -Hu pleroma /bin/bash`.
Then follow the install and configure steps from asdf documentation and install it for the user Pleroma: [https://asdf-vm.com/guide/getting-started.html](https://asdf-vm.com/guide/getting-started.html)
Install dependencies:
```
# apt install build-essential libssl-dev libncurses-dev git unzip curl
```
Add Erlang autotools configure options to Pleroma user's shell config (`.bashrc` for bash):
```sh
export KERL_CONFIGURE_OPTIONS="--without-javac --without-odbc --without-wx --without-ssh"
```
Source new profile settings:
```sh
$ source ~/.bashrc
```
Install the Erlang plugin:
```sh
$ asdf plugin add erlang https://github.com/asdf-vm/asdf-erlang.git
```
Build and install Erlang (replace `<version>` with a supported release meeting Pleroma's requirements, OTP 27 is recommended):
```sh
$ asdf install erlang <version>
```
!!!info
To get a list of available versions, use `asdf list all <plugin>`.
Install the Elixir plugin:
```sh
$ asdf plugin add elixir https://github.com/asdf-vm/asdf-elixir.git
```
Install Elixir (replace `<version>` with a supported release and `<OTP version>` with the asdf-installed Erlang/OTP version, Elixir 1.18 is recommended):
```sh
$ asdf install elixir <version>-otp-<OTP version>
```
If installing Pleroma, follow the [Pleroma source installation instructions](./debian_based_en.md) up to the point before downloading mix dependencies for Pleroma, ignoring the instructions to download Elixir/Erlang.
Then run the following in the Pleroma source code directory:
```sh
$ asdf set erlang <OTP version>
$ asdf set elixir <Elixir version>
```
If installing Pleroma, continue with the source install guide up to the point of installing the systemd Pleroma service.
Don't attempt to start the Pleroma service yet.
Create a `pleroma.sh` startup script owned by the Pleroma user in Pleroma user's home directory:
```sh
#!/bin/sh
# Modify to match your .bashrc settings used when installing asdf
export ASDF_DATA_DIR="/your/custom/data/dir"
export PATH="${ASDF_DATA_DIR:-$HOME/.asdf}/shims:$PATH"
exec mix phx.server
```
Make the startup script executable:
```sh
chmod u+x pleroma.sh
```
Override the Pleroma service file to point to the new start up script, run `# systemctl edit pleroma` and add the following:
```ini
[Service]
ExecStart=
ExecStart=/var/lib/pleroma/pleroma.sh
```
You can now start the Pleroma service.
If you are installing Pleroma, continue with creating your first user according to the install guide.
## pkgsrc
pkgsrc is NetBSD's portable package building framework also supporting Linux.
Its use is discouraged for novice users as it is harder to maintain and keep up-to-date compared to other options in this guide.
Before installing pkgsrc, choose which install method (tarball or CVS) you want to use.
Both update differently and have their caveats, check NetBSD documentation for more info: [https://www.netbsd.org/docs/pkgsrc/getting.html#uptodate](https://www.netbsd.org/docs/pkgsrc/getting.html#uptodate)
Using the stable branch is recommended.
!!!warning
For CVS install, `cvs` might not be installed on your system, in which case it can be installed with `# apt install cvs`.
pkgsrc supports both user-level and system-wide package installation, hence why most commands below don't have a `#`/`$` prepended to them.
For user-level installs, use the Pleroma user created in the [source install guide](./debian_based_en.md#install-pleromabe).
To start, follow the `quickstart -> install source packages` portion of the guide to acquire pkgsrc: [https://pkgsrc.org](https://pkgsrc.org)
### Bootstrap
Install build dependencies for bootstrap:
```
# apt install build-essential
```
For user-level installation run:
```sh
$ cd pkgsrc/bootstrap
$ ./bootstrap --prefer-pkgsrc yes --unprivileged --make-jobs <number of CPU cores - 1>
```
For system-wide installation run:
```
# cd pkgsrc/bootstrap
# ./bootstrap --prefer-pkgsrc yes --make-jobs <number of CPU cores - 1>
```
!!!info
For more bootstrap options, run `./bootstrap --help`
Add pgksrc packages to `$PATH` in shell configuration (`.bashrc` for bash), default prefix is `$HOME/pkg` for user-level install and `/usr/pkg` for system-wide install:
```sh
export PATH="$PATH:<prefix for pkgsrc install>/bin"
```
Source new profile settings:
```sh
source ~/.bashrc
```
!!!info
For system-wide installs, both the root user and Pleroma user need to have the `PATH` set accordingly.
### Package installation
Erlang:
```sh
cd pkgsrc/lang/erlang
bmake install clean
```
Elixir:
```sh
cd ../elixir
bmake install clean
```
### Pleroma startup script
If installing Pleroma, first follow the [Pleroma source installation instructions](./debian_based_en.md) up to the point of installing the systemd Pleroma service, ignoring the instruction to install Elixir/Erlang. Don't attempt to start the service yet.
Create a `pleroma.sh` startup script owned by the Pleroma user in Pleroma user's home directory:
```sh
#!/bin/sh
# Modify to match your .bashrc settings used when bootsrapping pkgsrc
export PATH="$PATH:<prefix for pkgsrc install>/bin"
exec mix phx.server
```
Make the startup script executable:
```sh
chmod u+x pleroma.sh
```
Override the Pleroma service file to point to the new start up script, run `# systemctl edit pleroma` and add the following:
```ini
[Service]
ExecStart=
ExecStart=/var/lib/pleroma/pleroma.sh
```
You can now start the Pleroma service.
If you are installing Pleroma, continue with creating your first user according to the install guide.
## Maintenance
Since Elixir and Erlang is no longer managed by the system package manager after following this guide, it is recommended to periodically check their respective websites/repositories for newer versions with security bug fixes.
When updating Elixir, Erlang or both, stop Pleroma before proceeding with the update.
Pleroma will likely need to be recompiled after the update by running the following command in the Pleroma source code directory as the Pleroma user:
```sh
MIX_ENV=prod mix compile
```
If Pleroma fails to compile or start after upgrading Elixir/Erlang, run the following as a troubleshooting step in the Pleroma source code directory as the Pleroma user:
```sh
mix clean
mix deps.clean --all
mix deps.get
MIX_ENV=prod mix compile
```
Then try to start Pleroma again. If that doesn't help, join the Matrix room or IRC channel mentioned below, or make an issue on [git.pleroma.social/pleroma/pleroma](https://git.pleroma.social/pleroma/pleroma/issues/new/choose) explaining the problem.
## Questions
Questions about the guide or it didn’t work as it should, ask in [#pleroma:libera.chat](https://matrix.to/#/#pleroma:libera.chat) via Matrix or **#pleroma** on **libera.chat** via IRC.
<!-- vim ts=4:sw=4 -->

130
docs/installation/migrating_from_rebased.md Normal file
View file

@ -0,0 +1,130 @@
# Migrating from Rebased
Rebased and Pleroma currently have compatible database schemas, so removing
Rebased-specific database objects is optional. The cleanup described below is
destructive and can take a long time on large databases.
This guide requires a Pleroma release that contains this document and native
webhook support. If you are reading the development version of this guide
before those changes reach `stable`, wait for the corresponding stable release.
## Prepare and back up
Plan a maintenance window and read the [backup guide](../administration/backup.md)
before starting.
Stop Rebased and confirm it is stopped before making the backup. Omit the
backup guide's final restart step and leave Rebased stopped:
```bash
sudo systemctl stop pleroma
sudo systemctl is-active pleroma
```
Make a verified database backup and a copy of the Rebased installation,
including its configuration, uploads, and `instance` directory. Also save the
current systemd unit if you plan to replace it.
## Install Pleroma
Do not use `git pull --rebase` to change an existing Rebased checkout into a
Pleroma checkout. The forks use different branches, and rebasing can replay
Rebased-specific commits over Pleroma. Move the old checkout aside and clone a
clean copy of Pleroma instead. The paths below assume the default source
installation in `/opt/pleroma`:
```bash
sudo mv /opt/pleroma /opt/pleroma.rebased
sudo mkdir /opt/pleroma
sudo chown pleroma:pleroma /opt/pleroma
sudo -Hu pleroma git clone -b stable https://git.pleroma.social/pleroma/pleroma /opt/pleroma
```
Restore the instance-specific files from the old checkout. This usually
includes `config/prod.secret.exs`, `config/prod.exported_from_db.secret.exs` if
present, `uploads`, and `instance`. Diff and restore any other non-stock
configuration files, preserve their ownership, and review them before starting
Pleroma.
Pleroma requires Elixir 1.17 or newer and Erlang/OTP 27 or newer. Install supported Erlang and Elixir
versions as described in the installation guide for your distribution before
running any Mix command. Do not reuse Rebased's `.tool-versions`. If you keep
using [`asdf`](https://asdf-vm.com/), select supported versions in the
`pleroma` user's home and use the asdf Mix shim for every command below.
Set `MIX` to the appropriate binary, verify the selected runtime, then install
the dependencies and compile Pleroma:
```bash
cd /opt/pleroma
MIX=/usr/bin/mix
# If using asdf instead:
# MIX=/var/lib/pleroma/.asdf/shims/mix
sudo -Hu pleroma "$MIX" --version
sudo -Hu pleroma MIX_ENV=prod "$MIX" deps.get
sudo -Hu pleroma MIX_ENV=prod "$MIX" compile
```
Compare your existing systemd unit with `installation/pleroma.service`. If you
replace it, review its user, paths, and environment first. Reload systemd after
making changes:
```bash
sudo systemctl daemon-reload
```
Rebased does not bundle a frontend, so `instance/static` may contain an
installed frontend. Pleroma includes a bundled frontend. You can remove the
separately installed frontend after the migration, but do not remove other
instance files such as custom emojis.
## Database migration
Rebased's extra columns, index, and enum labels do not prevent Pleroma from
starting. Existing notifications with Rebased-only event types are not
supported by Pleroma, however, and can cause notification API errors. Run the
cleanup below, or archive and remove those notification rows separately before
starting Pleroma. Skipping the cleanup preserves them if you need to restore
Rebased.
Pleroma supports Rebased's webhook schema. The cleanup does not roll back the
shared webhook migrations, so webhook configuration is preserved.
To perform the optional cleanup, run the rollback while the service is stopped:
```bash
MIX=/usr/bin/mix
# If using asdf instead:
# MIX=/var/lib/pleroma/.asdf/shims/mix
sudo -Hu pleroma MIX_ENV=prod "$MIX" ecto.rollback --migrations-path priv/repo/optional_migrations/rebased_rollbacks --all
```
This deletes the Rebased-only `accepts_email_list`, `location`, and
`last_move_at` user fields and the `pleroma:participation_accepted`,
`pleroma:participation_request`, `pleroma:event_reminder`, and
`pleroma:event_update` notifications. It also rewrites the notifications table
and can require substantial time, free disk space, and write-ahead log capacity
on a large instance.
Apply all pending Pleroma migrations:
```bash
sudo -Hu pleroma MIX_ENV=prod "$MIX" ecto.migrate
```
## Start and verify Pleroma
Start Pleroma and inspect its status and logs:
```bash
sudo systemctl start pleroma
sudo systemctl status pleroma
sudo journalctl -u pleroma -n 100 --no-pager
```
Verify that the instance, uploads, frontend, and webhook configuration work as
expected before deleting `/opt/pleroma.rebased` or the database backup. If the
migration fails, keep the service stopped and restore both the database backup
and the previous checkout before restarting Rebased. If you changed the
systemd unit or runtime setup, restore those too and run
`systemctl daemon-reload` before restarting the service.

52
docs/installation/migrating_from_source_otp_en.md
View file

@ -76,35 +76,51 @@ $EDITOR /etc/pleroma/config.exs
```
## Installing the release
Before proceeding, get the flavour from [Detecting flavour](otp_en.md#detecting-flavour) section in OTP installation guide.
Delete all files in pleroma user's directory:
```sh
# Delete all files in pleroma user's directory
rm -r ~pleroma/*
```
# Set the flavour environment variable to the string you got in Detecting flavour section.
# For example if the flavour is `amd64-musl` the command will be
Set the flavour environment variable to the string you got in Detecting flavour section.
For example if the flavour is `amd64-musl` the command will be:
```sh
export FLAVOUR="amd64-musl"
```
# Clone the release build into a temporary directory and unpack it
# Replace `stable` with `unstable` if you want to run the unstable branch
sudo -Hu pleroma "
curl 'https://git.pleroma.social/api/v4/projects/2/jobs/artifacts/stable/download?job=$FLAVOUR' -o /tmp/pleroma.zip
unzip /tmp/pleroma.zip -d /tmp/
"
Clone the release build into a temporary directory and unpack it.
Replace `stable` with `develop`, if you want to run the develop branch.
# Move the release to the home directory and delete temporary files
sudo -Hu pleroma "
mv /tmp/release/* ~pleroma/
rmdir /tmp/release
rm /tmp/pleroma.zip
"
```sh
sudo -Hu pleroma curl "https://git.pleroma.social/api/packages/pleroma/generic/pleroma-otp-stable-$FLAVOUR/latest/pleroma.zip" -o /tmp/pleroma.zip
sudo -Hu pleroma unzip /tmp/pleroma.zip -d /tmp/
```
# Start the instance to verify that everything is working as expected
Move the release to the home directory and delete temporary files:
```sh
sudo -Hu pleroma mv /tmp/release/* ~pleroma/
sudo -Hu pleroma rmdir /tmp/release
sudo -Hu pleroma rm /tmp/pleroma.zip
```
Start the instance to verify that everything is working as expected:
```sh
sudo -Hu pleroma "./bin/pleroma daemon"
```
# Wait for about 20 seconds and query the instance endpoint, if it shows your uri, name and email correctly, you are configured correctly
Wait for about 20 seconds and query the instance endpoint, if it shows your uri, name and email correctly, you are configured correctly:
```sh
sleep 20 && curl http://localhost:4000/api/v1/instance
```
# Stop the instance
Stop the instance:
```sh
sudo -Hu pleroma "./bin/pleroma stop"
```

8
docs/installation/openbsd_en.md
View file

@ -67,13 +67,19 @@ Pleroma will be run by a dedicated \_pleroma user. Before creating it, insert th
```
pleroma:\
:datasize=1536M:\
:datasize=4096M:\
:openfiles-max=4096:\
:openfiles-cur=1024:\
:setenv=LC_ALL=en_US.UTF-8,VIX_COMPILATION_MODE=PLATFORM_PROVIDED_LIBVIPS,MIX_ENV=prod:\
:tc=daemon:
```
Then run:
```
# cap_mkdb /etc/login.conf
```
This creates a "pleroma" login class and sets higher values than default for datasize and openfiles (see [login.conf(5)](https://man.openbsd.org/login.conf)), this is required to avoid having Pleroma crash some time after starting.
Create the \_pleroma user, assign it the pleroma login class and create its home directory (/home/\_pleroma/):

113
docs/installation/otp_en.md
View file

@ -32,6 +32,7 @@ Other than things bundled in the OTP release Pleroma depends on:
* curl (to download the release build)
* unzip (needed to unpack release builds)
* ncurses (ERTS won't run without it)
* libvips (for image processing)
* PostgreSQL (also utilizes extensions in postgresql-contrib)
* nginx (could be swapped with another reverse proxy but this guide covers only it)
* certbot (for Let's Encrypt certificates, could be swapped with another ACME client, but this guide covers only it)
@ -46,7 +47,13 @@ Other than things bundled in the OTP release Pleroma depends on:
=== "Debian/Ubuntu"
```
apt install curl unzip libncurses5 postgresql postgresql-contrib nginx certbot libmagic-dev
apt install curl unzip libncurses6 libvips42t64 postgresql postgresql-contrib nginx certbot libmagic-dev
```
On Debian 12 and Ubuntu 22.04, replace `libvips42t64` with `libvips42`:
```
apt install curl unzip libncurses6 libvips42 postgresql postgresql-contrib nginx certbot libmagic-dev
```
### Installing optional packages
@ -88,9 +95,23 @@ RUM indexes are an alternative indexing scheme that is not included in PostgreSQ
```
=== "Debian/Ubuntu"
Install the package matching your distribution's PostgreSQL version:
```
# Available only on Buster/19.04
apt install postgresql-11-rum
# Debian 13
apt install postgresql-17-rum
# Debian 12
apt install postgresql-15-rum
# Ubuntu 26.04
apt install postgresql-18-rum
# Ubuntu 24.04
apt install postgresql-16-rum
# Ubuntu 22.04
apt install postgresql-14-rum
```
#### (Optional) Performance configuration
@ -109,60 +130,90 @@ Restart PostgreSQL to apply configuration changes:
```
### Installing Pleroma
Create a Pleroma user:
```sh
# Create a Pleroma user
adduser --system --shell /bin/false --home /opt/pleroma pleroma
```
# Set the flavour environment variable to the string you got in Detecting flavour section.
# For example if the flavour is `amd64-musl` the command will be
Set the flavour environment variable to the string you got in Detecting flavour section. For example if the flavour is `amd64-musl` the command will be:
```sh
export FLAVOUR="amd64-musl"
```
# Clone the release build into a temporary directory and unpack it
sudo -Hu pleroma "
curl 'https://git.pleroma.social/api/v4/projects/2/jobs/artifacts/stable/download?job=$FLAVOUR' -o /tmp/pleroma.zip
unzip /tmp/pleroma.zip -d /tmp/
"
Clone the release build into a temporary directory and unpack it:
# Move the release to the home directory and delete temporary files
sudo -Hu pleroma "
mv /tmp/release/* /opt/pleroma
rmdir /tmp/release
rm /tmp/pleroma.zip
"
# Create uploads directory and set proper permissions (skip if planning to use a remote uploader)
# Note: It does not have to be `/var/lib/pleroma/uploads`, the config generator will ask about the upload directory later
```sh
sudo -Hu pleroma curl "https://git.pleroma.social/api/packages/pleroma/generic/pleroma-otp-stable-$FLAVOUR/latest/pleroma.zip" -o /tmp/pleroma.zip
sudo -Hu pleroma unzip /tmp/pleroma.zip -d /tmp/
```
Move the release to the home directory and delete temporary files:
```sh
sudo -Hu pleroma mv /tmp/release/* /opt/pleroma
sudo -Hu pleroma rmdir /tmp/release
sudo -Hu pleroma rm /tmp/pleroma.zip
```
Create uploads directory and set proper permissions (skip if planning to use a remote uploader).
Note: It does not have to be `/var/lib/pleroma/uploads`, the config generator will ask about the upload directory later.
```sh
mkdir -p /var/lib/pleroma/uploads
chown -R pleroma /var/lib/pleroma
```
# Create custom public files directory (custom emojis, frontend bundle overrides, robots.txt, etc.)
# Note: It does not have to be `/var/lib/pleroma/static`, the config generator will ask about the custom public files directory later
Create custom public files directory (custom emojis, frontend bundle overrides, robots.txt, etc.)
Note: It does not have to be `/var/lib/pleroma/static`, the config generator will ask about the custom public files directory later.
```sh
mkdir -p /var/lib/pleroma/static
chown -R pleroma /var/lib/pleroma
```
# Create a config directory
Create a config directory:
```sh
mkdir -p /etc/pleroma
chown -R pleroma /etc/pleroma
```
# Run the config generator
Run the config generator:
```sh
sudo -Hu pleroma "./bin/pleroma_ctl instance gen --output /etc/pleroma/config.exs --output-psql /tmp/setup_db.psql"
```
# Create the postgres database
Create the postgres database and schema:
```sh
sudo -u postgres -s $SHELL -lc "psql -f /tmp/setup_db.psql"
# Create the database schema
sudo -Hu pleroma "./bin/pleroma_ctl migrate"
```
# If you have installed RUM indexes uncommend and run
# sudo -Hu pleroma "./bin/pleroma_ctl migrate --migrations-path priv/repo/optional_migrations/rum_indexing/"
(Optional) If you have installed RUM indexes:
# Start the instance to verify that everything is working as expected
```sh
sudo -Hu pleroma "./bin/pleroma_ctl migrate --migrations-path priv/repo/optional_migrations/rum_indexing/"
```
Start the instance to verify that everything is working as expected:
```sh
sudo -Hu pleroma "./bin/pleroma daemon"
```
# Wait for about 20 seconds and query the instance endpoint, if it shows your uri, name and email correctly, you are configured correctly
Wait for about 20 seconds and query the instance endpoint, if it shows your uri, name and email correctly, you are configured correctly:
```sh
sleep 20 && curl http://localhost:4000/api/v1/instance
```
# Stop the instance
Stop the instance:
```sh
sudo -Hu pleroma "./bin/pleroma stop"
```

7
installation/init.d/pleroma
View file

@ -37,6 +37,13 @@ depend() {
need nginx postgresql
}
start_pre() {
ebegin "running migrations if any"
su -s /bin/sh -c "cd \"${directory}\" && exec /usr/bin/mix ecto.migrate" \
"${command_user%%:*}"
eend "$?"
}
healthcheck() {
# put pleroma_health=YES in /etc/conf.d/pleroma if you want healthchecking
# and make sure you have curl installed

13
lib/mix/pleroma.ex
View file

@ -62,7 +62,8 @@ defmodule Mix.Pleroma do
{Majic.Pool,
[name: Pleroma.MajicPool, pool_size: Pleroma.Config.get([:majic_pool, :size], 2)]}
] ++
http_children(adapter)
http_children(adapter) ++
search_children()
cachex_children = Enum.map(@cachex_children, &Pleroma.Application.build_cachex(&1, []))
@ -141,4 +142,14 @@ defmodule Mix.Pleroma do
end
defp http_children(_), do: []
defp search_children do
case Pleroma.Config.get([Pleroma.Search, :module]) do
Pleroma.Search.ParadeDB ->
[Pleroma.Search.ParadeDB.Repo]
_ ->
[]
end
end
end

105
lib/mix/tasks/pleroma/config.ex
View file

@ -252,40 +252,43 @@ defmodule Mix.Tasks.Pleroma.Config do
if whitelisted_configs in [nil, false] do
shell_error("No unwanted settings in ConfigDB. No changes made.")
else
whitelisted_groups =
whitelisted_configs
|> Enum.filter(fn
{_group} -> true
_ -> false
end)
|> Enum.map(fn {group} -> group end)
whitelisted_keys =
whitelisted_configs
|> Enum.filter(fn
{_group, _key} -> true
_ -> false
end)
{whitelisted_groups, whitelisted_keys} = get_filters(whitelisted_configs)
filtered =
from(c in ConfigDB)
|> Repo.all()
|> Enum.filter(&not_whitelisted?(&1, whitelisted_groups, whitelisted_keys))
if not Enum.empty?(filtered) do
shell_info("The following settings will be removed from ConfigDB:\n")
Enum.each(filtered, &dump(&1))
do_config_filter(filtered, force)
end
end
if force or shell_prompt("Are you sure you want to continue?", "n") in ~w(Yn Y y) do
filtered_ids = Enum.map(filtered, fn %{id: id} -> id end)
# Removes blacklisted configuration sections
def run(["filter_blacklisted" | rest]) do
{options, [], []} =
OptionParser.parse(
rest,
strict: [force: :boolean],
aliases: [f: :force]
)
Repo.delete_all(from(c in ConfigDB, where: c.id in ^filtered_ids))
else
shell_error("No changes made.")
end
else
shell_error("No unwanted settings in ConfigDB. No changes made.")
end
force = Keyword.get(options, :force, false)
start_pleroma()
blacklisted_configs = Pleroma.Config.get(:database_config_blacklist)
if blacklisted_configs in [nil, false] do
shell_error("No unwanted settings in ConfigDB. No changes made.")
else
{blacklisted_groups, blacklisted_keys} = get_filters(blacklisted_configs)
filtered =
from(c in ConfigDB)
|> Repo.all()
|> Enum.filter(&blacklisted?(&1, blacklisted_groups, blacklisted_keys))
do_config_filter(filtered, force)
end
end
@ -332,9 +335,12 @@ defmodule Mix.Tasks.Pleroma.Config do
group
|> Pleroma.Config.Loader.filter_group(settings)
|> Enum.each(fn {key, value} ->
{:ok, _} = ConfigDB.update_or_create(%{group: group, key: key, value: value})
shell_info("Settings for key #{key} migrated.")
if ConfigDB.static?(group, key) do
shell_info("Settings for key #{key} must remain in static configuration; skipping.")
else
{:ok, _} = ConfigDB.update_or_create(%{group: group, key: key, value: value})
shell_info("Settings for key #{key} migrated.")
end
end)
shell_info("Settings for group #{inspect(group)} migrated.")
@ -490,8 +496,49 @@ defmodule Mix.Tasks.Pleroma.Config do
Ecto.Adapters.SQL.query!(Repo, "ALTER SEQUENCE config_id_seq RESTART;")
end
defp get_filters(config) do
groups =
config
|> Enum.filter(fn
{_group} -> true
_ -> false
end)
|> Enum.map(fn {group} -> group end)
keys =
config
|> Enum.filter(fn
{_group, _key} -> true
_ -> false
end)
{groups, keys}
end
defp not_whitelisted?(%{group: group, key: key}, whitelisted_groups, whitelisted_keys) do
not Enum.member?(whitelisted_groups, group) and
not Enum.member?(whitelisted_keys, {group, key})
end
defp blacklisted?(%{group: group, key: key}, blacklisted_groups, blacklisted_keys) do
Enum.member?(blacklisted_groups, group) or
Enum.member?(blacklisted_keys, {group, key})
end
defp do_config_filter(filtered, _) when length(filtered) == 0 do
shell_error("No unwanted settings in ConfigDB. No changes made.")
end
defp do_config_filter(filtered, force) do
shell_info("The following settings will be removed from ConfigDB:\n")
Enum.each(filtered, &dump(&1))
if force or shell_prompt("Are you sure you want to continue?", "n") in ~w(Yn Y y) do
filtered_ids = Enum.map(filtered, fn %{id: id} -> id end)
Repo.delete_all(from(c in ConfigDB, where: c.id in ^filtered_ids))
else
shell_error("No changes made.")
end
end
end

58
lib/mix/tasks/pleroma/search/indexer.ex
View file

@ -4,9 +4,6 @@
defmodule Mix.Tasks.Pleroma.Search.Indexer do
import Mix.Pleroma
import Ecto.Query
alias Pleroma.Workers.SearchIndexingWorker
def run(["create_index"]) do
start_pleroma()
@ -35,49 +32,30 @@ defmodule Mix.Tasks.Pleroma.Search.Indexer do
strict: [
chunk: :integer,
limit: :integer,
step: :integer
step: :integer,
before: :string
]
)
start_pleroma()
chunk_size = Keyword.get(options, :chunk, 100)
limit = Keyword.get(options, :limit, 100_000)
per_step = Keyword.get(options, :step, 1000)
result =
Pleroma.Search.Backfill.run(
options ++
[
on_page: fn page ->
IO.puts(
"Queued #{page.count} activities (#{page.enqueued} total), " <>
"checkpoint: #{page.next_cursor}"
)
end
]
)
chunks = max(div(limit, per_step), 1)
IO.puts("Queued #{result.enqueued} activities")
1..chunks
|> Enum.each(fn step ->
q =
from(a in Pleroma.Activity,
limit: ^per_step,
offset: ^per_step * (^step - 1),
select: [:id],
order_by: [desc: :id]
)
{:ok, ids} =
Pleroma.Repo.transaction(fn ->
Pleroma.Repo.stream(q, timeout: :infinity)
|> Enum.map(fn a ->
a.id
end)
end)
IO.puts("Got #{length(ids)} activities, adding to indexer")
ids
|> Enum.chunk_every(chunk_size)
|> Enum.each(fn chunk ->
IO.puts("Adding #{length(chunk)} activities to indexing queue")
chunk
|> Enum.map(fn id ->
SearchIndexingWorker.new(%{"op" => "add_to_index", "activity" => id})
end)
|> Oban.insert_all()
end)
end)
unless result.exhausted do
IO.puts("Resume with --before #{result.next_cursor}")
end
end
end

75
lib/pleroma/application.ex
View file

@ -43,6 +43,7 @@ defmodule Pleroma.Application do
# every time the application is restarted, so we disable module
# conflicts at runtime
Code.compiler_options(ignore_module_conflict: true)
configure_logger()
Pleroma.Telemetry.Logger.attach()
Config.Holder.save_default()
Pleroma.HTML.compile_scrubbers()
@ -65,6 +66,7 @@ defmodule Pleroma.Application do
if match?({Tesla.Adapter.Finch, _}, adapter) do
Logger.info("Starting Finch")
use_castore_for_otp_cacerts()
Finch.start_link(name: MyFinch)
end
@ -80,12 +82,15 @@ defmodule Pleroma.Application do
[
Pleroma.PromEx,
Pleroma.LDAP,
Pleroma.Repo,
Config.TransferTask,
Pleroma.Emoji,
Pleroma.Web.Plugs.RateLimiter.Supervisor,
{Task.Supervisor, name: Pleroma.TaskSupervisor}
Pleroma.Repo
] ++
search_children() ++
[
Config.TransferTask,
Pleroma.Emoji,
Pleroma.Web.Plugs.RateLimiter.Supervisor,
{Task.Supervisor, name: Pleroma.TaskSupervisor}
] ++
cachex_children() ++
http_children(adapter) ++
[
@ -113,6 +118,44 @@ defmodule Pleroma.Application do
Supervisor.start_link(children, opts)
end
def configure_logger do
Config.get([:logger, :backends], [])
|> Enum.each(fn backend ->
case backend_to_logger(backend) do
{:ok, logger} ->
add_logger(logger)
{:error, :console_backend} ->
Logger.warning(":console is no longer considered a backend and is enabled by default")
end
end)
end
defp add_logger(backend) do
case LoggerBackends.add(backend) do
{:ok, _} ->
Logger.debug("Successfully added logger backend: #{inspect(backend)}")
{:error, :already_present} ->
:ok
{:error, reason} ->
Logger.error("Failed to add logger backend #{inspect(backend)}: #{inspect(reason)}")
end
end
defp backend_to_logger(:console), do: {:error, :console_backend}
defp backend_to_logger({:ex_syslogger = backend, name}) do
Logger.warning(
"Configuration {:#{backend}, :#{name}} is incorrect. Use {ExSyslogger, :#{name}} instead!"
)
{:ok, {ExSyslogger, name}}
end
defp backend_to_logger(backend), do: {:ok, backend}
def load_custom_modules do
dir = Config.get([:modules, :runtime_dir])
@ -159,6 +202,16 @@ defmodule Pleroma.Application do
]
end
defp search_children do
case Config.get([Pleroma.Search, :module]) do
Pleroma.Search.ParadeDB ->
[Pleroma.Search.ParadeDB.Repo]
_ ->
[]
end
end
defp emoji_packs_expiration,
do: expiration(default: :timer.seconds(5 * 60), interval: :timer.seconds(60))
@ -252,6 +305,15 @@ defmodule Pleroma.Application do
end
end
# Mint trusts the OTP CA store, which comes from the OS. Make it the CAStore
# bundle that the other adapters trust. OTP 24 has no such store and Mint
# uses CAStore there anyway.
defp use_castore_for_otp_cacerts do
if Code.ensure_loaded?(:public_key) and function_exported?(:public_key, :cacerts_load, 1) do
:ok = :public_key.cacerts_load(CAStore.file_path())
end
end
defp http_children_hackney do
pools = [:federation, :media]
@ -278,7 +340,8 @@ defmodule Pleroma.Application do
config = Config.get(ConcurrentLimiter, [])
[
Pleroma.Search
Pleroma.Search,
Pleroma.Webhook.Notify
]
|> Enum.each(fn module ->
mod_config = Keyword.get(config, module, [])

2
lib/pleroma/chat.ex
View file

@ -108,4 +108,6 @@ defmodule Pleroma.Chat do
|> cast(%{pinned: false}, [:pinned])
|> Repo.update()
end
def enabled?, do: Pleroma.Config.get([__MODULE__, :enabled], true)
end

73
lib/pleroma/config/deprecation_warnings.ex
View file

@ -20,6 +20,8 @@ defmodule Pleroma.Config.DeprecationWarnings do
"\n* `config :pleroma, :instance, mrf_transparency_exclusions` is now `config :pleroma, :mrf, transparency_exclusions`"}
]
@logger_formatter_config_knobs [:colors, :format, :metadata, :truncate, :utc_log]
def check_exiftool_filter do
filters = Config.get([Pleroma.Upload]) |> Keyword.get(:filters, [])
@ -218,7 +220,8 @@ defmodule Pleroma.Config.DeprecationWarnings do
check_quarantined_instances_tuples(),
check_transparency_exclusions_tuples(),
check_simple_policy_tuples(),
check_exiftool_filter()
check_exiftool_filter(),
check_deprecated_logger_config()
]
|> Enum.reduce(:ok, fn
:ok, :ok -> :ok
@ -415,4 +418,72 @@ defmodule Pleroma.Config.DeprecationWarnings do
:ok
end
end
defp merge_deprecated_logger_config(config) do
formatter_config = Enum.filter(config, fn {k, _} -> k in @logger_formatter_config_knobs end)
formatter = Logger.default_formatter(formatter_config)
log_level = Keyword.get(config, :level, nil)
Logger.debug("""
Reconfiguring console Logger with deprecated configuration syntax.
Handler configuration:
log_level: #{inspect(log_level)}
Formatter:
#{inspect(formatter)}
""")
if Config.get(:env) != :test do
if log_level, do: :logger.update_handler_config(:default, :level, log_level)
:logger.set_handler_config(:default, :formatter, formatter)
end
end
@spec check_deprecated_logger_config() :: :ok | :error
def check_deprecated_logger_config do
backends_config = Application.get_env(:logger, :backends)
console_config = Application.get_env(:logger, :console)
# NOTE: No need to merge the old backends config since it still works.
# And new configuration will just add new Logger backends.
backend =
if backends_config do
Logger.warning("""
!!!DEPRECATION WARNING!!!
Your configuration is using deprecated syntax for configuring backends of Elixir's logger.
`config :logger, backends: [...]` is deprecated syntax due to changes in Elixir.
Please update your configuration at your earliest convenience to use:
`config :pleroma, :logger, backends: [...]`
Note: `:console` is no longer considered a backend and is used by default, you can disable it using:
`config :logger, :default_handler: false`
""")
true
else
false
end
console =
if console_config do
Logger.warning("""
!!!DEPRECATION WARNING!!!
Your configuration is using deprecated syntax for configuring logging to console.
`config :logger, :console` is deprecated syntax due to changes in Elixir.
Please update your configuration at your earliest convenience to use
`config :logger, default_handler` and `config :logger, :default_formatter`.
Note: `:default_handler` is used only for the `level` setting. All other configurations go under
`:default_formatter`. For more info visit: https://hexdocs.pm/logger/Logger.html#module-backends-and-backwards-compatibility
""")
merge_deprecated_logger_config(console_config)
true
else
false
end
if backend or console, do: :error, else: :ok
end
end

3
lib/pleroma/config/transfer_task.ex
View file

@ -47,11 +47,12 @@ defmodule Pleroma.Config.TransferTask do
settings =
(Repo.all(ConfigDB) ++ deleted_settings)
|> Enum.reject(&ConfigDB.static?/1)
|> Enum.map(&merge_with_default/1)
started_applications = Application.started_applications()
reject = [nil, :postgrex]
reject = [nil, :postgrex, :logger]
reject =
if restart_pleroma? do

26
lib/pleroma/config_db.ex
View file

@ -23,6 +23,12 @@ defmodule Pleroma.ConfigDB do
{:pleroma, :mrf_keyword, :replace}
]
@static_config [
{:pleroma, Pleroma.Search},
{:pleroma, Pleroma.Search.ParadeDB},
{:pleroma, Pleroma.Search.ParadeDB.Repo}
]
schema "config" do
field(:key, Pleroma.EctoType.Config.Atom)
field(:group, Pleroma.EctoType.Config.Atom)
@ -32,16 +38,32 @@ defmodule Pleroma.ConfigDB do
timestamps()
end
@spec get_all_as_keyword() :: keyword()
def get_all_as_keyword do
@spec get_all_as_keyword(keyword()) :: keyword()
def get_all_as_keyword(options \\ []) do
ConfigDB
|> select([c], {c.group, c.key, c.value})
|> Repo.all()
|> maybe_reject_static(options)
|> Enum.reduce([], fn {group, key, value}, acc ->
Keyword.update(acc, group, [{key, value}], &Keyword.merge(&1, [{key, value}]))
end)
end
def static?(%{group: group, key: key}), do: static?(group, key)
def static?(_config), do: false
def static?(group, key) do
Enum.any?(@static_config, fn {static_group, static_key} ->
group in [static_group, inspect(static_group)] and key in [static_key, inspect(static_key)]
end)
end
defp maybe_reject_static(configs, exclude_static: true) do
Enum.reject(configs, fn {group, key, _value} -> static?(group, key) end)
end
defp maybe_reject_static(configs, _options), do: configs
@spec get_all_by_group(atom() | String.t()) :: [t()]
def get_all_by_group(group) do
from(c in ConfigDB, where: c.group == ^group) |> Repo.all()

23
lib/pleroma/docs/generator.ex
View file

@ -12,13 +12,27 @@ defmodule Pleroma.Docs.Generator do
@spec list_behaviour_implementations(behaviour :: module()) :: [module()]
def list_behaviour_implementations(behaviour) do
:code.all_loaded()
|> Enum.filter(fn {module, _} ->
# :code.all_loaded() no longer works on Elixir 1.19 and newer since modules are lazily loaded.
# We can get a complete list of modules by interating over Application.loaded_applications/0,
# but that includes every module which would get loaded below, which is very slow.
# Hardcode the Pleroma application instead, which is good enough for now.
application_modules =
case :application.get_key(:pleroma, :modules) do
{:ok, modules} -> modules
:undefined -> []
end
# At compile time or in tests some modules will be outside of the Pleroma application.
loaded_modules = Enum.map(:code.all_loaded(), fn {module, _} -> module end)
(application_modules ++ loaded_modules)
|> Enum.uniq()
|> Enum.filter(fn module ->
Code.ensure_loaded(module)
# This shouldn't be needed as all modules are expected to have module_info/1,
# but in test environments some transient modules `:elixir_compiler_XX`
# are loaded for some reason (where XX is a random integer).
Code.ensure_loaded(module)
if function_exported?(module, :module_info, 1) do
module.module_info(:attributes)
|> Keyword.get_values(:behaviour)
@ -26,7 +40,6 @@ defmodule Pleroma.Docs.Generator do
|> Enum.member?(behaviour)
end
end)
|> Enum.map(fn {module, _} -> module end)
end
@doc """

24
lib/pleroma/emails/mailer.ex
View file

@ -36,10 +36,11 @@ defmodule Pleroma.Emails.Mailer do
def deliver(email, config \\ [])
def deliver(email, config) do
# temporary hackney fix until hackney max_connections bug is fixed
# https://git.pleroma.social/pleroma/pleroma/-/issues/2101
# Swoosh HTTP adapters use Hackney, which otherwise trusts its certifi bundle
email =
Swoosh.Email.put_private(email, :hackney_options, ssl_options: [versions: [:"tlsv1.2"]])
Swoosh.Email.put_private(email, :hackney_options,
ssl_options: Pleroma.HTTP.TLS.ssl_options()
)
case enabled?() do
true -> Swoosh.Mailer.deliver(email, parse_config(config))
@ -67,6 +68,21 @@ defmodule Pleroma.Emails.Mailer do
end
defp parse_config(config) do
Swoosh.Mailer.parse_config(@otp_app, __MODULE__, @mailer_config, config)
@otp_app
|> Swoosh.Mailer.parse_config(__MODULE__, @mailer_config, config)
|> put_mua_cacertfile()
end
@doc false
# Mua otherwise trusts the OS CA store. Keep a CA that the admin set.
def put_mua_cacertfile(config) do
ssl = Keyword.get(config, :ssl, [])
if config[:adapter] == Swoosh.Adapters.Mua and
not Keyword.has_key?(ssl, :cacertfile) and not Keyword.has_key?(ssl, :cacerts) do
Keyword.put(config, :ssl, [{:cacertfile, CAStore.file_path()} | ssl])
else
config
end
end
end

10
lib/pleroma/following_relationship.ex
View file

@ -258,10 +258,16 @@ defmodule Pleroma.FollowingRelationship do
[_, user_actor: user_actor],
fragment(
# "(actor's domain NOT in domain_blocks) OR (actor IS in followed AP IDs)"
#
# ARRAY(SELECT ...) makes the followed-ids lookup an InitPlan that is
# evaluated and costed once. As a plain subselect, the planner charges
# its startup cost on every rescan of the actor lookup, which inflates
# the cost of the index plan enough for it to lose to a plan that
# materialises every active user for every candidate row.
"""
NOT (substring(? from '.*://([^/]*)') = ANY(?)) OR
? = ANY(SELECT ap_id FROM users AS u INNER JOIN following_relationships AS fr
ON u.id = fr.following_id WHERE fr.follower_id = ? AND fr.state = ?)
? = ANY(ARRAY(SELECT ap_id FROM users AS u INNER JOIN following_relationships AS fr
ON u.id = fr.following_id WHERE fr.follower_id = ? AND fr.state = ?))
""",
user_actor.ap_id,
^user.domain_blocks,

156
lib/pleroma/formatter.ex
View file

@ -3,12 +3,15 @@
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Formatter do
require Logger
alias Pleroma.HTML
alias Pleroma.User
@safe_mention_regex ~r/^(\s*(?<mentions>(@.+?\s+){1,})+)(?<rest>.*)/s
@link_regex ~r"((?:http(s)?:\/\/)?[\w.-]+(?:\.[\w\.-]+)+[\w\-\._~%:/?#[\]@!\$&'\(\)\*\+,;=.]+)|[0-9a-z+\-\.]+:[0-9a-z$-_.+!*'(),]+"ui
@markdown_characters_regex ~r/(`|\*|_|{|}|[|]|\(|\)|#|\+|-|\.|!)/
@markdown_void_elements ~w[area base br col command embed hr img input keygen link meta param source track wbr]
defp linkify_opts do
Pleroma.Config.get(Pleroma.Formatter) ++
@ -124,14 +127,157 @@ defmodule Pleroma.Formatter do
end
def markdown_to_html(text) do
Earmark.as_html!(text, %Earmark.Options{compact_output: true, smartypants: false})
markdown_to_html(text, %{})
end
def markdown_to_html(text, opts) do
Earmark.as_html!(
text,
%Earmark.Options{compact_output: true, smartypants: false} |> Map.merge(opts)
)
breaks = Map.get(opts, :breaks, false)
{_status, ast, _messages} = EarmarkParser.as_ast(text, breaks: breaks, gfm: true)
ast
|> maybe_strip_email_autolinks(text, Map.get(opts, :strip_email_autolinks, false))
|> render_markdown_nodes(false)
|> IO.iodata_to_binary()
rescue
exception ->
Logger.warning("Markdown rendering failed in #{inspect(exception.__struct__)}")
markdown_fallback(text, Map.get(opts, :breaks, false))
end
defp markdown_fallback(text, breaks) do
text = text |> escape_html() |> IO.iodata_to_binary() |> maybe_render_markdown_breaks(breaks)
"<p>" <> text <> "</p>"
end
defp maybe_render_markdown_breaks(text, true), do: String.replace(text, ~r/\r?\n/, "<br>")
defp maybe_render_markdown_breaks(text, false), do: text
defp maybe_strip_email_autolinks(ast, _text, false), do: ast
defp maybe_strip_email_autolinks(ast, text, true) do
emails =
~r/<([^ >]+@[^ >]+)>/
|> Regex.scan(text, capture: :all_but_first)
|> Enum.map(fn [email] -> String.replace_prefix(email, "mailto:", "") end)
|> MapSet.new()
strip_email_autolinks(ast, emails)
end
defp strip_email_autolinks(nodes, emails) when is_list(nodes) do
Enum.map(nodes, &strip_email_autolinks(&1, emails))
end
defp strip_email_autolinks({"a", attrs, [email], _meta} = node, emails)
when is_binary(email) do
if List.keyfind(attrs, "href", 0) == {"href", "mailto:" <> email} and
MapSet.member?(emails, email) do
email
else
node
end
end
defp strip_email_autolinks({tag, attrs, children, meta}, emails) do
{tag, attrs, strip_email_autolinks(children, emails), meta}
end
defp strip_email_autolinks(text, _emails) when is_binary(text), do: text
defp render_markdown_nodes(nodes, verbatim) when is_list(nodes) do
Enum.map(nodes, &render_markdown_node(&1, verbatim))
end
defp render_markdown_node(text, true) when is_binary(text),
do: strip_dangerous_markdown_html(text)
defp render_markdown_node(text, false) when is_binary(text) do
text
|> strip_dangerous_markdown_html()
|> escape_markdown_html()
end
defp render_markdown_node({:comment, _attrs, _children, _meta}, _verbatim), do: []
defp render_markdown_node({tag, attrs, children, meta}, verbatim) do
normalized_tag = String.downcase(tag)
if normalized_tag in ["script", "style"] do
[]
else
render_markdown_element(tag, normalized_tag, attrs, children, meta, verbatim)
end
end
defp render_markdown_element("code", _normalized_tag, attrs, children, _meta, _verbatim) do
["<code", render_markdown_attrs(attrs), ">", render_markdown_code(children), "</code>"]
end
defp render_markdown_element(tag, "hr", attrs, _children, _meta, _verbatim) do
attrs = Enum.reject(attrs, &match?({"class", class} when class in ~w[thin medium thick], &1))
["<", tag, render_markdown_attrs(attrs), "/>"]
end
defp render_markdown_element(tag, normalized_tag, attrs, _children, _meta, _verbatim)
when normalized_tag in @markdown_void_elements do
["<", tag, render_markdown_attrs(attrs), ">"]
end
defp render_markdown_element(tag, _normalized_tag, attrs, children, meta, verbatim) do
children_verbatim = verbatim or Map.get(meta, :verbatim, false)
[
"<",
tag,
render_markdown_attrs(attrs),
">",
render_markdown_nodes(children, children_verbatim),
"</",
tag,
">"
]
end
defp render_markdown_attrs(attrs) do
Enum.map(attrs, fn {name, value} ->
value = value |> to_string() |> HtmlEntities.decode() |> sanitize_markdown_url(name)
[" ", to_string(name), "=\"", escape_html(value), "\""]
end)
end
defp render_markdown_code(children) do
Enum.map(children, &escape_html/1)
end
defp sanitize_markdown_url(value, name) do
if String.downcase(to_string(name)) in ["href", "src"] do
# Percent-encode instead of cutting, so the URL stays complete but can
# never close the attribute or open a tag.
String.replace(value, ~r/["<>]/, fn
"\"" -> "%22"
"<" -> "%3C"
">" -> "%3E"
end)
else
value
end
end
defp strip_dangerous_markdown_html(text) do
text
|> String.replace(~r/<(script|style)\b[^>]*>.*?<\/\1>/is, "")
|> String.replace(~r/<(script|style)\b[^>]*>.*\z/is, "")
|> String.replace(~r/<\/(script|style)\s*>/i, "")
end
defp escape_markdown_html(text) do
text |> HtmlEntities.decode() |> escape_html()
end
defp escape_html(text) do
{:safe, escaped} = Phoenix.HTML.html_escape(text)
escaped
end
def html_escape({text, mentions, hashtags}, type) do

9
lib/pleroma/gun.ex
View file

@ -6,9 +6,12 @@ defmodule Pleroma.Gun do
@callback open(charlist(), pos_integer(), map()) :: {:ok, pid()}
@callback info(pid()) :: map()
@callback close(pid()) :: :ok
@callback cancel(pid(), reference() | [reference()]) :: :ok
@callback await_up(pid, pos_integer()) :: {:ok, atom()} | {:error, atom()}
@callback connect(pid(), map()) :: reference()
@callback await(pid(), reference()) :: {:response, :fin, 200, []}
@callback await(pid(), reference()) :: tuple()
@callback await_tunnel_up(pid(), reference() | :undefined, timeout()) ::
{:ok, atom()} | {:error, term()}
@callback set_owner(pid(), pid()) :: :ok
defp api, do: Pleroma.Config.get([Pleroma.Gun], Pleroma.Gun.API)
@ -19,11 +22,15 @@ defmodule Pleroma.Gun do
def close(pid), do: api().close(pid)
def cancel(pid, stream), do: api().cancel(pid, stream)
def await_up(pid, timeout \\ 5_000), do: api().await_up(pid, timeout)
def connect(pid, opts), do: api().connect(pid, opts)
def await(pid, ref), do: api().await(pid, ref)
def await_tunnel_up(pid, ref, timeout), do: api().await_tunnel_up(pid, ref, timeout)
def set_owner(pid, owner), do: api().set_owner(pid, owner)
end

25
lib/pleroma/gun/api.ex
View file

@ -32,6 +32,9 @@ defmodule Pleroma.Gun.API do
@impl Gun
defdelegate close(pid), to: :gun
@impl Gun
defdelegate cancel(pid, stream), to: :gun
@impl Gun
defdelegate await_up(pid, timeout \\ 5_000), to: :gun
@ -41,6 +44,28 @@ defmodule Pleroma.Gun.API do
@impl Gun
defdelegate await(pid, ref), to: :gun
@impl Gun
def await_tunnel_up(pid, stream_ref, timeout) do
monitor_ref = Process.monitor(pid)
result =
receive do
{:gun_tunnel_up, ^pid, ^stream_ref, protocol} ->
{:ok, protocol}
{:gun_down, ^pid, _protocol, reason, _killed_streams} ->
{:error, {:down, reason}}
{:DOWN, ^monitor_ref, :process, ^pid, reason} ->
{:error, {:down, reason}}
after
timeout -> {:error, :timeout}
end
Process.demonitor(monitor_ref, [:flush])
result
end
@impl Gun
defdelegate set_owner(pid, owner), to: :gun
end

139
lib/pleroma/gun/conn.ex
View file

@ -42,18 +42,34 @@ defmodule Pleroma.Gun.Conn do
Map.put(opts, :tls_opts, tls_opts)
end
defp do_open(%URI{scheme: "http"} = uri, %{proxy: {proxy_host, proxy_port}} = opts) do
with open_opts <- opts |> proxy_open_opts() |> Map.put(:protocols, [:http]),
{:ok, conn} <- Gun.open(proxy_host, proxy_port, open_opts),
{:ok, protocol} <- await_up(conn, opts[:connect_timeout]) do
{:ok, conn, protocol, :forward_proxy}
else
error ->
Logger.warning(
"Opening forward proxy connection to #{compose_uri_log(uri)} failed with error #{inspect(error)}"
)
error
end
end
defp do_open(uri, %{proxy: {proxy_host, proxy_port}} = opts) do
connect_opts =
uri
|> destination_opts()
|> add_http2_opts(uri.scheme, Map.get(opts, :tls_opts, []))
|> add_proxy_auth(opts)
with open_opts <- Map.delete(opts, :tls_opts),
with open_opts <- proxy_open_opts(opts),
{:ok, conn} <- Gun.open(proxy_host, proxy_port, open_opts),
{:ok, protocol} <- Gun.await_up(conn, opts[:connect_timeout]),
stream <- Gun.connect(conn, connect_opts),
{:response, :fin, 200, _} <- Gun.await(conn, stream) do
{:ok, conn, protocol}
{:ok, _proxy_protocol} <- await_up(conn, opts[:connect_timeout]) do
conn
|> Gun.connect(connect_opts)
|> await_proxy_connect(conn, opts[:connect_timeout])
else
error ->
Logger.warning(
@ -64,30 +80,29 @@ defmodule Pleroma.Gun.Conn do
end
end
defp do_open(uri, %{proxy: {proxy_type, proxy_host, proxy_port}} = opts) do
version =
proxy_type
|> to_string()
|> String.last()
|> case do
"4" -> 4
_ -> 5
end
defp do_open(_uri, %{proxy: {:socks4, _proxy_host, _proxy_port}}) do
{:error, :socks4_unsupported}
end
defp do_open(uri, %{proxy: {proxy_type, proxy_host, proxy_port}} = opts)
when proxy_type in [:socks, :socks5] do
socks_opts =
uri
|> destination_opts()
|> add_http2_opts(uri.scheme, Map.get(opts, :tls_opts, []))
|> Map.put(:version, version)
|> Map.put(:version, 5)
|> add_socks_proxy_auth(opts)
opts =
open_opts =
opts
|> proxy_open_opts()
|> Map.put(:protocols, [:socks])
|> Map.put(:socks_opts, socks_opts)
with {:ok, conn} <- Gun.open(proxy_host, proxy_port, opts),
{:ok, protocol} <- Gun.await_up(conn, opts[:connect_timeout]) do
{:ok, conn, protocol}
with {:ok, conn} <- Gun.open(proxy_host, proxy_port, open_opts),
{:ok, :socks} <- await_up(conn, opts[:connect_timeout]),
{:ok, protocol} <- await_tunnel_up(conn, :undefined, opts[:connect_timeout]) do
{:ok, conn, protocol, nil}
else
error ->
Logger.warning(
@ -100,10 +115,11 @@ defmodule Pleroma.Gun.Conn do
defp do_open(%URI{host: host, port: port} = uri, opts) do
host = Pleroma.HTTP.AdapterHelper.parse_host(host)
opts = Map.put(opts, :transport, transport(uri.scheme))
with {:ok, conn} <- Gun.open(host, port, opts),
{:ok, protocol} <- Gun.await_up(conn, opts[:connect_timeout]) do
{:ok, conn, protocol}
{:ok, protocol} <- await_up(conn, opts[:connect_timeout]) do
{:ok, conn, protocol, nil}
else
error ->
Logger.warning(
@ -120,11 +136,90 @@ defmodule Pleroma.Gun.Conn do
end
defp add_http2_opts(opts, "https", tls_opts) do
Map.merge(opts, %{protocols: [:http2], transport: :tls, tls_opts: tls_opts})
Map.merge(opts, %{protocols: [:http2, :http], transport: :tls, tls_opts: tls_opts})
end
defp add_http2_opts(opts, _, _), do: opts
defp add_proxy_auth(connect_opts, %{proxy_auth: {username, password}})
when is_binary(username) and is_binary(password) do
Map.merge(connect_opts, %{username: username, password: password})
end
defp add_proxy_auth(connect_opts, _), do: connect_opts
defp add_socks_proxy_auth(socks_opts, %{proxy_auth: {username, password}})
when is_binary(username) and is_binary(password) do
Map.put(socks_opts, :auth, [{:username_password, username, password}])
end
defp add_socks_proxy_auth(socks_opts, _), do: socks_opts
defp proxy_open_opts(opts) do
proxy_tls_opts = Map.get(opts, :proxy_tls_opts, [])
opts =
opts
|> Map.drop([:proxy, :proxy_auth, :proxy_tls_opts, :tls_opts])
|> Map.put_new(:transport, :tcp)
if opts.transport == :tls do
Map.put(opts, :tls_opts, proxy_tls_opts)
else
opts
end
end
defp await_up(conn, timeout) do
case Gun.await_up(conn, timeout) do
{:ok, protocol} ->
{:ok, protocol}
error ->
Gun.close(conn)
error
end
end
defp await_proxy_connect(stream, conn, timeout) do
case Gun.await(conn, stream) do
{:response, :fin, 200, _headers} ->
case await_tunnel_up(conn, stream, timeout) do
{:ok, protocol} -> {:ok, conn, protocol, stream}
error -> error
end
{:response, _fin, 407, _headers} ->
close_with_error(conn, :proxy_auth_failed)
{:response, _fin, status, _headers} when is_integer(status) ->
close_with_error(conn, {:proxy_connect_failed, status})
error ->
Gun.close(conn)
error
end
end
defp close_with_error(conn, reason) do
Gun.close(conn)
{:error, reason}
end
defp transport("https"), do: :tls
defp transport(_), do: :tcp
defp await_tunnel_up(conn, stream_ref, timeout) do
case Gun.await_tunnel_up(conn, stream_ref, timeout) do
{:ok, protocol} ->
{:ok, protocol}
error ->
Gun.close(conn)
error
end
end
def compose_uri_log(%URI{scheme: scheme, host: host, path: path}) do
"#{scheme}://#{host}#{path}"
end

225
lib/pleroma/gun/connection_pool.ex
View file

@ -4,53 +4,132 @@
defmodule Pleroma.Gun.ConnectionPool do
@registry __MODULE__
@locks Module.concat(__MODULE__, Locks)
alias Pleroma.Gun.ConnectionPool.WorkerSupervisor
@connection_option_keys [
:http_opts,
:http2_opts,
:protocols,
:proxy,
:proxy_auth,
:proxy_tls_opts,
:retry,
:retry_timeout,
:socks_opts,
:tcp_opts,
:tls_opts,
:transport,
:ws_opts
]
def children do
[
{Registry, keys: :unique, name: @registry},
{Registry, keys: :duplicate, name: @registry},
{Registry, keys: :unique, name: @locks},
Pleroma.Gun.ConnectionPool.WorkerSupervisor
]
end
@spec get_conn(URI.t(), keyword()) :: {:ok, pid()} | {:error, term()}
def get_conn(uri, opts) do
key = "#{uri.scheme}:#{uri.host}:#{uri.port}"
key = connection_key(uri, opts)
case Registry.lookup(@registry, key) do
# The key has already been registered, but connection is not up yet
[{worker_pid, nil}] ->
get_gun_pid_from_worker(worker_pid, true)
[{worker_pid, {gun_pid, _used_by, _crf, _last_reference}}] ->
GenServer.call(worker_pid, :add_client)
case checkout_existing(Registry.lookup(@registry, key)) do
{:ok, gun_pid} ->
{:ok, gun_pid}
[] ->
# :gun.set_owner fails in :connected state for whatevever reason,
# so we open the connection in the process directly and send it's pid back
# We trust gun to handle timeouts by itself
case WorkerSupervisor.start_worker([key, uri, opts, self()]) do
{:ok, worker_pid} ->
get_gun_pid_from_worker(worker_pid, false)
:not_available ->
provision_or_wait(key, uri, opts)
end
end
{:error, {:already_started, worker_pid}} ->
get_gun_pid_from_worker(worker_pid, true)
defp checkout_existing([]), do: :not_available
err ->
err
defp checkout_existing([{_worker_pid, nil} | workers]), do: checkout_existing(workers)
defp checkout_existing([{worker_pid, _value} | workers]) do
case checkout_worker(worker_pid) do
{:ok, gun_pid} -> {:ok, gun_pid}
_ -> checkout_existing(workers)
end
end
defp provision_or_wait(key, uri, opts) do
result = with_route_lock(key, fn -> provision(key, uri, opts) end)
case result do
{:ok, gun_pid} ->
{:ok, gun_pid}
{:new, worker_pid} ->
get_gun_pid_from_worker(worker_pid)
{:wait, worker_pid} ->
case checkout_worker(worker_pid, :infinity) do
{:ok, gun_pid} -> {:ok, gun_pid}
:busy -> get_conn(uri, opts)
error -> error
end
error ->
error
end
end
defp provision(key, uri, opts) do
workers = Registry.lookup(@registry, key)
case checkout_existing(workers) do
{:ok, gun_pid} ->
{:ok, gun_pid}
:not_available ->
case Enum.find(workers, fn {_worker_pid, value} -> is_nil(value) end) do
{worker_pid, nil} -> {:wait, worker_pid}
nil -> start_worker(key, uri, opts)
end
end
end
defp get_gun_pid_from_worker(worker_pid, register) do
defp with_route_lock(key, function) do
case Registry.register(@locks, key, nil) do
{:ok, _} ->
try do
function.()
after
Registry.unregister(@locks, key)
end
{:error, {:already_registered, _owner}} ->
Process.sleep(1)
with_route_lock(key, function)
end
end
defp start_worker(key, uri, opts) do
# :gun.set_owner fails in :connected state, so the worker opens the connection itself.
case WorkerSupervisor.start_worker([key, uri, opts, self()]) do
{:ok, worker_pid} -> {:new, worker_pid}
error -> error
end
end
defp checkout_worker(worker_pid, timeout \\ 5_000) do
try do
GenServer.call(worker_pid, :checkout, timeout)
catch
:exit, reason -> {:error, {:connection_worker_exit, reason}}
end
end
defp get_gun_pid_from_worker(worker_pid) do
# GenServer.call will block the process for timeout length if
# the server crashes on startup (which will happen if gun fails to connect)
# so instead we use cast + monitor
ref = Process.monitor(worker_pid)
if register, do: GenServer.cast(worker_pid, {:add_client, self()})
receive do
{:conn_pid, pid} ->
@ -68,19 +147,115 @@ defmodule Pleroma.Gun.ConnectionPool do
@spec release_conn(pid()) :: :ok
def release_conn(conn_pid) do
# :ets.fun2ms(fn {_, {worker_pid, {gun_pid, _, _, _}}} when gun_pid == conn_pid ->
# :ets.fun2ms(fn {_, {worker_pid, {gun_pid, _tunnel}}} when gun_pid == conn_pid ->
# worker_pid end)
query_result =
Registry.select(@registry, [
{{:_, :"$1", {:"$2", :_, :_, :_}}, [{:==, :"$2", conn_pid}], [:"$1"]}
{{:_, :"$1", {:"$2", :_}}, [{:==, :"$2", conn_pid}], [:"$1"]}
])
case query_result do
[worker_pid] ->
GenServer.call(worker_pid, :remove_client)
try do
GenServer.call(worker_pid, :remove_client)
catch
:exit, _ -> :ok
end
[] ->
:ok
end
end
@spec discard_conn(pid()) :: :ok
def discard_conn(conn_pid) do
case worker_for_conn(conn_pid) do
nil ->
:ok
worker_pid ->
DynamicSupervisor.terminate_child(WorkerSupervisor, worker_pid)
end
end
@spec cancel_stream(pid(), reference() | [reference()]) :: :ok
def cancel_stream(conn_pid, stream) do
case worker_for_conn(conn_pid) do
nil ->
Pleroma.Gun.cancel(conn_pid, stream)
worker_pid ->
try do
GenServer.call(worker_pid, {:cancel_stream, stream})
catch
:exit, _ -> :ok
end
end
end
@spec register_stream(pid(), reference() | [reference()]) :: :ok
def register_stream(conn_pid, stream) do
call_worker(conn_pid, {:register_stream, stream})
end
@spec finish_stream(pid(), reference() | [reference()]) :: :ok
def finish_stream(conn_pid, stream) do
call_worker(conn_pid, {:finish_stream, stream})
end
@spec release_stream(pid(), reference() | [reference()]) :: :ok
def release_stream(conn_pid, stream) do
call_worker(conn_pid, {:release_stream, stream})
end
@spec tunnel_ref(pid()) :: reference() | :forward_proxy | nil
def tunnel_ref(conn_pid) do
case Registry.select(@registry, [
{{:_, :_, {:"$1", :"$2"}}, [{:==, :"$1", conn_pid}], [:"$2"]}
]) do
[tunnel] -> tunnel
[] -> nil
end
end
defp worker_for_conn(conn_pid) do
case Registry.select(@registry, [
{{:_, :"$1", {:"$2", :_}}, [{:==, :"$2", conn_pid}], [:"$1"]}
]) do
[worker_pid] -> worker_pid
[] -> nil
end
end
defp call_worker(conn_pid, message) do
case worker_for_conn(conn_pid) do
nil ->
:ok
worker_pid ->
try do
GenServer.call(worker_pid, message)
catch
:exit, _ -> :ok
end
end
end
defp connection_key(uri, opts) do
origin = {
String.downcase(uri.scheme),
String.downcase(uri.host),
uri.port
}
profile =
opts
|> Map.new()
|> Map.take(@connection_option_keys)
|> :erlang.term_to_binary()
|> then(&:crypto.hash(:sha256, &1))
fingerprint = Base.encode16(profile, case: :lower)
Enum.join(Tuple.to_list(origin) ++ [fingerprint], ":")
end
end

51
lib/pleroma/gun/connection_pool/reclaimer.ex
View file

@ -9,11 +9,11 @@ defmodule Pleroma.Gun.ConnectionPool.Reclaimer do
def start_monitor do
pid =
case GenServer.start(__MODULE__, [], name: {:via, Registry, {registry(), "reclaimer"}}) do
case GenServer.start(__MODULE__, [], name: __MODULE__) do
{:ok, pid} ->
pid
{:error, {:already_registered, pid}} ->
{:error, {:already_started, pid}} ->
pid
end
@ -41,17 +41,16 @@ defmodule Pleroma.Gun.ConnectionPool.Reclaimer do
reclaim_max: reclaim_max
})
# :ets.fun2ms(
# fn {_, {worker_pid, {_, used_by, crf, last_reference}}} when used_by == [] ->
# {worker_pid, crf, last_reference} end)
unused_conns =
workers =
Registry.select(
registry(),
[
{{:_, :"$1", {:_, :"$2", :"$3", :"$4"}}, [{:==, :"$2", []}], [{{:"$1", :"$3", :"$4"}}]}
{{:_, :"$1", {:_, :_}}, [], [:"$1"]}
]
)
unused_conns = Enum.flat_map(workers, &reclaim_info/1)
case unused_conns do
[] ->
:telemetry.execute(
@ -65,25 +64,43 @@ defmodule Pleroma.Gun.ConnectionPool.Reclaimer do
{:stop, :no_unused_conns, nil}
unused_conns ->
reclaimed =
reclaimed_count =
unused_conns
|> Enum.sort(fn {_pid1, crf1, last_reference1}, {_pid2, crf2, last_reference2} ->
crf1 <= crf2 and last_reference1 <= last_reference2
end)
|> Enum.take(reclaim_max)
|> Enum.sort_by(fn {_pid, crf, last_reference} -> {crf, last_reference} end)
|> Enum.reduce_while(0, fn
_worker, count when count == reclaim_max ->
{:halt, count}
reclaimed
|> Enum.each(fn {pid, _, _} ->
DynamicSupervisor.terminate_child(Pleroma.Gun.ConnectionPool.WorkerSupervisor, pid)
end)
{worker_pid, _crf, _last_reference}, count ->
if reclaim(worker_pid), do: {:cont, count + 1}, else: {:cont, count}
end)
:telemetry.execute(
[:pleroma, :connection_pool, :reclaim, :stop],
%{reclaimed_count: Enum.count(reclaimed)},
%{reclaimed_count: reclaimed_count},
%{max_connections: max_connections}
)
{:stop, :normal, nil}
end
end
defp reclaim_info(worker_pid) do
try do
case GenServer.call(worker_pid, :reclaim_info) do
{:idle, crf, last_reference} -> [{worker_pid, crf, last_reference}]
:in_use -> []
end
catch
:exit, _ -> []
end
end
defp reclaim(worker_pid) do
try do
GenServer.call(worker_pid, :reclaim) == :reclaimed
catch
:exit, _ -> false
end
end
end

268
lib/pleroma/gun/connection_pool/worker.ex
View file

@ -8,25 +8,22 @@ defmodule Pleroma.Gun.ConnectionPool.Worker do
defp registry, do: Pleroma.Gun.ConnectionPool
def start_link([key | _] = opts) do
GenServer.start_link(__MODULE__, opts, name: {:via, Registry, {registry(), key}})
end
def start_link(opts), do: GenServer.start_link(__MODULE__, opts)
@impl true
def init([_key, _uri, _opts, _client_pid] = opts) do
def init([key, _uri, _opts, _client_pid] = opts) do
{:ok, _} = Registry.register(registry(), key, nil)
{:ok, nil, {:continue, {:connect, opts}}}
end
@impl true
def handle_continue({:connect, [key, uri, opts, client_pid]}, _) do
with {:ok, conn_pid, protocol} <- Gun.Conn.open(uri, opts),
with {:ok, conn_pid, protocol, tunnel} <- Gun.Conn.open(uri, opts),
Process.link(conn_pid) do
time = :erlang.monotonic_time(:millisecond)
{_, _} =
Registry.update_value(registry(), key, fn _ ->
{conn_pid, [client_pid], 1, time}
end)
Registry.unregister(registry(), key)
{:ok, _} = Registry.register(registry(), key, {conn_pid, tunnel})
send(client_pid, {:conn_pid, conn_pid})
@ -34,8 +31,14 @@ defmodule Pleroma.Gun.ConnectionPool.Worker do
%{
key: key,
timer: nil,
client_monitors: %{client_pid => Process.monitor(client_pid)},
protocol: protocol
clients: %{
client_pid => %{count: 1, monitor: Process.monitor(client_pid), streams: MapSet.new()}
},
conn_pid: conn_pid,
protocol: protocol,
proxied: Keyword.has_key?(opts, :proxy),
crf: 1,
last_reference: time
}, :hibernate}
else
err ->
@ -44,106 +47,225 @@ defmodule Pleroma.Gun.ConnectionPool.Worker do
end
@impl true
def handle_cast({:add_client, client_pid}, state) do
case handle_call(:add_client, {client_pid, nil}, state) do
{:reply, conn_pid, state, :hibernate} ->
send(client_pid, {:conn_pid, conn_pid})
{:noreply, state, :hibernate}
end
def handle_call(:checkout, _from, %{protocol: protocol, clients: clients} = state)
when protocol != :http2 and map_size(clients) > 0 do
{:reply, :busy, state, :hibernate}
end
@impl true
def handle_cast({:remove_client, client_pid}, state) do
case handle_call(:remove_client, {client_pid, nil}, state) do
{:reply, _, state, :hibernate} ->
{:noreply, state, :hibernate}
end
end
@impl true
def handle_call(:add_client, {client_pid, _}, %{key: key, protocol: protocol} = state) do
def handle_call(:checkout, {client_pid, _}, %{key: key, protocol: protocol} = state) do
time = :erlang.monotonic_time(:millisecond)
{{conn_pid, used_by, _, _}, _} =
Registry.update_value(registry(), key, fn {conn_pid, used_by, crf, last_reference} ->
{conn_pid, [client_pid | used_by], crf(time - last_reference, crf), time}
end)
[{_, {conn_pid, _tunnel}}] =
Registry.lookup(registry(), key) |> Enum.filter(&(elem(&1, 0) == self()))
state =
state
|> cancel_idle_timer()
|> add_client(client_pid)
|> Map.put(:crf, crf(time - state.last_reference, state.crf))
|> Map.put(:last_reference, time)
:telemetry.execute(
[:pleroma, :connection_pool, :client, :add],
%{client_pid: client_pid, clients: used_by},
%{client_pid: client_pid, clients: Map.keys(state.clients)},
%{key: state.key, protocol: protocol}
)
state =
if state.timer != nil do
Process.cancel_timer(state[:timer])
%{state | timer: nil}
else
state
end
ref = Process.monitor(client_pid)
state = put_in(state.client_monitors[client_pid], ref)
{:reply, conn_pid, state, :hibernate}
{:reply, {:ok, conn_pid}, state, :hibernate}
end
@impl true
def handle_call(:remove_client, {client_pid, _}, %{key: key} = state) do
{{_conn_pid, used_by, _crf, _last_reference}, _} =
Registry.update_value(registry(), key, fn {conn_pid, used_by, crf, last_reference} ->
{conn_pid, List.delete(used_by, client_pid), crf, last_reference}
end)
def handle_call(:remove_client, {client_pid, _}, state) do
case active_streams(state, client_pid) do
[] ->
{:reply, :ok, remove_client(state, client_pid, true), :hibernate}
{ref, state} = pop_in(state.client_monitors[client_pid])
streams when state.protocol == :http2 ->
Enum.each(streams, &(:ok = Gun.cancel(state.conn_pid, &1)))
{:reply, :ok, remove_client(state, client_pid, true), :hibernate}
Process.demonitor(ref, [:flush])
_streams ->
{:stop, :normal, :ok, state}
end
end
timer =
if used_by == [] do
max_idle = Pleroma.Config.get([:connections_pool, :max_idle_time], 30_000)
Process.send_after(self(), :idle_close, max_idle)
else
nil
end
def handle_call({:register_stream, stream}, {client_pid, _}, state) do
{:reply, :ok, update_stream(state, client_pid, &MapSet.put(&1, stream)), :hibernate}
end
{:reply, :ok, %{state | timer: timer}, :hibernate}
def handle_call({:finish_stream, stream}, {client_pid, _}, state) do
{:reply, :ok, update_stream(state, client_pid, &MapSet.delete(&1, stream)), :hibernate}
end
def handle_call({:release_stream, stream}, {client_pid, _}, state) do
state = update_stream(state, client_pid, &MapSet.delete(&1, stream))
{:reply, :ok, remove_client(state, client_pid, true), :hibernate}
end
def handle_call({:cancel_stream, stream}, {client_pid, _}, %{protocol: :http2} = state) do
:ok = Gun.cancel(state.conn_pid, stream)
state = update_stream(state, client_pid, &MapSet.delete(&1, stream))
{:reply, :ok, remove_client(state, client_pid, true), :hibernate}
end
def handle_call({:cancel_stream, _stream}, _from, state) do
{:stop, :normal, :ok, state}
end
def handle_call(:reclaim_info, _from, %{clients: clients} = state)
when map_size(clients) == 0 do
{:reply, {:idle, state.crf, state.last_reference}, state, :hibernate}
end
def handle_call(:reclaim_info, _from, state) do
{:reply, :in_use, state, :hibernate}
end
def handle_call(:reclaim, _from, %{clients: clients} = state) when map_size(clients) == 0 do
{:stop, :normal, :reclaimed, state}
end
def handle_call(:reclaim, _from, state) do
{:reply, :in_use, state, :hibernate}
end
@impl true
def handle_info(:idle_close, state) do
# Gun monitors the owner process, and will close the connection automatically
# when it's terminated
def handle_info({:idle_close, token}, %{timer: {_timer_ref, token}, clients: clients} = state)
when map_size(clients) == 0 do
# Gun monitors the owner process, and will close the connection automatically.
{:stop, :normal, state}
end
def handle_info({:idle_close, _token}, state), do: {:noreply, state, :hibernate}
@impl true
def handle_info({:gun_up, _pid, _protocol}, state) do
{:noreply, state, :hibernate}
def handle_info({:gun_up, _pid, protocol}, state) do
{:noreply, %{state | protocol: protocol}, :hibernate}
end
# Gracefully shutdown if the connection got closed without any streams left
@impl true
def handle_info({:gun_error, conn_pid, stream, _reason}, %{conn_pid: conn_pid} = state) do
{:noreply, delete_stream(state, stream), :hibernate}
end
@impl true
def handle_info({:gun_down, _pid, _protocol, _reason, _streams}, %{proxied: true} = state) do
{:stop, :normal, state}
end
# Gracefully shutdown if the connection got closed without any streams left.
@impl true
def handle_info({:gun_down, _pid, _protocol, _reason, []}, state) do
{:stop, :normal, state}
end
# Otherwise, wait for retry
# Otherwise, wait for retry.
@impl true
def handle_info({:gun_down, _pid, _protocol, _reason, _killed_streams}, state) do
{:noreply, state, :hibernate}
end
@impl true
def handle_info({:DOWN, _ref, :process, pid, reason}, state) do
:telemetry.execute(
[:pleroma, :connection_pool, :client, :dead],
%{client_pid: pid, reason: reason},
%{key: state.key}
)
def handle_info({:DOWN, ref, :process, pid, reason}, state) do
case state.clients[pid] do
%{monitor: ^ref} ->
:telemetry.execute(
[:pleroma, :connection_pool, :client, :dead],
%{client_pid: pid, reason: reason},
%{key: state.key}
)
handle_cast({:remove_client, pid}, state)
handle_dead_client(state, pid)
_ ->
{:noreply, state, :hibernate}
end
end
defp add_client(state, client_pid) do
update_in(state.clients, fn clients ->
case clients[client_pid] do
nil ->
Map.put(clients, client_pid, %{
count: 1,
monitor: Process.monitor(client_pid),
streams: MapSet.new()
})
client ->
Map.put(clients, client_pid, %{client | count: client.count + 1})
end
end)
end
defp handle_dead_client(state, client_pid) do
streams = active_streams(state, client_pid)
cond do
streams == [] ->
{:noreply, remove_client(state, client_pid, false, true), :hibernate}
state.protocol == :http2 ->
Enum.each(streams, &(:ok = Gun.cancel(state.conn_pid, &1)))
{:noreply, remove_client(state, client_pid, false, true), :hibernate}
true ->
{:stop, :normal, state}
end
end
defp active_streams(state, client_pid) do
case state.clients[client_pid] do
%{streams: streams} -> MapSet.to_list(streams)
nil -> []
end
end
defp update_stream(state, client_pid, update) do
update_in(state.clients, fn clients ->
case clients[client_pid] do
nil -> clients
client -> Map.put(clients, client_pid, %{client | streams: update.(client.streams)})
end
end)
end
defp delete_stream(state, stream) do
update_in(state.clients, fn clients ->
Map.new(clients, fn {pid, client} ->
{pid, %{client | streams: MapSet.delete(client.streams, stream)}}
end)
end)
end
defp remove_client(state, client_pid, demonitor, all \\ false) do
case state.clients[client_pid] do
nil ->
state
%{count: count} = client when count > 1 and not all ->
put_in(state.clients[client_pid], %{client | count: count - 1})
client ->
if demonitor, do: Process.demonitor(client.monitor, [:flush])
state = update_in(state.clients, &Map.delete(&1, client_pid))
if map_size(state.clients) == 0, do: schedule_idle_close(state), else: state
end
end
defp cancel_idle_timer(%{timer: nil} = state), do: state
defp cancel_idle_timer(%{timer: {timer_ref, _token}} = state) do
Process.cancel_timer(timer_ref)
%{state | timer: nil}
end
defp schedule_idle_close(state) do
max_idle = Pleroma.Config.get([:connections_pool, :max_idle_time], 30_000)
token = make_ref()
timer_ref = Process.send_after(self(), {:idle_close, token}, max_idle)
%{state | timer: {timer_ref, token}}
end
# LRFU policy: https://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.55.1478

9
lib/pleroma/gun/connection_pool/worker_supervisor.ex
View file

@ -26,7 +26,14 @@ defmodule Pleroma.Gun.ConnectionPool.WorkerSupervisor do
def start_worker(opts, true) do
case DynamicSupervisor.start_child(__MODULE__, {Worker, opts}) do
{:error, :max_children} ->
:telemetry.execute([:pleroma, :connection_pool, :provision_failure], %{opts: opts})
[key | _] = opts
:telemetry.execute(
[:pleroma, :connection_pool, :provision_failure],
%{},
%{key: key}
)
{:error, :pool_full}
res ->

40
lib/pleroma/html/uri_attributes.ex Normal file
View file

@ -0,0 +1,40 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.HTML.URIAttributes do
@moduledoc """
URI attribute rules for the scrubbers.
Works like `FastSanitize.Sanitizer.Meta.allow_tag_with_uri_attributes/3`, but
checks the scheme the way a browser reads the URL. Browsers remove tabs and
newlines anywhere and control characters and spaces at the start, so
`java\\tscript:` is a `javascript:` link to them, while `URI.parse/1` finds no
scheme and FastSanitize lets it through.
"""
defmacro allow_tag_with_uri_attributes(tag, attrs, valid_schemes) do
for attr <- attrs do
quote do
def scrub_attribute(unquote(tag), {unquote(attr), "&" <> _}), do: nil
def scrub_attribute(unquote(tag), {unquote(attr), uri} = attribute) do
if Pleroma.HTML.URIAttributes.valid_scheme?(uri, unquote(valid_schemes)),
do: attribute
end
end
end
end
@spec valid_scheme?(String.t(), [String.t()]) :: boolean()
def valid_scheme?(uri, valid_schemes) do
scheme =
uri
|> String.replace(~r/[\t\n\r]/, "")
|> String.replace(~r/\A[\x00-\x20]+/, "")
|> URI.parse()
|> Map.get(:scheme)
is_nil(scheme) or scheme in valid_schemes
end
end

6
lib/pleroma/http.ex
View file

@ -130,5 +130,9 @@ defmodule Pleroma.HTTP do
end
defp default_middleware,
do: [Tesla.Middleware.FollowRedirects, Pleroma.Tesla.Middleware.EncodeUrl]
do: [
Tesla.Middleware.FollowRedirects,
Pleroma.Tesla.Middleware.EncodeUrl,
Pleroma.Tesla.Middleware.MaxBody
]
end

253
lib/pleroma/http/adapter/gun.ex Normal file
View file

@ -0,0 +1,253 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.HTTP.Adapter.Gun do
@behaviour Tesla.Adapter
alias Pleroma.Gun.ConnectionPool
alias Tesla.Multipart
@default_timeout 1_000
@impl Tesla.Adapter
def call(env, opts) do
adapter_opts =
Tesla.Adapter.opts(
[close_conn: true, body_as: :plain, send_body: :at_once, receive: true],
env,
opts
)
|> Map.new()
with {:ok, status, headers, body} <- request(env, adapter_opts) do
env = %{env | status: status, headers: format_headers(headers), body: body}
{:ok, put_stream_ref(env, body)}
end
end
defp put_stream_ref(env, %{stream: stream}) do
put_in(env.opts[:adapter][:stream], stream)
end
defp put_stream_ref(env, _body), do: env
defp request(env, %{conn: conn, tunnel: tunnel} = opts) do
uri = URI.parse(Tesla.build_url(env))
path = request_path(uri, tunnel)
method = Tesla.Adapter.Shared.format_method(env.method)
{headers, body, send_body} = prepare_body(env.headers, env.body, opts[:send_body])
headers = maybe_add_forward_proxy_headers(headers, uri, opts, tunnel)
request_opts = request_opts(opts[:reply_to], tunnel)
stream =
try do
open_stream(conn, method, path, headers, body, request_opts, send_body)
catch
kind, reason ->
ConnectionPool.release_conn(conn)
:erlang.raise(kind, reason, __STACKTRACE__)
end
:ok = ConnectionPool.register_stream(conn, stream)
:ok = send_stream_body(conn, stream, body, send_body)
response = read_response(conn, stream, opts)
case response do
{:error, reason} ->
ConnectionPool.cancel_stream(conn, stream)
{:error, {:gun_stream_error, reason}}
response ->
unless opts[:body_as] in [:chunks, :stream] and stream_body?(response) do
:ok = ConnectionPool.finish_stream(conn, stream)
end
if opts[:close_conn] and opts[:body_as] not in [:stream, :chunks] do
:gun.close(conn)
end
response
end
end
defp stream_body?({:ok, _status, _headers, %{stream: _stream}}), do: true
defp stream_body?({:ok, _status, _headers, %Stream{}}), do: true
defp stream_body?(_response), do: false
defp request_opts(reply_to, nil), do: %{reply_to: reply_to || self()}
defp request_opts(reply_to, :forward_proxy), do: %{reply_to: reply_to || self()}
defp request_opts(reply_to, tunnel) do
%{reply_to: reply_to || self(), tunnel: tunnel}
end
defp request_path(uri, :forward_proxy) do
path = Tesla.Adapter.Shared.prepare_path(uri.path, uri.query)
"#{uri.scheme}://#{authority(uri)}#{path}"
end
defp request_path(uri, _tunnel) do
Tesla.Adapter.Shared.prepare_path(uri.path, uri.query)
end
defp maybe_add_forward_proxy_headers(headers, uri, opts, :forward_proxy) do
headers
|> put_header_new("host", authority(uri))
|> maybe_put_proxy_authorization(opts)
end
defp maybe_add_forward_proxy_headers(headers, _uri, _opts, _tunnel), do: headers
defp maybe_put_proxy_authorization(headers, %{proxy_auth: {username, password}})
when is_binary(username) and is_binary(password) do
value = "Basic " <> Base.encode64(username <> ":" <> password)
List.keystore(headers, "proxy-authorization", 0, {"proxy-authorization", value})
end
defp maybe_put_proxy_authorization(headers, _opts), do: headers
defp put_header_new(headers, name, value) do
if List.keymember?(headers, name, 0), do: headers, else: [{name, value} | headers]
end
defp authority(%URI{scheme: scheme, host: host, port: port}) do
host = if String.contains?(host, ":"), do: "[#{host}]", else: host
if port == URI.default_port(scheme), do: host, else: "#{host}:#{port}"
end
defp prepare_body(headers, %Multipart{} = multipart, _send_body) do
{format_headers(headers ++ Multipart.headers(multipart)), Multipart.body(multipart), :stream}
end
defp prepare_body(headers, body, _send_body)
when is_struct(body, Stream) or is_function(body) do
{format_headers(headers), body, :stream}
end
defp prepare_body(headers, body, send_body) do
{format_headers(headers), body || "", send_body}
end
defp open_stream(conn, method, path, headers, _body, request_opts, :stream) do
:gun.headers(conn, method, path, headers, request_opts)
end
defp open_stream(conn, method, path, headers, body, request_opts, :at_once) do
:gun.request(conn, method, path, headers, body, request_opts)
end
defp send_stream_body(conn, stream, body, :stream) do
try do
for data <- body, do: :ok = :gun.data(conn, stream, :nofin, data)
:gun.data(conn, stream, :fin, "")
catch
kind, reason ->
ConnectionPool.cancel_stream(conn, stream)
:erlang.raise(kind, reason, __STACKTRACE__)
end
end
defp send_stream_body(_conn, _stream, _body, :at_once), do: :ok
defp read_response(conn, stream, opts) do
receive? = opts[:receive]
receive do
{:gun_response, ^conn, ^stream, :fin, status, headers} ->
{:ok, status, headers, ""}
{:gun_response, ^conn, ^stream, :nofin, status, headers} ->
format_response(conn, stream, opts, status, headers, opts[:body_as])
{:gun_up, ^conn, _protocol} when receive? ->
read_response(conn, stream, opts)
{:gun_error, ^conn, reason} ->
{:error, reason}
{:gun_error, ^conn, ^stream, reason} ->
{:error, reason}
{:gun_down, ^conn, _protocol, _reason, _killed_streams} when receive? ->
read_response(conn, stream, opts)
{:DOWN, _ref, :process, ^conn, reason} ->
{:error, reason}
after
opts[:timeout] || @default_timeout -> {:error, :recv_response_timeout}
end
end
defp format_response(conn, stream, opts, status, headers, :plain) do
case read_body(conn, stream, opts) do
{:ok, body} ->
{:ok, status, headers, body}
{:error, error} ->
:ok = :gun.flush(stream)
{:error, error}
end
end
defp format_response(conn, stream, opts, status, headers, :stream) do
body =
Stream.resource(
fn -> :reading end,
fn
:reading ->
case Tesla.Adapter.Gun.read_chunk(conn, stream, opts) do
{:nofin, part} -> {[part], :reading}
{:fin, part} -> {[part], :done}
{:error, reason} -> raise "Gun stream failed: #{inspect(reason)}"
end
:done ->
{:halt, :done}
end,
fn _state ->
if opts[:close_conn], do: :gun.close(conn)
end
)
{:ok, status, headers, body}
end
defp format_response(conn, stream, opts, status, headers, :chunks) do
{:ok, status, headers, %{pid: conn, stream: stream, opts: Map.to_list(opts)}}
end
defp read_body(conn, stream, opts, acc \\ "") do
receive do
{:gun_data, ^conn, ^stream, :fin, body} ->
append_body(acc, body, opts[:max_body])
{:gun_data, ^conn, ^stream, :nofin, body} ->
with {:ok, acc} <- append_body(acc, body, opts[:max_body]) do
read_body(conn, stream, opts, acc)
end
{:gun_error, ^conn, ^stream, reason} ->
{:error, reason}
{:DOWN, _ref, :process, ^conn, reason} ->
{:error, reason}
after
opts[:timeout] || @default_timeout -> {:error, :recv_body_timeout}
end
end
defp append_body(acc, part, nil), do: {:ok, acc <> part}
defp append_body(acc, part, limit) do
body = acc <> part
if byte_size(body) <= limit, do: {:ok, body}, else: {:error, :body_too_large}
end
defp format_headers(headers) do
Enum.map(headers, fn {key, value} ->
{String.downcase(to_string(key)), to_string(value)}
end)
end
end

18
lib/pleroma/http/adapter_helper.ex
View file

@ -46,6 +46,24 @@ defmodule Pleroma.HTTP.AdapterHelper do
|> adapter_helper().options(uri)
end
@doc """
The configured Tesla adapter with TLS options, for clients that do not go
through `Pleroma.HTTP`.
"""
@spec tesla_adapter() :: module() | {module(), keyword()}
def tesla_adapter do
case adapter() do
Tesla.Adapter.Hackney ->
{Tesla.Adapter.Hackney, Pleroma.HTTP.TLS.put_hackney_ssl_options([])}
Tesla.Adapter.Gun ->
{Tesla.Adapter.Gun, certificates_verification: true}
adapter ->
adapter
end
end
defp adapter, do: Application.get_env(:tesla, :adapter)
defp adapter_helper do

14
lib/pleroma/http/adapter_helper/gun.ex
View file

@ -28,6 +28,7 @@ defmodule Pleroma.HTTP.AdapterHelper.Gun do
@defaults
|> Keyword.merge(config_opts)
|> ssl_options_to_tls_opts()
|> add_scheme_opts(uri)
|> AdapterHelper.maybe_add_proxy(proxy)
|> Keyword.merge(incoming_opts)
@ -35,6 +36,15 @@ defmodule Pleroma.HTTP.AdapterHelper.Gun do
|> maybe_stream()
end
# [:http, :adapter, :ssl_options] is Hackney's name for the TLS options,
# and AdminFE sets it. Keep those settings when Gun is the adapter.
defp ssl_options_to_tls_opts(opts) do
case Keyword.pop(opts, :ssl_options) do
{nil, opts} -> opts
{ssl_options, opts} -> Keyword.put_new(opts, :tls_opts, ssl_options)
end
end
defp add_scheme_opts(opts, %{scheme: "http"}), do: opts
defp add_scheme_opts(opts, %{scheme: "https"}) do
@ -48,10 +58,10 @@ defmodule Pleroma.HTTP.AdapterHelper.Gun do
Keyword.put(opts, :timeout, recv_timeout)
end
# Gun uses [body_as: :stream]
# Keep the pool lease until the caller consumes or cancels the body.
defp maybe_stream(opts) do
case Keyword.pop(opts, :stream, nil) do
{true, opts} -> Keyword.put(opts, :body_as, :stream)
{true, opts} -> Keyword.put(opts, :body_as, :chunks)
{_, opts} -> opts
end
end

19
lib/pleroma/http/adapter_helper/hackney.ex
View file

@ -28,6 +28,25 @@ defmodule Pleroma.HTTP.AdapterHelper.Hackney do
|> add_scheme_opts(uri)
|> maybe_add_with_body()
|> Pleroma.HTTP.AdapterHelper.maybe_add_proxy(proxy)
|> Keyword.put(:follow_redirect, false)
|> Keyword.put(:force_redirect, false)
|> normalize_proxy()
|> Pleroma.HTTP.TLS.put_hackney_ssl_options()
end
# Hackney 4 ignores a proxy whose host is an IP tuple and connects directly.
# ConfigDB turns a numeric proxy host into such a tuple.
defp normalize_proxy(opts) do
case Keyword.fetch(opts, :proxy) do
{:ok, {type, host, port}} when is_tuple(host) ->
Keyword.put(opts, :proxy, {type, :inet.ntoa(host), port})
{:ok, {host, port}} when is_tuple(host) ->
Keyword.put(opts, :proxy, {:inet.ntoa(host), port})
_ ->
opts
end
end
defp add_scheme_opts(opts, _), do: opts

49
lib/pleroma/http/tls.ex Normal file
View file

@ -0,0 +1,49 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.HTTP.TLS do
@moduledoc """
TLS client options that trust the CAStore bundle.
Hackney otherwise trusts its own certifi bundle. Hackney 1 drops all of its
default TLS options, including certificate checks, as soon as any
`ssl_options` are given. Hackney 4 merges them, so `partial_chain` must be
set too: Hackney's own one trusts any certificate in the peer chain that is
in certifi.
"""
@spec ssl_options(keyword()) :: keyword()
def ssl_options(overrides \\ []) do
defaults = [
verify: :verify_peer,
cacertfile: CAStore.file_path(),
depth: 20,
customize_hostname_check: [match_fun: :public_key.pkix_verify_hostname_match_fun(:https)],
partial_chain: &__MODULE__.reject_partial_chain/1
]
# Hackney drops cacerts when cacertfile is set too.
defaults =
if Keyword.has_key?(overrides, :cacerts),
do: Keyword.delete(defaults, :cacertfile),
else: defaults
Keyword.merge(defaults, overrides)
end
@doc false
def reject_partial_chain(_certs), do: :unknown_ca
@spec put_hackney_ssl_options(keyword()) :: keyword()
def put_hackney_ssl_options(opts) do
# Hackney 4 ignores the top-level insecure option and falls back to certifi.
if opts[:insecure] do
Keyword.put(opts, :ssl_options, verify: :verify_none)
else
opts
|> Keyword.put(:ssl_options, ssl_options(Keyword.get(opts, :ssl_options, [])))
|> Keyword.put(:proxy_ssl_options, ssl_options(Keyword.get(opts, :proxy_ssl_options, [])))
end
end
end

42
lib/pleroma/list.ex
View file

@ -9,6 +9,7 @@ defmodule Pleroma.List do
import Ecto.Changeset
alias Pleroma.Activity
alias Pleroma.Emoji
alias Pleroma.Repo
alias Pleroma.User
@ -18,14 +19,53 @@ defmodule Pleroma.List do
field(:following, {:array, :string}, default: [])
field(:ap_id, :string)
field(:exclusive, :boolean, default: false)
field(:emoji, :string)
timestamps()
end
def update_changeset(list, attrs \\ %{}) do
list
|> cast(attrs, [:title, :exclusive])
|> cast(attrs, [:title, :exclusive, :emoji])
|> validate_required([:title])
|> fix_emoji()
|> validate_emoji()
end
defp fix_emoji(changeset) do
with {:emoji_field, emoji} when is_binary(emoji) <-
{:emoji_field, get_field(changeset, :emoji)},
{:fixed_emoji, emoji} <-
{:fixed_emoji,
emoji
|> Emoji.fully_qualify_emoji()
|> Emoji.maybe_quote()} do
put_change(changeset, :emoji, emoji)
else
{:emoji_field, _} -> changeset
end
end
defp validate_emoji(changeset) do
validate_change(changeset, :emoji, fn
:emoji, nil ->
[]
:emoji, emoji ->
if Emoji.unicode?(emoji) or valid_local_custom_emoji?(emoji) do
[]
else
[emoji: "Invalid emoji"]
end
end)
end
defp valid_local_custom_emoji?(emoji) do
with %{file: _path} <- Emoji.get(emoji) do
true
else
_ -> false
end
end
def follow_changeset(list, attrs \\ %{}) do

10
lib/pleroma/moderation_log.ex
View file

@ -346,6 +346,16 @@ defmodule Pleroma.ModerationLog do
"@#{actor_nickname} approved users: #{users_to_nicknames_string(users)}"
end
def get_log_entry_message(%ModerationLog{
data: %{
"actor" => %{"nickname" => actor_nickname},
"action" => "reject",
"subject" => users
}
}) do
"@#{actor_nickname} rejected users: #{users_to_nicknames_string(users)}"
end
def get_log_entry_message(%ModerationLog{
data: %{
"actor" => %{"nickname" => actor_nickname},

101
lib/pleroma/notification.ex
View file

@ -35,6 +35,7 @@ defmodule Pleroma.Notification do
# remember to add a migration to add it to the `notifications_type` enum
# as well.
field(:type, :string)
field(:group_key, :string)
belongs_to(:user, User, type: FlakeId.Ecto.CompatType)
belongs_to(:activity, Activity, type: FlakeId.Ecto.CompatType)
@ -49,7 +50,7 @@ defmodule Pleroma.Notification do
|> type_from_activity()
notification
|> changeset(%{type: type})
|> changeset(%{type: type, group_key: grouped_notification_key(type, activity)})
|> Repo.update()
end
end
@ -62,6 +63,99 @@ defmodule Pleroma.Notification do
|> Repo.aggregate(:count, :id)
end
@groupable_notification_types ~w{favourite follow reblog}
@group_bucket_seconds 12 * 60 * 60
def groupable_notification_types, do: @groupable_notification_types
def group_notifications(notifications, grouped_types \\ nil) do
grouped_types = normalize_grouped_types(grouped_types)
{group_keys, grouped_notifications} =
Enum.reduce(notifications, {[], %{}}, fn notification,
{group_keys, grouped_notifications} ->
group_key = group_key(notification, grouped_types)
if Map.has_key?(grouped_notifications, group_key) do
{group_keys, Map.update!(grouped_notifications, group_key, &[notification | &1])}
else
{[group_key | group_keys], Map.put(grouped_notifications, group_key, [notification])}
end
end)
group_keys
|> Enum.reverse()
|> Enum.map(fn group_key ->
grouped_notifications
|> Map.fetch!(group_key)
|> Enum.reverse()
end)
end
def group_key(notification, grouped_types \\ nil)
def group_key(%Notification{} = notification, nil) do
group_key(notification, @groupable_notification_types)
end
def group_key(%Notification{type: type, group_key: group_key} = notification, grouped_types) do
grouped_types = normalize_grouped_types(grouped_types)
if type in @groupable_notification_types and type in grouped_types and is_binary(group_key) do
group_key
else
ungrouped_group_key(notification)
end
end
def normalize_grouped_types(nil), do: @groupable_notification_types
def normalize_grouped_types(types) when is_list(types), do: types
def normalize_grouped_types(type), do: [type]
defp ungrouped_group_key(%Notification{id: id}), do: "ungrouped-#{id}"
defp grouped_notification_key(type, activity) when type in @groupable_notification_types do
with target_id when is_binary(target_id) <- group_target_id(type, activity) do
# Mastodon uses Redis to reuse a recent bucket for rolling 12h windows. Pleroma keeps keys
# deterministic and self-contained; clients must treat group_key as opaque either way.
"#{type}-#{target_id}-#{group_time_bucket(activity)}"
end
end
defp grouped_notification_key(_type, _activity), do: nil
defp group_time_bucket(%Activity{inserted_at: inserted_at}) when not is_nil(inserted_at) do
inserted_at
|> NaiveDateTime.to_erl()
|> :calendar.datetime_to_gregorian_seconds()
|> div(@group_bucket_seconds)
end
defp group_time_bucket(_),
do: group_time_bucket(%Activity{inserted_at: NaiveDateTime.utc_now()})
defp group_target_id(type, activity) when type in ["favourite", "reblog"] do
with object_id when is_binary(object_id) <- object_id_for(activity),
%Activity{id: id} <- Activity.get_create_by_object_ap_id(object_id) do
to_string(id)
else
_ -> nil
end
end
defp group_target_id("follow", %{data: %{"object" => ap_id}}) do
case User.get_cached_by_ap_id(ap_id) do
%User{id: id} -> to_string(id)
_ -> nil
end
end
defp group_target_id(_, _), do: nil
defp object_id_for(%{data: %{"object" => %{"id" => id}}}) when is_binary(id), do: id
defp object_id_for(%{data: %{"object" => id}}) when is_binary(id), do: id
defp object_id_for(_), do: nil
@notification_types ~w{
favourite
follow
@ -79,7 +173,7 @@ defmodule Pleroma.Notification do
def changeset(%Notification{} = notification, attrs) do
notification
|> cast(attrs, [:seen, :type])
|> cast(attrs, [:seen, :type, :group_key])
|> validate_inclusion(:type, @notification_types)
end
@ -458,7 +552,8 @@ defmodule Pleroma.Notification do
user_id: user.id,
activity: activity,
seen: mark_as_read?(activity, user),
type: type
type: type,
group_key: grouped_notification_key(type, activity)
})
|> Marker.multi_set_last_read_id(user, "notifications")
|> Repo.transaction()

36
lib/pleroma/object/containment.ex
View file

@ -15,12 +15,28 @@ defmodule Pleroma.Object.Containment do
actor
end
def get_actor(%{"actor" => nil, "attributedTo" => actor}) when not is_nil(actor) do
get_actor(%{"actor" => actor})
end
def get_actor(%{"actor" => nil}), do: nil
def get_actor(%{"actor" => []}), do: nil
def get_actor(%{"actor" => actor}) when is_list(actor) do
if is_binary(Enum.at(actor, 0)) do
Enum.at(actor, 0)
else
Enum.find(actor, fn %{"type" => type} -> type in ["Person", "Service", "Application"] end)
|> Map.get("id")
case actor do
[first | _] when is_binary(first) ->
first
_ ->
Enum.find_value(actor, fn
%{"type" => type, "id" => id}
when type in ["Person", "Service", "Application"] and is_binary(id) ->
id
_ ->
nil
end)
end
end
@ -28,10 +44,6 @@ defmodule Pleroma.Object.Containment do
id
end
def get_actor(%{"actor" => nil, "attributedTo" => actor}) when not is_nil(actor) do
get_actor(%{"actor" => actor})
end
def get_object(%{"object" => id}) when is_binary(id) do
id
end
@ -67,9 +79,11 @@ defmodule Pleroma.Object.Containment do
def contain_origin(id, %{"actor" => _actor} = params) do
id_uri = URI.parse(id)
actor_uri = URI.parse(get_actor(params))
compare_uris(actor_uri, id_uri)
case get_actor(params) do
actor when is_binary(actor) -> compare_uris(URI.parse(actor), id_uri)
_ -> :error
end
end
def contain_origin(id, %{"attributedTo" => actor} = params),

207
lib/pleroma/reverse_proxy.ex
View file

@ -12,7 +12,7 @@ defmodule Pleroma.ReverseProxy do
@keep_resp_headers @resp_cache_headers ++
~w(content-length content-type content-disposition content-encoding) ++
~w(content-range accept-ranges vary)
@default_cache_control_header "public, max-age=1209600, immutable"
@default_cache_control_header "public, max-age=1209600, immutable, no-transform"
@valid_resp_codes [200, 206, 304]
@max_read_duration :timer.seconds(30)
@max_body_length :infinity
@ -23,6 +23,16 @@ defmodule Pleroma.ReverseProxy do
@cachex Pleroma.Config.get([:cachex, :provider], Cachex)
defmodule StreamError do
@moduledoc false
defexception [:url, :reason]
@impl true
def message(%{url: url, reason: reason}) do
"reverse proxy stream from #{url} failed: #{inspect(reason)}"
end
end
def max_read_duration_default, do: @max_read_duration
def default_cache_control_header, do: @default_cache_control_header
@ -69,11 +79,15 @@ defmodule Pleroma.ReverseProxy do
@default_options [pool: :media]
@inline_content_types [
"image/apng",
"image/avif",
"image/bmp",
"image/gif",
"image/jpeg",
"image/jpg",
"image/png",
"image/svg+xml",
"image/webp",
"audio/mpeg",
"audio/mp3",
"video/webm",
@ -98,7 +112,13 @@ defmodule Pleroma.ReverseProxy do
def call(_conn, _url, _opts \\ [])
def call(conn = %{method: method}, url, opts) when method in @methods do
client_opts = Keyword.merge(@default_options, Keyword.get(opts, :http, []))
client_opts =
@default_options
|> Keyword.merge(Keyword.get(opts, :http, []))
|> Keyword.put_new(
:recv_timeout,
receive_timeout(opts)
)
req_headers = build_req_headers(conn.req_headers, opts)
@ -110,12 +130,8 @@ defmodule Pleroma.ReverseProxy do
end
with {:ok, nil} <- @cachex.get(:failed_proxy_url_cache, url),
{:ok, code, headers, client} <- request(method, url, req_headers, client_opts),
:ok <-
header_length_constraint(
headers,
Keyword.get(opts, :max_body_length, @max_body_length)
) do
{:ok, code, headers, client} <-
request_with_constraints(method, url, req_headers, client_opts, opts) do
response(conn, client, url, code, headers, opts)
else
{:ok, true} ->
@ -170,7 +186,8 @@ defmodule Pleroma.ReverseProxy do
{:ok, code, headers} when code in @valid_resp_codes ->
{:ok, code, downcase_headers(headers)}
{:ok, code, _, _} ->
{:ok, code, _, client} ->
client().close(client)
{:error, {:invalid_http_response, code}}
{:ok, code, _} ->
@ -181,13 +198,38 @@ defmodule Pleroma.ReverseProxy do
end
end
defp request_with_constraints(method, url, headers, client_opts, opts) do
method
|> request(url, headers, client_opts)
|> constrain_response(Keyword.get(opts, :max_body_length, @max_body_length))
end
defp constrain_response({:ok, code, headers, client}, limit) do
case header_length_constraint(headers, limit) do
{:ok, headers} ->
{:ok, code, headers, client}
error ->
client().close(client)
error
end
end
defp constrain_response({:ok, code, headers}, limit) do
case header_length_constraint(headers, limit) do
{:ok, headers} -> {:ok, code, headers}
error -> error
end
end
defp constrain_response(response, _limit), do: response
defp response(conn, client, url, status, headers, opts) do
Logger.debug("#{__MODULE__} #{status} #{url} #{inspect(headers)}")
result =
conn
|> put_resp_headers(build_resp_headers(headers, opts))
|> streaming_compat
|> send_chunked(status)
|> chunk_reply(client, opts)
@ -195,17 +237,18 @@ defmodule Pleroma.ReverseProxy do
{:ok, conn} ->
halt(conn)
{:error, :closed, conn} ->
{:error, {:downstream, _error}, conn} ->
client().close(client)
halt(conn)
{:error, error, conn} ->
{:error, {:upstream, error}, conn} ->
Logger.warning(
"#{__MODULE__} request to #{url} failed while reading/chunking: #{inspect(error)}"
)
client().close(client)
halt(conn)
track_failed_url(url, error, opts)
raise_stream_error(conn, url, error)
end
end
@ -227,11 +270,19 @@ defmodule Pleroma.ReverseProxy do
sent_so_far,
Keyword.get(opts, :max_body_length, @max_body_length)
),
{:ok, conn} <- chunk(conn, data) do
{:ok, conn} <- write_chunk(conn, data) do
chunk_reply(conn, client, opts, sent_so_far, duration)
else
:done -> {:ok, conn}
{:error, error} -> {:error, error, conn}
{:error, {source, error}} -> {:error, {source, error}, conn}
{:error, error} -> {:error, {:upstream, error}, conn}
end
end
defp write_chunk(conn, data) do
case chunk(conn, data) do
{:error, error} -> {:error, {:downstream, error}}
result -> result
end
end
@ -281,7 +332,7 @@ defmodule Pleroma.ReverseProxy do
|> Enum.filter(fn {k, _} -> k in @keep_req_headers end)
|> build_req_range_or_encoding_header(opts)
|> build_req_user_agent_header(opts)
|> Keyword.merge(Keyword.get(opts, :req_headers, []))
|> merge_headers(Keyword.get(opts, :req_headers, []))
end
# Disable content-encoding if any @range_headers are requested (see #1823).
@ -310,7 +361,30 @@ defmodule Pleroma.ReverseProxy do
|> build_resp_cache_headers(opts)
|> sanitise_content_type()
|> build_resp_content_disposition_header(opts)
|> Keyword.merge(Keyword.get(opts, :resp_headers, []))
|> merge_headers(Keyword.get(opts, :resp_headers, []))
|> ensure_no_transform()
end
defp merge_headers(headers, extra_headers) do
Enum.reduce(extra_headers, headers, fn {key, value}, headers ->
List.keystore(headers, String.downcase(key), 0, {String.downcase(key), value})
end)
end
defp ensure_no_transform(headers) do
{_, cache_control} =
List.keyfind(headers, "cache-control", 0, {"cache-control", @default_cache_control_header})
directives =
cache_control
|> String.split(",")
|> Enum.map(&(&1 |> String.trim() |> String.downcase()))
if "no-transform" in directives do
headers
else
replace_header(headers, "cache-control", cache_control <> ", no-transform")
end
end
defp sanitise_content_type(headers) do
@ -384,29 +458,46 @@ defmodule Pleroma.ReverseProxy do
disposition = "attachment; filename=\"#{name}\""
List.keystore(headers, "content-disposition", 0, {"content-disposition", disposition})
replace_header(headers, "content-disposition", disposition)
else
headers
end
end
defp header_length_constraint(headers, limit) when is_integer(limit) and limit > 0 do
with {_, size} <- List.keyfind(headers, "content-length", 0),
{size, _} <- Integer.parse(size),
true <- size <= limit do
:ok
else
false ->
{:error, :body_too_large}
defp replace_header(headers, key, value) do
[{key, value} | Enum.reject(headers, fn {header, _} -> header == key end)]
end
defp header_length_constraint(headers, limit) do
lengths =
for {"content-length", value} <- headers,
do: parse_content_length(value)
case Enum.uniq(lengths) do
[] ->
{:ok, headers}
[size] when is_integer(size) ->
case body_size_constraint(size, limit) do
:ok -> {:ok, replace_header(headers, "content-length", to_string(size))}
error -> error
end
_ ->
:ok
{:error, :invalid_content_length}
end
end
defp header_length_constraint(_, _), do: :ok
defp parse_content_length(value) when is_binary(value) do
case Integer.parse(value) do
{size, ""} when size >= 0 -> size
_ -> :invalid
end
end
defp body_size_constraint(size, limit) when is_integer(limit) and limit > 0 and size >= limit do
defp parse_content_length(_value), do: :invalid
defp body_size_constraint(size, limit) when is_integer(limit) and limit > 0 and size > limit do
{:error, :body_too_large}
end
@ -421,7 +512,16 @@ defmodule Pleroma.ReverseProxy do
end
end
defp check_read_duration(_, _), do: {:ok, :no_duration_limit, :no_duration_limit}
defp check_read_duration(_, _), do: {:ok, :no_duration_limit}
defp receive_timeout(opts) do
case Keyword.get(opts, :max_read_duration, @max_read_duration) do
timeout when is_integer(timeout) and timeout > 0 -> timeout
_ -> :infinity
end
end
defp increase_read_duration(:no_duration_limit), do: {:ok, :no_duration_limit}
defp increase_read_duration({previous_duration, started})
when is_integer(previous_duration) and is_integer(started) do
@ -431,6 +531,40 @@ defmodule Pleroma.ReverseProxy do
defp client, do: Pleroma.ReverseProxy.Client.Wrapper
# Neither Plug adapter exposes a public way to abort a committed response.
# Cowboy finalizes chunked HTTP/1 responses when the request process exits,
# so terminate the connection first to leave the response visibly incomplete.
defp raise_stream_error(
%Plug.Conn{adapter: {Plug.Cowboy.Conn, %{pid: connection_pid}}} = conn,
url,
error
) do
if Plug.Conn.get_http_protocol(conn) in [:"HTTP/1.0", :"HTTP/1.1"] do
Process.exit(connection_pid, :kill)
end
raise StreamError, url: url, reason: error
end
# Bandit turns this exception into RST_STREAM; a regular exception sends an
# empty DATA frame with END_STREAM and makes the partial body look complete.
defp raise_stream_error(
%Plug.Conn{
adapter: {Bandit.Adapter, %{transport: %{stream_id: stream_id}}}
},
url,
error
) do
raise Bandit.HTTP2.Errors.StreamError,
message: StreamError.message(%StreamError{url: url, reason: error}),
error_code: Bandit.HTTP2.Errors.internal_error(),
stream_id: stream_id
end
defp raise_stream_error(_conn, url, error) do
raise StreamError, url: url, reason: error
end
defp track_failed_url(url, error, opts) do
ttl =
unless error in [:body_too_large, 400, 204] do
@ -442,19 +576,6 @@ defmodule Pleroma.ReverseProxy do
@cachex.put(:failed_proxy_url_cache, url, true, ttl: ttl)
end
# When Cowboy handles a chunked response with a content-length header it streams
# over HTTP 1.1 instead of chunking. Bandit cannot stream over HTTP 1.1 so the header
# must be stripped or it breaks RFC compliance for Transfer Encoding: Chunked. RFC9112§6.2
#
# HTTP2 is always streamed for all adapters.
defp streaming_compat(conn) do
with Phoenix.Endpoint.Cowboy2Adapter <- Pleroma.Web.Endpoint.config(:adapter) do
conn
else
_ -> delete_resp_header(conn, "content-length")
end
end
# Only when Tesla adapter is Hackney or Finch does the URL
# need encoding before Reverse Proxying as both end up
# using the raw Hackney client and cannot leverage our

133
lib/pleroma/reverse_proxy/client/hackney.ex
View file

@ -28,17 +28,18 @@ defmodule Pleroma.ReverseProxy.Client.Hackney do
@impl true
def request(method, url, headers, body, opts \\ []) do
opts =
Keyword.put_new(opts, :path_encode_fun, fn path ->
path
end)
opts
|> Keyword.put_new(:path_encode_fun, fn path -> path end)
|> Pleroma.HTTP.TLS.put_hackney_ssl_options()
if opts[:follow_redirect] != false do
{_state, req_opts} = Access.get_and_update(opts, :follow_redirect, fn a -> {a, false} end)
env = %{method: method, headers: headers, body: body, req_opts: req_opts}
res = :hackney.request(method, url, headers, body, req_opts)
res = do_request(method, url, headers, body, req_opts)
case res do
{:ok, code, resp_headers, _client} when code in @redirect_statuses ->
{:ok, code, resp_headers, client} when code in @redirect_statuses ->
close(client)
redirect(url, resp_headers, env, @redirect_limit)
{:ok, code, resp_headers} when code in @redirect_statuses ->
@ -48,11 +49,34 @@ defmodule Pleroma.ReverseProxy.Client.Hackney do
res
end
else
:hackney.request(method, url, headers, body, opts)
do_request(method, url, headers, body, opts)
end
end
@impl true
def stream_body(%{released: true}), do: :done
def stream_body(%{ref: ref, recv_timeout: recv_timeout} = client) do
_ = :hackney.stream_next(ref)
case receive_response(ref, recv_timeout) do
{:ok, data} when is_binary(data) ->
{:ok, data, client}
# Reaching the end of the body does not free the connection: Hackney parks
# a finished conn instead of stopping it, and an unpooled one has neither
# an idle timeout nor a live owner monitor to reap it, so it would sit
# there forever pinning every binary it read. Release it here, mirroring
# what the gun client does, so callers only close what they abandon.
{:ok, :done} ->
close(client)
:done
{:error, error} ->
{:error, error}
end
end
def stream_body(ref) do
case :hackney.stream_body(ref) do
:done -> :done
@ -62,8 +86,98 @@ defmodule Pleroma.ReverseProxy.Client.Hackney do
end
@impl true
def close(%{released: true}), do: :ok
def close(%{ref: ref}), do: :hackney.close(ref)
def close(ref), do: :hackney.close(ref)
defp do_request(method, url, headers, body, opts) do
recv_timeout = Keyword.get(opts, :recv_timeout, :infinity)
req_opts =
opts
|> Keyword.put(:async, :once)
|> Keyword.put(:stream_to, self())
with {:ok, ref} <- :hackney.request(method, url, headers, body, req_opts) do
result =
with {:ok, {:status, code, _reason}} <- receive_response(ref, recv_timeout),
{:ok, {:headers, resp_headers}} <- receive_response(ref, recv_timeout) do
{:ok, code, resp_headers, build_client(ref, recv_timeout, method, code, opts)}
end
case result do
{:ok, _, _, _} = response ->
response
error ->
close(ref)
flush_responses(ref)
error
end
end
end
# A response that cannot carry a body (204, 304, or a reply to HEAD) is
# complete as soon as the headers are parsed. Hackney queues `done` for us and,
# if the connection is pooled, checks it straight back in, so the ref stops
# being ours at that point: pulling the stream or stopping the connection
# afterwards would disturb whichever request checks it out next. Mark it so
# `stream_body/1` and `close/1` leave it alone.
#
# This relies on hackney never running its own redirect handling for us --
# `request/5` forces `follow_redirect: false` -- otherwise a redirect status
# would be checked back in too and would need the same treatment.
defp build_client(ref, recv_timeout, method, code, opts) do
client = %{ref: ref, recv_timeout: recv_timeout}
if released?(method, code) and pooled?(opts) do
flush_responses(ref)
Map.put(client, :released, true)
else
client
end
end
defp released?(method, _code) when method in [:head, :HEAD, "head", "HEAD"], do: true
defp released?(_method, code) when code in [204, 304], do: true
defp released?(_method, _code), do: false
# Mirrors hackney's own pooling decision: :hackney.use_pool/1, plus the fact
# that a proxy tunnel always bypasses the pool. An unpooled connection is never
# checked in, and parks in `connected` with no idle timeout, so it leaks
# forever unless we stop it ourselves.
defp pooled?(opts) do
no_proxy?(Keyword.get(opts, :proxy)) and use_pool?(Keyword.get(opts, :pool))
end
# Hackney reads `false` as "no proxy", same as an absent option.
defp no_proxy?(nil), do: true
defp no_proxy?(false), do: true
defp no_proxy?(_proxy), do: false
defp use_pool?(false), do: false
defp use_pool?(nil), do: Application.get_env(:hackney, :use_default_pool, true)
defp use_pool?(_pool_name), do: true
defp receive_response(ref, recv_timeout) do
receive do
{:hackney_response, ^ref, {:error, error}} -> {:error, error}
{:hackney_response, ^ref, response} -> {:ok, response}
after
recv_timeout -> {:error, :timeout}
end
end
defp flush_responses(ref) do
receive do
{:hackney_response, ^ref, _response} -> flush_responses(ref)
after
0 -> :ok
end
end
defp redirect(url, resp_headers, env, limit) when limit == 0 do
new_url = absolute_redirect_url(url, resp_headers)
@ -71,17 +185,18 @@ defmodule Pleroma.ReverseProxy.Client.Hackney do
"#{__MODULE__}: Handling redirect #{url} -> #{new_url}; redirect limit was reached - returning response after final redirect"
)
:hackney.request(env.method, new_url, env.headers, env.body, env.req_opts)
do_request(env.method, new_url, env.headers, env.body, env.req_opts)
end
defp redirect(url, resp_headers, env, limit) do
new_url = absolute_redirect_url(url, resp_headers)
Logger.debug("#{__MODULE__}: handling redirect #{url} -> #{new_url}; limit = #{limit}")
res = :hackney.request(env.method, new_url, env.headers, env.body, env.req_opts)
res = do_request(env.method, new_url, env.headers, env.body, env.req_opts)
case res do
{:ok, code, new_resp_headers, _client} when code in @redirect_statuses ->
{:ok, code, new_resp_headers, client} when code in @redirect_statuses ->
close(client)
redirect(new_url, new_resp_headers, env, limit - 1)
{:ok, code, new_resp_headers} when code in @redirect_statuses ->

16
lib/pleroma/reverse_proxy/client/tesla.ex
View file

@ -6,6 +6,7 @@ defmodule Pleroma.ReverseProxy.Client.Tesla do
@behaviour Pleroma.ReverseProxy.Client
alias Pleroma.Gun.ConnectionPool
alias Pleroma.HTTP.AdapterHelper
@type headers() :: [{String.t(), String.t()}]
@type status() :: pos_integer()
@ -20,6 +21,13 @@ defmodule Pleroma.ReverseProxy.Client.Tesla do
def request(method, url, headers, body, opts \\ []) do
check_adapter()
# MediaProxyController inserts MediaProxy proxy configuration into the ReverseProxy call.
# This config can be unformatted and in String representation, while this works for Hackney,
# it does not work for Tesla and the proxy needs to be reformatted.
# Gun AdapterHelper doesn't do it, since it does not overwrite proxy config.
proxy = AdapterHelper.format_proxy(opts[:proxy])
opts = if proxy, do: Keyword.put(opts, :proxy, proxy), else: opts
opts = Keyword.put(opts, :body_as, :chunks)
with {:ok, response} <-
@ -45,8 +53,8 @@ defmodule Pleroma.ReverseProxy.Client.Tesla do
@impl true
@spec stream_body(map()) ::
{:ok, binary(), map()} | {:error, atom() | String.t()} | :done | no_return()
def stream_body(%{pid: pid, fin: true}) do
ConnectionPool.release_conn(pid)
def stream_body(%{pid: pid, stream: stream, fin: true}) do
ConnectionPool.release_stream(pid, stream)
:done
end
@ -70,6 +78,10 @@ defmodule Pleroma.ReverseProxy.Client.Tesla do
@impl true
@spec close(map) :: :ok | no_return()
def close(%{pid: pid, stream: stream}) do
ConnectionPool.cancel_stream(pid, stream)
end
def close(%{pid: pid}) do
ConnectionPool.release_conn(pid)
end

75
lib/pleroma/search.ex
View file

@ -5,13 +5,12 @@ defmodule Pleroma.Search do
alias Pleroma.Workers.SearchIndexingWorker
@spec add_to_index(Activity.t()) :: {:ok, Oban.Job.t() | :noop} | {:error, Oban.Job.changeset()}
def add_to_index(%Activity{id: activity_id, object: %Object{} = object} = activity) do
with {_, true} <- {:indexable, indexable?(activity)},
{_, "public"} <- {:visibility, Visibility.get_visibility(object)} do
def add_to_index(%Activity{id: activity_id, object: %Object{}} = activity) do
if indexable?(activity) do
SearchIndexingWorker.new(%{"op" => "add_to_index", "activity" => activity_id})
|> Oban.insert()
else
_ -> {:ok, :noop}
{:ok, :noop}
end
end
@ -41,40 +40,60 @@ defmodule Pleroma.Search do
def object_to_search_data(%Object{} = object) do
data = object.data
content_str =
case data["content"] do
[nil | rest] -> to_string(rest)
str -> str
end
content =
with {:ok, scrubbed} <-
FastSanitize.Sanitizer.scrub(content_str, Pleroma.HTML.Scrubber.SearchIndexing),
trimmed <- String.trim(scrubbed) do
trimmed
end
# Make sure we have a non-empty string
if content != "" do
{:ok, published, _} = DateTime.from_iso8601(data["published"])
data
|> search_texts()
|> Enum.map(&sanitize_search_text/1)
|> Enum.reject(&(&1 in ["", "."]))
|> Enum.join(" ")
with true <- content != "",
published when is_binary(published) <- data["published"],
{:ok, published, _} <- DateTime.from_iso8601(published) do
%{
id: object.id,
content: content,
ap: data["id"],
published: published |> DateTime.to_unix()
}
else
_ -> nil
end
end
defp indexable?(%Activity{
data: %{"type" => "Create"},
object: %Object{
data: %{"content" => content, "published" => published, "type" => "Note"}
}
})
when not is_nil(content) and content not in ["", "."] and not is_nil(published),
do: true
def indexable?(%Activity{
data: %{"type" => "Create"},
object: %Object{data: %{"published" => published, "type" => "Note"}} = object
})
when not is_nil(published) do
Visibility.get_visibility(object) in ["public", "unlisted"] and
not is_nil(object_to_search_data(object))
end
defp indexable?(_), do: false
def indexable?(_), do: false
defp search_texts(data) do
[data["content"], data["summary"] | attachment_names(data["attachment"])]
end
defp attachment_names(attachments) when is_list(attachments) do
Enum.map(attachments, fn
%{"summary" => summary} when is_binary(summary) -> summary
%{"name" => name} -> name
_ -> nil
end)
end
defp attachment_names(_), do: []
defp sanitize_search_text([nil | rest]), do: sanitize_search_text(to_string(rest))
defp sanitize_search_text(text) when is_binary(text) do
case FastSanitize.Sanitizer.scrub(text, Pleroma.HTML.Scrubber.SearchIndexing) do
{:ok, scrubbed} -> String.trim(scrubbed)
_ -> ""
end
end
defp sanitize_search_text(_), do: ""
end

137
lib/pleroma/search/backfill.ex Normal file
View file

@ -0,0 +1,137 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2026 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Search.Backfill do
import Ecto.Query
alias Pleroma.Activity
alias Pleroma.Workers.SearchIndexingWorker
@default_chunk_size 100
@default_step_size 1_000
@default_limit 100_000
@doc """
Enqueues existing Create activities for search indexing.
The returned `next_cursor` can be passed as `before` to resume a bounded run.
Pass `limit: :infinity` to continue until the oldest activity is reached.
"""
def run(options \\ []) do
options =
Keyword.validate!(options,
chunk: @default_chunk_size,
step: @default_step_size,
limit: @default_limit,
before: nil,
on_page: nil
)
chunk_size = positive_option!(options, :chunk, @default_chunk_size)
step_size = positive_option!(options, :step, @default_step_size)
limit = limit_option!(options)
before = cursor_option!(options[:before])
on_page = callback_option!(options[:on_page])
do_run(before, limit, step_size, chunk_size, on_page, 0)
end
defp do_run(before, remaining, step_size, chunk_size, on_page, enqueued) do
page_size = page_size(remaining, step_size)
ids = fetch_ids(before, page_size)
case ids do
[] ->
result(enqueued, before, true)
ids ->
enqueue(ids, chunk_size)
page_count = length(ids)
next_cursor = List.last(ids)
total = enqueued + page_count
remaining = decrement(remaining, page_count)
on_page.(%{count: page_count, enqueued: total, next_cursor: next_cursor})
cond do
remaining == 0 -> result(total, next_cursor, false)
page_count < page_size -> result(total, next_cursor, true)
true -> do_run(next_cursor, remaining, step_size, chunk_size, on_page, total)
end
end
end
defp fetch_ids(before, limit) do
Activity
|> where([a], fragment("?->>'type' = 'Create'", a.data))
|> maybe_before(before)
|> order_by([a], desc: a.id)
|> limit(^limit)
|> select([a], a.id)
|> Pleroma.Repo.all()
end
defp maybe_before(query, nil), do: query
defp maybe_before(query, before), do: where(query, [a], a.id < ^before)
defp enqueue(ids, chunk_size) do
ids
|> Enum.chunk_every(chunk_size)
|> Enum.each(fn chunk ->
chunk
|> Enum.map(&SearchIndexingWorker.new(%{"op" => "add_to_index", "activity" => &1}))
|> Oban.insert_all()
end)
end
defp page_size(:infinity, step_size), do: step_size
defp page_size(remaining, step_size), do: min(remaining, step_size)
defp decrement(:infinity, _count), do: :infinity
defp decrement(remaining, count), do: remaining - count
defp result(enqueued, next_cursor, exhausted) do
%{enqueued: enqueued, next_cursor: next_cursor, exhausted: exhausted}
end
defp positive_option!(options, key, default) do
case Keyword.get(options, key, default) do
value when is_integer(value) and value > 0 -> value
value -> raise ArgumentError, "#{key} must be a positive integer, got: #{inspect(value)}"
end
end
defp limit_option!(options) do
case Keyword.fetch!(options, :limit) do
:infinity -> :infinity
value when is_integer(value) and value > 0 -> value
value -> raise ArgumentError, "limit must be a positive integer, got: #{inspect(value)}"
end
end
defp cursor_option!(nil), do: nil
defp cursor_option!(cursor) do
valid? =
try do
match?({:ok, <<_::binary-size(16)>>}, FlakeId.Ecto.CompatType.dump(cursor))
rescue
_ -> false
end
if valid? do
cursor
else
raise ArgumentError, "before must be a valid activity id, got: #{inspect(cursor)}"
end
end
defp callback_option!(nil), do: fn _ -> :ok end
defp callback_option!(callback) when is_function(callback, 1), do: callback
defp callback_option!(callback) do
raise ArgumentError, "on_page must be a function of arity 1, got: #{inspect(callback)}"
end
end

287
lib/pleroma/search/parade_db.ex Normal file
View file

@ -0,0 +1,287 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Search.ParadeDB do
@behaviour Pleroma.Search.SearchBackend
require Logger
require Pleroma.Constants
import Ecto.Query
import Pleroma.Search.DatabaseSearch,
only: [
maybe_fetch: 3,
maybe_restrict_author: 2,
maybe_restrict_blocked: 2,
maybe_restrict_local: 2
]
alias Pleroma.Activity
alias Pleroma.Config.Getting, as: Config
alias Pleroma.Search
@max_limit 40
@default_table "pleroma_search_documents"
defp client_impl do
Config.get([Pleroma.Search.ParadeDB, :client_impl]) ||
Pleroma.Search.ParadeDB.Client.Postgres
end
defp table do
Config.get([Pleroma.Search.ParadeDB, :table], @default_table)
end
defp fuzzy_distance do
case Config.get([Pleroma.Search.ParadeDB, :fuzzy_distance], 0) do
value when is_integer(value) and value in 0..2 ->
value
value when is_binary(value) ->
case Integer.parse(value) do
{parsed, ""} when parsed in 0..2 -> parsed
_ -> 0
end
_ ->
0
end
end
@impl true
def create_index do
with {:ok, _} <- client_impl().query("CREATE EXTENSION IF NOT EXISTS pg_search", []),
{:ok, _} <- client_impl().query(create_table_sql(table()), []),
{:ok, _} <- client_impl().query(create_object_id_index_sql(table()), []),
{:ok, _} <- client_impl().query(create_bm25_index_sql(table()), []) do
:ok
else
{:error, error} ->
Logger.error("ParadeDB create_index failed: #{inspect(error)}")
{:error, error}
end
end
@impl true
def drop_index do
case client_impl().query("DROP TABLE IF EXISTS #{table()}", []) do
{:ok, _} ->
:ok
{:error, error} ->
Logger.error("ParadeDB drop_index failed: #{inspect(error)}")
{:error, error}
end
end
@impl true
def healthcheck_endpoints, do: nil
@impl true
def add_to_index(activity) do
with true <- Search.indexable?(activity),
maybe_search_data when not is_nil(maybe_search_data) <-
Search.object_to_search_data(activity.object) do
dumped_activity_id = dump_activity_id(activity.id)
actor_ap_id = activity.data["actor"]
published_at = DateTime.from_unix!(maybe_search_data.published)
if is_nil(dumped_activity_id) do
Logger.error("ParadeDB add_to_index failed: invalid activity id #{inspect(activity.id)}")
:ok
else
sql =
"""
INSERT INTO #{table()} (id, object_id, object_ap_id, actor_ap_id, content, published_at)
VALUES ($1, $2, $3, $4, $5, $6)
ON CONFLICT (id) DO UPDATE SET
object_id = EXCLUDED.object_id,
object_ap_id = EXCLUDED.object_ap_id,
actor_ap_id = EXCLUDED.actor_ap_id,
content = EXCLUDED.content,
published_at = EXCLUDED.published_at
"""
params = [
dumped_activity_id,
maybe_search_data.id,
maybe_search_data.ap,
actor_ap_id,
maybe_search_data.content,
published_at
]
case client_impl().query(sql, params) do
{:ok, _} ->
:ok
{:error, error} ->
Logger.error("ParadeDB add_to_index failed: #{inspect(error)}")
{:error, error}
end
end
else
_ -> :ok
end
end
@impl true
def remove_from_index(object) do
sql = "DELETE FROM #{table()} WHERE object_id = $1"
case client_impl().query(sql, [object.id]) do
{:ok, _} ->
:ok
{:error, error} ->
Logger.error("ParadeDB remove_from_index failed: #{inspect(error)}")
{:error, error}
end
end
@impl true
def search(user, query, options \\ []) do
limit = options |> Keyword.get(:limit, @max_limit) |> min(@max_limit)
offset = Keyword.get(options, :offset, 0)
author = Keyword.get(options, :author)
case search_ids(query, limit, offset, author) do
{:ok, []} ->
maybe_fetch([], user, query)
{:ok, ids} ->
ids
|> fetch_activities(user, query, author)
{:error, error} ->
Logger.error("ParadeDB search failed: #{inspect(error)}")
maybe_fetch([], user, query)
end
end
defp fetch_activities(ids, user, query, author) do
dumped_ids =
ids
|> Enum.map(&dump_activity_id/1)
|> Enum.reject(&is_nil/1)
from(a in Activity, where: a.id in ^dumped_ids)
|> Activity.with_preloaded_object()
|> Activity.restrict_deactivated_users()
|> maybe_restrict_public(user)
|> maybe_restrict_local(user)
|> maybe_restrict_author(author)
|> maybe_restrict_blocked(user)
|> order_by(
[a],
fragment(
"array_position(?, ?)",
^dumped_ids,
a.id
)
)
|> Pleroma.Repo.all()
|> maybe_fetch(user, query)
end
defp maybe_restrict_public(query, %Pleroma.User{}) do
intended_recipients = [
Pleroma.Constants.as_public(),
Pleroma.Web.ActivityPub.Utils.as_local_public()
]
from(a in query,
where: fragment("?->>'type' = 'Create'", a.data),
where: fragment("? && ?", ^intended_recipients, a.recipients)
)
end
defp maybe_restrict_public(query, _user) do
from(a in query,
where: fragment("?->>'type' = 'Create'", a.data),
where: ^Pleroma.Constants.as_public() in a.recipients
)
end
defp search_ids(query, limit, offset, author) do
base_sql =
"SELECT id FROM #{table()} WHERE content &&& #{search_query_param_sql(fuzzy_distance())}"
params = [query]
{base_sql, params} =
case author do
%Pleroma.User{ap_id: ap_id} when is_binary(ap_id) ->
{base_sql <> " AND actor_ap_id = $2", params ++ [ap_id]}
_ ->
{base_sql, params}
end
limit_arg = length(params) + 1
offset_arg = length(params) + 2
sql =
base_sql <>
" ORDER BY published_at DESC, id DESC LIMIT $#{limit_arg} OFFSET $#{offset_arg}"
params = params ++ [limit, offset]
with {:ok, %{rows: rows}} <- client_impl().query(sql, params) do
{:ok, Enum.map(rows, fn [id] -> id end)}
end
end
defp search_query_param_sql(0), do: "$1"
defp search_query_param_sql(distance), do: "($1::text)::pdb.fuzzy(#{distance})"
defp create_table_sql(table) do
"""
CREATE TABLE IF NOT EXISTS #{table} (
id uuid PRIMARY KEY,
object_id bigint NOT NULL,
object_ap_id text NOT NULL,
actor_ap_id text,
content text NOT NULL,
published_at timestamptz
)
"""
end
defp create_object_id_index_sql(table) do
"CREATE INDEX IF NOT EXISTS #{table}_object_id_idx ON #{table} (object_id)"
end
defp create_bm25_index_sql(table) do
"""
CREATE INDEX IF NOT EXISTS #{table}_bm25_idx ON #{table}
USING bm25 (id, content, (actor_ap_id::pdb.literal), published_at)
WITH (key_field = 'id')
"""
end
defp dump_activity_id(<<_::binary-size(16)>> = uuid), do: uuid
defp dump_activity_id(id) when is_binary(id) do
case Ecto.UUID.dump(id) do
{:ok, <<_::binary-size(16)>> = uuid} ->
uuid
:error ->
dump_flake_activity_id(id)
end
end
defp dump_activity_id(_), do: nil
defp dump_flake_activity_id(id) do
case FlakeId.Ecto.CompatType.dump(id) do
{:ok, <<_::binary-size(16)>> = uuid} -> uuid
_ -> nil
end
rescue
_ -> nil
end
end

24
lib/pleroma/search/parade_db/client.ex Normal file
View file

@ -0,0 +1,24 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Search.ParadeDB.Client do
@moduledoc false
@type sql :: String.t()
@type params :: list()
@type result :: %{optional(:rows) => list(list())}
@callback query(sql(), params()) :: {:ok, result()} | {:error, any()}
end
defmodule Pleroma.Search.ParadeDB.Client.Postgres do
@moduledoc false
@behaviour Pleroma.Search.ParadeDB.Client
@impl true
def query(sql, params \\ []) do
Ecto.Adapters.SQL.query(Pleroma.Search.ParadeDB.Repo, sql, params)
end
end

24
lib/pleroma/search/parade_db/repo.ex Normal file
View file

@ -0,0 +1,24 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Search.ParadeDB.Repo do
use Ecto.Repo,
otp_app: :pleroma,
adapter: Ecto.Adapters.Postgres,
migration_timestamps: [type: :naive_datetime_usec]
@doc """
Loads the ParadeDB database URL from `PARADEDB_DATABASE_URL`.
If the env var is not set, falls back to `config :pleroma, Pleroma.Search.ParadeDB, url: ...`.
"""
def init(_, opts) do
config = Application.get_env(:pleroma, Pleroma.Search.ParadeDB, [])
url = System.get_env("PARADEDB_DATABASE_URL") || config[:url]
url = if is_nil(url), do: nil, else: to_string(url)
{:ok, Keyword.put(opts, :url, url)}
end
end

4
lib/pleroma/search/qdrant_search.ex
View file

@ -160,7 +160,7 @@ defmodule Pleroma.Search.QdrantSearch.OpenAIClient do
end
defp client do
Tesla.client(middleware())
Tesla.client(middleware(), Pleroma.HTTP.AdapterHelper.tesla_adapter())
end
defp middleware do
@ -191,7 +191,7 @@ defmodule Pleroma.Search.QdrantSearch.QdrantClient do
end
defp client do
Tesla.client(middleware())
Tesla.client(middleware(), Pleroma.HTTP.AdapterHelper.tesla_adapter())
end
defp middleware do

2
lib/pleroma/telemetry/logger.ex
View file

@ -55,8 +55,8 @@ defmodule Pleroma.Telemetry.Logger do
def handle_event(
[:pleroma, :connection_pool, :provision_failure],
%{opts: [key | _]},
_,
%{key: key},
_
) do
Logger.debug(fn ->

60
lib/pleroma/tesla/middleware/connection_pool.ex
View file

@ -12,25 +12,38 @@ defmodule Pleroma.Tesla.Middleware.ConnectionPool do
alias Pleroma.Gun.ConnectionPool
@impl Tesla.Middleware
def call(%Tesla.Env{url: url, opts: opts} = env, next, _) do
def call(%Tesla.Env{opts: opts} = env, next, middleware_opts) do
if opts[:adapter][:body_as] == :stream do
{:error, :pooled_streaming_requires_chunks}
else
do_call(env, next, middleware_opts)
end
end
defp do_call(%Tesla.Env{url: url, opts: opts} = env, next, _) do
uri = URI.parse(url)
# Avoid leaking connections when the middleware is called twice
# with body_as: :chunks. We assume only the middleware can set
# opts[:adapter][:conn]
if opts[:adapter][:conn] do
ConnectionPool.release_conn(opts[:adapter][:conn])
end
cleanup_previous_request(env)
case ConnectionPool.get_conn(uri, opts[:adapter]) do
{:ok, conn_pid} ->
adapter_opts = Keyword.merge(opts[:adapter], conn: conn_pid, close_conn: false)
tunnel = ConnectionPool.tunnel_ref(conn_pid)
adapter_opts =
opts[:adapter]
|> Keyword.drop([:conn, :stream, :tunnel])
|> Keyword.merge(conn: conn_pid, close_conn: false, tunnel: tunnel)
opts = Keyword.put(opts, :adapter, adapter_opts)
env = %{env | opts: opts}
next = use_pool_adapter(next)
case Tesla.run(env, next) do
{:ok, env} ->
unless opts[:adapter][:body_as] == :chunks do
unless opts[:adapter][:body_as] == :chunks and chunk_client?(env.body) do
ConnectionPool.release_conn(conn_pid)
{_, res} = pop_in(env.opts[:adapter][:conn])
{:ok, res}
@ -38,8 +51,11 @@ defmodule Pleroma.Tesla.Middleware.ConnectionPool do
{:ok, env}
end
{:error, {:gun_stream_error, reason}} ->
{:error, reason}
err ->
ConnectionPool.release_conn(conn_pid)
ConnectionPool.discard_conn(conn_pid)
err
end
@ -47,4 +63,34 @@ defmodule Pleroma.Tesla.Middleware.ConnectionPool do
err
end
end
defp cleanup_previous_request(%Tesla.Env{
body: %{pid: pid, stream: stream},
opts: opts
}) do
if opts[:adapter][:conn], do: ConnectionPool.cancel_stream(pid, stream)
end
defp cleanup_previous_request(%Tesla.Env{opts: opts}) do
conn = opts[:adapter][:conn]
stream = opts[:adapter][:stream]
cond do
conn && stream -> ConnectionPool.cancel_stream(conn, stream)
conn -> ConnectionPool.release_conn(conn)
true -> :ok
end
end
defp chunk_client?(%{pid: pid, stream: stream}) when is_pid(pid) and not is_nil(stream),
do: true
defp chunk_client?(_body), do: false
defp use_pool_adapter(next) do
List.update_at(next, -1, fn
{Tesla.Adapter.Gun, function, args} -> {Pleroma.HTTP.Adapter.Gun, function, args}
adapter -> adapter
end)
end
end

72
lib/pleroma/tesla/middleware/max_body.ex Normal file
View file

@ -0,0 +1,72 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Tesla.Middleware.MaxBody do
@moduledoc """
Enforces the `:max_body` adapter option for every adapter.
`Pleroma.HTTP.Adapter.Gun` applies the limit as it accumulates a `:plain`
body, so an oversized response is abandoned mid-transfer. Hackney cannot:
since 4.0 it ignores `with_body` and always hands back a fully read binary
(`hackney:request/5`), so `Tesla.Adapter.Hackney` never takes the branch that
would honour `:max_body`. Without this middleware the option is silently
unenforced there.
The check therefore bounds what callers are handed, and refuses up front when
`content-length` already announces too much. It cannot bound peak memory on
the hackney path, because the adapter has read the body before any middleware
runs.
Streaming bodies are left alone: neither adapter limits those, and the caller
that owns the stream is the one that has to drain and release it.
"""
@behaviour Tesla.Middleware
@impl Tesla.Middleware
def call(%Tesla.Env{opts: opts} = env, next, _opts) do
case opts[:adapter][:max_body] do
limit when is_integer(limit) and limit >= 0 -> run(env, next, limit)
_ -> Tesla.run(env, next)
end
end
defp run(env, next, limit) do
with {:ok, env} <- Tesla.run(env, next),
:ok <- check(env, limit) do
{:ok, env}
end
end
defp check(%Tesla.Env{body: body} = env, limit) when is_binary(body) do
with :ok <- check_content_length(env, limit) do
check_body(body, limit)
end
end
# body_as: :chunks and response: :stream hand back a client the caller drains
# itself, applying its own limit (Pleroma.Web.RichMedia.Helpers). There is
# nothing to measure, and failing on the header alone would strand the pool
# lease that only the caller holds a handle to.
defp check(_env, _limit), do: :ok
# A bodyless response may still announce the entity's size.
defp check_content_length(%Tesla.Env{method: :head}, _limit), do: :ok
defp check_content_length(%Tesla.Env{status: status}, _limit) when status in [204, 304],
do: :ok
defp check_content_length(env, limit) do
with value when is_binary(value) <- Tesla.get_header(env, "content-length"),
{length, ""} <- Integer.parse(value) do
if length > limit, do: {:error, :body_too_large}, else: :ok
else
_ -> :ok
end
end
defp check_body(body, limit) do
if byte_size(body) > limit, do: {:error, :body_too_large}, else: :ok
end
end

54
lib/pleroma/user.ex
View file

@ -39,6 +39,7 @@ defmodule Pleroma.User do
alias Pleroma.Web.Endpoint
alias Pleroma.Web.OAuth
alias Pleroma.Web.RelMe
alias Pleroma.Webhook.Notify
alias Pleroma.Workers.BackgroundWorker
alias Pleroma.Workers.DeleteWorker
alias Pleroma.Workers.UserRefreshWorker
@ -272,13 +273,13 @@ defmodule Pleroma.User do
end
def cached_blocked_users_ap_ids(user) do
@cachex.fetch!(:user_cache, "blocked_users_ap_ids:#{user.ap_id}", fn _ ->
@cachex.fetch!(:user_cache, cache_key(:blocked_users_ap_ids, user), fn _ ->
blocked_users_ap_ids(user)
end)
end
def cached_muted_users_ap_ids(user) do
@cachex.fetch!(:user_cache, "muted_users_ap_ids:#{user.ap_id}", fn _ ->
@cachex.fetch!(:user_cache, cache_key(:muted_users_ap_ids, user), fn _ ->
muted_users_ap_ids(user)
end)
end
@ -326,7 +327,7 @@ defmodule Pleroma.User do
@spec visible_for(User.t(), User.t() | nil) ::
:visible
| :invisible
| :restricted_unauthenticated
| :restrict_unauthenticated
| :deactivated
| :confirmation_pending
def visible_for(user, for_user \\ nil)
@ -987,6 +988,7 @@ defmodule Pleroma.User do
@doc "Inserts provided changeset, performs post-registration actions (confirmation email sending etc.)"
def register(%Ecto.Changeset{} = changeset) do
with {:ok, user} <- Repo.insert(changeset) do
Notify.trigger_webhooks(user, :"account.created")
post_register_action(user)
end
end
@ -1253,13 +1255,18 @@ defmodule Pleroma.User do
get_cached_by_nickname(nickname)
end
defp cache_key(:id, %User{id: id}), do: cache_key(:id, id)
defp cache_key(:nickname, %User{nickname: nickname}), do: cache_key(:nickname, nickname)
defp cache_key(namespace, %User{ap_id: ap_id}), do: cache_key(namespace, ap_id)
defp cache_key(namespace, value), do: "#{namespace}:#{value}"
def set_cache({:ok, user}), do: set_cache(user)
def set_cache({:error, err}), do: {:error, err}
def set_cache(%User{} = user) do
@cachex.put(:user_cache, "ap_id:#{user.ap_id}", user)
@cachex.put(:user_cache, "nickname:#{user.nickname}", user)
@cachex.put(:user_cache, "friends_ap_ids:#{user.nickname}", get_user_friends_ap_ids(user))
@cachex.put(:user_cache, cache_key(:ap_id, user), user)
@cachex.put(:user_cache, cache_key(:nickname, user), user)
@cachex.put(:user_cache, cache_key(:friends_ap_ids, user), get_user_friends_ap_ids(user))
{:ok, user}
end
@ -1287,22 +1294,23 @@ defmodule Pleroma.User do
@spec get_cached_user_friends_ap_ids(User.t()) :: [String.t()]
def get_cached_user_friends_ap_ids(user) do
@cachex.fetch!(:user_cache, "friends_ap_ids:#{user.ap_id}", fn _ ->
@cachex.fetch!(:user_cache, cache_key(:friends_ap_ids, user), fn _ ->
get_user_friends_ap_ids(user)
end)
end
def invalidate_cache(user) do
@cachex.del(:user_cache, "ap_id:#{user.ap_id}")
@cachex.del(:user_cache, "nickname:#{user.nickname}")
@cachex.del(:user_cache, "friends_ap_ids:#{user.ap_id}")
@cachex.del(:user_cache, "blocked_users_ap_ids:#{user.ap_id}")
@cachex.del(:user_cache, "muted_users_ap_ids:#{user.ap_id}")
@cachex.del(:user_cache, cache_key(:ap_id, user))
@cachex.del(:user_cache, cache_key(:nickname, user))
@cachex.del(:user_cache, cache_key(:id, user))
@cachex.del(:user_cache, cache_key(:friends_ap_ids, user))
@cachex.del(:user_cache, cache_key(:blocked_users_ap_ids, user))
@cachex.del(:user_cache, cache_key(:muted_users_ap_ids, user))
end
@spec get_cached_by_ap_id(String.t()) :: User.t() | nil
def get_cached_by_ap_id(ap_id) do
key = "ap_id:#{ap_id}"
key = cache_key(:ap_id, ap_id)
with {:ok, nil} <- @cachex.get(:user_cache, key),
user when not is_nil(user) <- get_by_ap_id(ap_id),
@ -1315,14 +1323,14 @@ defmodule Pleroma.User do
end
def get_cached_by_id(id) do
key = "id:#{id}"
key = cache_key(:id, id)
ap_id =
@cachex.fetch!(:user_cache, key, fn _ ->
user = get_by_id(id)
if user do
@cachex.put(:user_cache, "ap_id:#{user.ap_id}", user)
@cachex.put(:user_cache, cache_key(:ap_id, user), user)
{:commit, user.ap_id}
else
{:ignore, ""}
@ -1333,7 +1341,7 @@ defmodule Pleroma.User do
end
def get_cached_by_nickname(nickname) do
key = "nickname:#{nickname}"
key = cache_key(:nickname, nickname)
@cachex.fetch!(:user_cache, key, fn _ ->
case get_or_fetch_by_nickname(nickname) do
@ -1651,7 +1659,7 @@ defmodule Pleroma.User do
|> Oban.insert()
end
@cachex.del(:user_cache, "muted_users_ap_ids:#{muter.ap_id}")
@cachex.del(:user_cache, cache_key(:muted_users_ap_ids, muter))
{:ok, Enum.filter([user_mute, user_notification_mute], & &1)}
end
@ -1661,7 +1669,7 @@ defmodule Pleroma.User do
with {:ok, user_mute} <- UserRelationship.delete_mute(muter, mutee),
{:ok, user_notification_mute} <-
UserRelationship.delete_notification_mute(muter, mutee) do
@cachex.del(:user_cache, "muted_users_ap_ids:#{muter.ap_id}")
@cachex.del(:user_cache, cache_key(:muted_users_ap_ids, muter))
{:ok, [user_mute, user_notification_mute]}
end
end
@ -1964,6 +1972,12 @@ defmodule Pleroma.User do
def approve(%User{} = user), do: {:ok, user}
def reject(%User{is_approved: false} = user) do
delete(user)
end
def reject(%User{} = _user), do: {:error, "User is approved"}
def confirm(users) when is_list(users) do
Repo.transaction(fn ->
Enum.map(users, fn user ->
@ -2815,7 +2829,7 @@ defmodule Pleroma.User do
{:ok, UserRelationship.t()} | {:error, Ecto.Changeset.t()}
defp add_to_block(%User{} = user, %User{} = blocked, expires_at) do
with {:ok, relationship} <- UserRelationship.create_block(user, blocked, expires_at) do
@cachex.del(:user_cache, "blocked_users_ap_ids:#{user.ap_id}")
@cachex.del(:user_cache, cache_key(:blocked_users_ap_ids, user))
{:ok, relationship}
end
end
@ -2824,7 +2838,7 @@ defmodule Pleroma.User do
{:ok, UserRelationship.t()} | {:ok, nil} | {:error, Ecto.Changeset.t()}
defp remove_from_block(%User{} = user, %User{} = blocked) do
with {:ok, relationship} <- UserRelationship.delete_block(user, blocked) do
@cachex.del(:user_cache, "blocked_users_ap_ids:#{user.ap_id}")
@cachex.del(:user_cache, cache_key(:blocked_users_ap_ids, user))
{:ok, relationship}
end
end

24
lib/pleroma/user/query.ex
View file

@ -63,7 +63,8 @@ defmodule Pleroma.User.Query do
limit: pos_integer(),
actor_types: [String.t()],
birthday_day: pos_integer(),
birthday_month: pos_integer()
birthday_month: pos_integer(),
staff: boolean()
}
| map()
@ -179,6 +180,10 @@ defmodule Pleroma.User.Query do
defp compose_query({:external, _}, query), do: location_query(query, false)
defp compose_query({:active, false}, query) do
where(query, [u], u.is_active == false or u.is_approved == false or u.is_confirmed == false)
end
defp compose_query({:active, _}, query) do
where(query, [u], u.is_active == true)
|> where([u], u.is_approved == true)
@ -202,6 +207,10 @@ defmodule Pleroma.User.Query do
where(query, [u], u.is_confirmed != ^bool)
end
defp compose_query({:need_approval, false}, query) do
where(query, [u], u.is_approved == true)
end
defp compose_query({:need_approval, _}, query) do
where(query, [u], u.is_approved == false)
end
@ -285,6 +294,19 @@ defmodule Pleroma.User.Query do
|> where([u], fragment("date_part('month', ?)", u.birthday) == ^month)
end
defp compose_query({:staff, false}, query) do
where(query, [u], u.is_admin == false and u.is_moderator == false)
end
defp compose_query({:staff, _}, query) do
where(query, [u], u.is_admin == true or u.is_moderator == true)
end
defp compose_query({:domain, domain}, query) do
query
|> where([u], like(u.nickname, ^"%@#{domain}"))
end
defp compose_query(_unsupported_param, query), do: query
defp location_query(query, local) do

Some files were not shown because too many files have changed in this diff Show more