diff --git a/.woodpecker/docker-armv7.yaml b/.woodpecker/docker-armv7.yaml deleted file mode 100644 index c921a56f9..000000000 --- a/.woodpecker/docker-armv7.yaml +++ /dev/null @@ -1,85 +0,0 @@ -when: - # Temporary PR validation while arm/v7 Docker builds are being brought up. - - event: pull_request - path: [ "**/*.ex", "**/*.eex", "**/*.exs", "mix.lock", ".woodpecker/**", "Dockerfile" ] - - event: push - branch: ${CI_REPO_DEFAULT_BRANCH} - path: [ "**/*.ex", "**/*.eex", "**/*.exs", "mix.lock", ".woodpecker/**", "Dockerfile" ] - - event: tag - - event: manual - branch: ${CI_REPO_DEFAULT_BRANCH} - - event: manual - branch: stable - -# Target the CAX21 arm64 host with native arm32 userspace support, not generic arm64 runners. -labels: - platform: linux/arm - armv7: native - -variables: - docker_variables: &docker_variables - repo: pleroma/pleroma - registry: git.pleroma.social - armv7_build_settings: &armv7_build_settings - <<: *docker_variables - build_args: - - ELIXIR_IMG=arm32v7/elixir - - ELIXIR_TAG=1.17.3-otp-26-alpine - - ALPINE_IMG=arm32v7/alpine - - ALPINE_VER=3.20 - extra_opts: --custom-platform=linux/arm/v7 - armv7_push_settings: &armv7_push_settings - <<: *armv7_build_settings - username: - from_secret: pleroma-ci-user - password: - from_secret: pleroma-ci-password - -steps: - docker-armv7-pr: - image: woodpeckerci/plugin-kaniko:2.3.1 - when: - - event: pull_request - settings: - <<: *armv7_build_settings - dry_run: true - tags: - - armv7-pr-check - - docker-develop-armv7: - image: woodpeckerci/plugin-kaniko:2.3.1 - when: - - event: push - branch: ${CI_REPO_DEFAULT_BRANCH} - - event: manual - branch: ${CI_REPO_DEFAULT_BRANCH} - settings: - <<: *armv7_push_settings - tags: - - latest-armv7 - - develop-armv7 - - ${CI_COMMIT_SHA:0:8}-armv7 - - docker-stable-armv7: - image: woodpeckerci/plugin-kaniko:2.3.1 - when: - - evaluate: 'CI_PIPELINE_EVENT == "manual" && CI_COMMIT_BRANCH == "stable" && CI_COMMIT_TAG == ""' - settings: - <<: *armv7_push_settings - tags: - - latest-armv7 - - stable-armv7 - - ${CI_COMMIT_SHA:0:8}-armv7 - - docker-stable-tag-armv7: - image: woodpeckerci/plugin-kaniko:2.3.1 - when: - - event: tag - - evaluate: 'CI_PIPELINE_EVENT == "manual" && CI_COMMIT_BRANCH == "stable" && CI_COMMIT_TAG != ""' - settings: - <<: *armv7_push_settings - tags: - - latest-armv7 - - stable-armv7 - - ${CI_COMMIT_SHA:0:8}-armv7 - - ${CI_COMMIT_TAG}-armv7 diff --git a/.woodpecker/docker-combine.yaml b/.woodpecker/docker-combine.yaml index 6c91d26e9..fc69430fe 100644 --- a/.woodpecker/docker-combine.yaml +++ b/.woodpecker/docker-combine.yaml @@ -18,7 +18,7 @@ labels: steps: docker-develop-combine: - image: git.fluffytail.org/phnt/wpc-docker-tagger:latest + image: &docker_image git.pleroma.social/phnt/wpc-docker-tagger:0.9.1 when: - event: push branch: ${CI_REPO_DEFAULT_BRANCH} @@ -27,18 +27,18 @@ steps: settings: &docker_settings registry: "git.pleroma.social" image: "pleroma/pleroma" - architectures: [amd64, arm64] + architectures: [amd64, arm64, armv7] tags: - latest - develop - - ${CI_COMMIT_SHA:0:8} + - "${CI_COMMIT_SHA:0:8}" username: from_secret: pleroma-ci-user password: from_secret: pleroma-ci-password docker-stable-combine: - image: git.fluffytail.org/phnt/wpc-docker-tagger:latest + image: *docker_image when: - evaluate: 'CI_PIPELINE_EVENT == "manual" && CI_COMMIT_BRANCH == "stable" && CI_COMMIT_TAG == ""' settings: @@ -46,10 +46,10 @@ steps: tags: &stable_docker_tags - latest - stable - - ${CI_COMMIT_SHA:0:8} + - "${CI_COMMIT_SHA:0:8}" docker-stable-tag-combine: - image: git.fluffytail.org/phnt/wpc-docker-tagger:latest + image: *docker_image when: - event: tag - evaluate: 'CI_PIPELINE_EVENT == "manual" && CI_COMMIT_BRANCH == "stable" && CI_COMMIT_TAG != ""' @@ -57,4 +57,4 @@ steps: <<: *docker_settings tags: - <<: *stable_docker_tags - - ${CI_COMMIT_TAG} + - "${CI_COMMIT_TAG}" diff --git a/.woodpecker/docker.yaml b/.woodpecker/docker.yaml index 9a5be78e2..bdfec6a61 100644 --- a/.woodpecker/docker.yaml +++ b/.woodpecker/docker.yaml @@ -12,6 +12,7 @@ matrix: platform: - linux/amd64 - linux/arm64 + - linux/arm # This is needed for the when clauses below. labels: @@ -26,11 +27,19 @@ variables: from_secret: pleroma-ci-user password: from_secret: pleroma-ci-password - kaniko_image: &kaniko_image woodpeckerci/plugin-kaniko:2.3.1 + armv7_docker_variables: &armv7_docker_variables + <<: *docker_variables + build_args: + - ELIXIR_IMG=docker.io/arm32v7/elixir + - ELIXIR_TAG=1.17.3-otp-27-alpine + - ALPINE_IMG=docker.io/arm32v7/alpine + - ALPINE_VER=3.20 + extra_opts: --custom-platform=linux/arm/v7 + kaniko_image: &kaniko_image docker.io/woodpeckerci/plugin-kaniko:2.3.2 steps: docker-develop-amd64: - image: woodpeckerci/plugin-kaniko:2.3.1 + image: *kaniko_image when: - evaluate: 'platform == "linux/amd64" && CI_COMMIT_BRANCH == "${CI_REPO_DEFAULT_BRANCH}"' settings: @@ -41,7 +50,7 @@ steps: - ${CI_COMMIT_SHA:0:8}-amd64 docker-develop-arm64: - image: woodpeckerci/plugin-kaniko:2.3.1 + image: *kaniko_image when: - evaluate: 'platform == "linux/arm64" && CI_COMMIT_BRANCH == "${CI_REPO_DEFAULT_BRANCH}"' settings: @@ -51,6 +60,17 @@ steps: - develop-arm64 - ${CI_COMMIT_SHA:0:8}-arm64 + docker-develop-armv7: + image: *kaniko_image + when: + - evaluate: 'platform == "linux/arm" && CI_COMMIT_BRANCH == "${CI_REPO_DEFAULT_BRANCH}"' + settings: + <<: *armv7_docker_variables + tags: + - latest-armv7 + - develop-armv7 + - ${CI_COMMIT_SHA:0:8}-armv7 + docker-stable-amd64: image: *kaniko_image when: @@ -106,3 +126,25 @@ steps: tags: - <<: *arm64_tags - ${CI_COMMIT_TAG}-arm64 + + docker-stable-armv7: + image: *kaniko_image + when: + - evaluate: 'platform == "linux/arm" && CI_PIPELINE_EVENT == "manual" && CI_COMMIT_BRANCH == "stable" && CI_COMMIT_TAG == ""' + settings: + <<: *armv7_docker_variables + tags: &armv7_tags + - latest-armv7 + - stable-armv7 + - ${CI_COMMIT_SHA:0:8}-armv7 + + docker-stable-tag-armv7: + image: *kaniko_image + when: + - evaluate: 'platform == "linux/arm" && CI_PIPELINE_EVENT == "tag"' + - evaluate: 'platform == "linux/arm" && CI_PIPELINE_EVENT == "manual" && CI_COMMIT_BRANCH == "stable" && CI_COMMIT_TAG != ""' + settings: + <<: *armv7_docker_variables + tags: + - <<: *armv7_tags + - ${CI_COMMIT_TAG}-armv7 diff --git a/.woodpecker/docs.yaml b/.woodpecker/docs.yaml new file mode 100644 index 000000000..2a6023abd --- /dev/null +++ b/.woodpecker/docs.yaml @@ -0,0 +1,45 @@ +when: + - event: push + branch: ${CI_REPO_DEFAULT_BRANCH} + path: [ "docs/**" ] + - event: tag + - event: manual + evaluate: 'CI_COMMIT_BRANCH == "stable" || CI_COMMIT_BRANCH == CI_REPO_DEFAULT_BRANCH' + +labels: + platform: linux/amd64 + +skip_clone: true + +variables: + trigger_settings: &trigger_settings + deploy: 'docs' + last-successful: true + token: + from_secret: pleroma-ci-woodpecker-token + repositories: + - pleroma/docs@master + docker_image: &docker_image docker.io/woodpeckerci/plugin-trigger:1.1.1 + +steps: + trigger-deploy-stable: + when: + - event: tag + - event: manual + branch: stable + image: *docker_image + settings: + <<: *trigger_settings + params: + - BRANCH=stable + + trigger-deploy-develop: + when: + - event: push + - event: manual + branch: ${CI_REPO_DEFAULT_BRANCH} + image: *docker_image + settings: + <<: *trigger_settings + params: + - BRANCH=develop diff --git a/.woodpecker/lint.yaml b/.woodpecker/lint.yaml index 8a5718c5d..1ed040440 100644 --- a/.woodpecker/lint.yaml +++ b/.woodpecker/lint.yaml @@ -18,7 +18,7 @@ variables: steps: mix-format: image: &elixir-image - docker.io/elixir:1.15-alpine + docker.io/elixir:1.17-otp-27-alpine entrypoint: *script_file_entrypoint failure: ignore commands: diff --git a/.woodpecker/otp-musl.yaml b/.woodpecker/otp-musl.yaml index 60558b893..4443d992a 100644 --- a/.woodpecker/otp-musl.yaml +++ b/.woodpecker/otp-musl.yaml @@ -33,10 +33,10 @@ variables: - mkdir release - export PLEROMA_BUILD_BRANCH=${CI_COMMIT_BRANCH} - mix release --path release - build_image_amd64: &build_image_amd64 docker.io/hexpm/elixir-amd64:1.17.3-erlang-27.3.4.2-alpine-3.22.1 + build_image_amd64: &build_image_amd64 docker.io/hexpm/elixir:1.17.3-erlang-27.3.4.17-alpine-3.22.5 build_image_arm: &build_image_arm docker.io/arm32v7/elixir:1.17.3-alpine - build_image_arm64: &build_image_arm64 docker.io/hexpm/elixir-arm64:1.17.3-erlang-27.3.4.2-alpine-3.22.1 - artifacts_uploader_image: &artifacts_uploader_image docker.io/woodpeckercommunity/plugin-gitea-package:0.5.0 + build_image_arm64: &build_image_arm64 docker.io/hexpm/elixir:1.17.3-erlang-27.3.4.17-alpine-3.22.5 + artifacts_uploader_image: &artifacts_uploader_image docker.io/woodpeckercommunity/plugin-gitea-package:0.5.1 artifacts_uploader_settings: &artifacts_uploader_settings user: from_secret: pleroma-ci-user diff --git a/.woodpecker/otp.yaml b/.woodpecker/otp.yaml index 9a33c228e..6b7a8124e 100644 --- a/.woodpecker/otp.yaml +++ b/.woodpecker/otp.yaml @@ -33,10 +33,10 @@ variables: - mkdir release - export PLEROMA_BUILD_BRANCH=${CI_COMMIT_BRANCH} - mix release --path release - build_image_amd64: &build_image_amd64 docker.io/hexpm/elixir-amd64:1.17.3-erlang-27.3.4.2-ubuntu-noble-20250716 + build_image_amd64: &build_image_amd64 docker.io/hexpm/elixir:1.17.3-erlang-27.3.4.17-ubuntu-noble-20260810 build_image_arm: &build_image_arm docker.io/arm32v7/elixir:1.17.3 - build_image_arm64: &build_image_arm64 docker.io/hexpm/elixir-arm64:1.17.3-erlang-27.3.4.2-ubuntu-noble-20250716 - artifacts_uploader_image: &artifacts_uploader_image docker.io/woodpeckercommunity/plugin-gitea-package:0.5.0 + build_image_arm64: &build_image_arm64 docker.io/hexpm/elixir:1.17.3-erlang-27.3.4.17-ubuntu-noble-20260810 + artifacts_uploader_image: &artifacts_uploader_image docker.io/woodpeckercommunity/plugin-gitea-package:0.5.1 artifacts_uploader_settings: &artifacts_uploader_settings user: from_secret: pleroma-ci-user diff --git a/.woodpecker/paradedb-integration.yaml b/.woodpecker/paradedb-integration.yaml new file mode 100644 index 000000000..40994662f --- /dev/null +++ b/.woodpecker/paradedb-integration.yaml @@ -0,0 +1,66 @@ +when: + - event: pull_request + path: + - ".woodpecker/**" + - "lib/pleroma/search/**" + - "test/pleroma/search/**" + - "mix.exs" + - "mix.lock" + evaluate: 'CI_COMMIT_SOURCE_BRANCH != "weblate"' + - event: push + branch: ${CI_REPO_DEFAULT_BRANCH} + path: + - ".woodpecker/**" + - "lib/pleroma/search/**" + - "test/pleroma/search/**" + - "mix.exs" + - "mix.lock" + +depends_on: + - lint + +labels: + platform: linux/amd64 + +variables: + script_file_entrypoint: &script_file_entrypoint + - /bin/sh + - -c + - 'printf "%s" "$CI_SCRIPT" | base64 -d > /tmp/ci-script.sh && /bin/sh -xe /tmp/ci-script.sh' + +steps: + paradedb-integration: + image: docker.io/elixir:1.19-otp-28-alpine + entrypoint: *script_file_entrypoint + environment: + MIX_ENV: test + VIX_COMPILATION_MODE: PLATFORM_PROVIDED_LIBVIPS + DB_HOST: postgres + DB_PORT: 5432 + PARADEDB_TEST_URL: postgres://postgres:postgres@paradedb:5432/paradedb + commands: + - apk add --no-cache build-base cmake exiftool ffmpeg vips-dev file-dev git openssl postgresql-client + - mix local.hex --force + - mix local.rebar --force + - mix deps.get + - timeout 60 sh -c 'until pg_isready -h postgres -U postgres -d pleroma_test; do sleep 1; done' || { echo "PostgreSQL did not become ready" >&2; exit 1; } + - timeout 60 sh -c 'until pg_isready -h paradedb -U postgres -d paradedb; do sleep 1; done' || { echo "ParadeDB did not become ready" >&2; exit 1; } + - mix ecto.create + - mix ecto.migrate + - mix test --include integration test/pleroma/search/parade_db_integration_test.exs + +services: + postgres: + image: docker.io/postgres:13-alpine + failure: ignore + environment: + POSTGRES_DB: pleroma_test + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + paradedb: + image: docker.io/paradedb/paradedb:0.24.3-pg15@sha256:997d62f6c2605becb7c45878af3f8c8c698f355006b675b3ce212d4eb2d14f2e + failure: ignore + environment: + POSTGRES_DB: paradedb + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres diff --git a/.woodpecker/spec-build.yaml b/.woodpecker/spec-build.yaml new file mode 100644 index 000000000..0024a62f1 --- /dev/null +++ b/.woodpecker/spec-build.yaml @@ -0,0 +1,103 @@ +when: + - event: push + branch: ${CI_REPO_DEFAULT_BRANCH} + path: + - ".woodpecker/spec-build.yaml" + - "mix.exs" + - "mix.lock" + - "lib/pleroma/web.ex" + - "lib/pleroma/web/**/*.ex" + - event: tag + - event: manual + evaluate: 'CI_COMMIT_BRANCH == "stable" || CI_COMMIT_BRANCH == CI_REPO_DEFAULT_BRANCH' + +labels: + platform: linux/amd64 + +concurrency: 1 + +variables: + artifacts_uploader_image: &artifacts_uploader_image docker.io/woodpeckercommunity/plugin-gitea-package:0.5.1 + artifacts_uploader_settings: &artifacts_uploader_settings + user: + from_secret: pleroma-ci-user + password: + from_secret: pleroma-ci-password + owner: 'pleroma' + package_name: api-docs-spec + file_source: ./spec.json + file_name: spec.json + update: 'true' + trigger_image: &trigger_image docker.io/woodpeckerci/plugin-trigger:1.1.1 + trigger_settings: &trigger_settings + deploy: 'api-docs' + last-successful: true + token: + from_secret: pleroma-ci-woodpecker-token + repositories: + - pleroma/api-docs@master + +steps: + build-spec: + image: docker.io/hexpm/elixir:1.17.3-erlang-27.3.4.17-alpine-3.22.5 + environment: + MIX_ENV: prod + VIX_COMPILATION_MODE: PLATFORM_PROVIDED_LIBVIPS + commands: + - apk update --no-cache + - apk add git build-base cmake file-dev openssl vips-dev zip + - mix local.hex --force + - mix local.rebar --force + - mix deps.get --only prod + - mix pleroma.openapi_spec spec.json + - | + printf 'BRANCH=stable\nSPEC_VERSION=%s\n' "$${CI_COMMIT_SHA}" > trigger-deploy-stable.params + printf 'BRANCH=develop\nSPEC_VERSION=%s\n' "$${CI_COMMIT_SHA}" > trigger-deploy-develop.params + + upload-spec: + image: *artifacts_uploader_image + settings: + <<: *artifacts_uploader_settings + package_version: ${CI_COMMIT_SHA} + + upload-spec-stable: + when: + - event: tag + - event: manual + branch: stable + image: *artifacts_uploader_image + settings: + <<: *artifacts_uploader_settings + package_version: stable + + upload-spec-develop: + when: + - event: push + - event: manual + branch: ${CI_REPO_DEFAULT_BRANCH} + image: *artifacts_uploader_image + settings: + <<: *artifacts_uploader_settings + package_version: develop + + trigger-deploy-stable: + when: + - event: tag + - event: manual + branch: stable + image: *trigger_image + settings: + <<: *trigger_settings + params: + - trigger-deploy-stable.params + + trigger-deploy-develop: + when: + - event: push + - event: manual + branch: ${CI_REPO_DEFAULT_BRANCH} + image: *trigger_image + settings: + <<: *trigger_settings + params: + - trigger-deploy-develop.params diff --git a/.woodpecker/unit-testing-elixir-1.15.yaml b/.woodpecker/unit-testing-elixir-1.17.yaml similarity index 84% rename from .woodpecker/unit-testing-elixir-1.15.yaml rename to .woodpecker/unit-testing-elixir-1.17.yaml index a5bcc0ef3..020bacb8c 100644 --- a/.woodpecker/unit-testing-elixir-1.15.yaml +++ b/.woodpecker/unit-testing-elixir-1.17.yaml @@ -19,15 +19,16 @@ variables: - 'printf "%s" "$CI_SCRIPT" | base64 -d > /tmp/ci-script.sh && /bin/sh -xe /tmp/ci-script.sh' steps: - unit-testing-elixir-1.15: - image: elixir:1.15-alpine + unit-testing-elixir-1.17: + image: elixir:1.17-otp-27-alpine entrypoint: *script_file_entrypoint environment: MIX_ENV: test + VIX_COMPILATION_MODE: PLATFORM_PROVIDED_LIBVIPS DB_HOST: postgres DB_PORT: 5432 commands: - - apk add --no-cache build-base cmake exiftool ffmpeg file-dev git openssl + - apk add --no-cache build-base cmake exiftool ffmpeg vips-dev file-dev git openssl - adduser -D -h /home/testuser testuser - mkdir -p /home/testuser/.mix /home/testuser/.hex - chown -R testuser:testuser . /home/testuser @@ -39,6 +40,7 @@ steps: services: postgres: image: postgres:13-alpine + failure: ignore environment: POSTGRES_DB: pleroma_test POSTGRES_USER: postgres diff --git a/.woodpecker/unit-testing-elixir-1.18.yaml b/.woodpecker/unit-testing-elixir-1.19.yaml similarity index 84% rename from .woodpecker/unit-testing-elixir-1.18.yaml rename to .woodpecker/unit-testing-elixir-1.19.yaml index 6e576fad9..156fc350b 100644 --- a/.woodpecker/unit-testing-elixir-1.18.yaml +++ b/.woodpecker/unit-testing-elixir-1.19.yaml @@ -19,15 +19,16 @@ variables: - 'printf "%s" "$CI_SCRIPT" | base64 -d > /tmp/ci-script.sh && /bin/sh -xe /tmp/ci-script.sh' steps: - unit-testing-elixir-1.18: - image: elixir:1.18-otp-27-alpine + unit-testing-elixir-1.19: + image: elixir:1.19-otp-28-alpine entrypoint: *script_file_entrypoint environment: MIX_ENV: test + VIX_COMPILATION_MODE: PLATFORM_PROVIDED_LIBVIPS DB_HOST: postgres DB_PORT: 5432 commands: - - apk add --no-cache build-base cmake exiftool ffmpeg file-dev git openssl + - apk add --no-cache build-base cmake exiftool ffmpeg vips-dev file-dev git openssl - adduser -D -h /home/testuser testuser - mkdir -p /home/testuser/.mix /home/testuser/.hex - chown -R testuser:testuser . /home/testuser @@ -39,6 +40,7 @@ steps: services: postgres: image: postgres:13-alpine + failure: ignore environment: POSTGRES_DB: pleroma_test POSTGRES_USER: postgres diff --git a/CHANGELOG.md b/CHANGELOG.md index f03be3f29..2f799a0ed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,115 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). +## 2.11.0 + +### Security + +- Update Bandit to address HTTP request handling vulnerabilities. +- Update bandit to version 1.12.4 to fix a DoS vulnerability +- Update bandit to version 1.12.5 to fix an HTTP/2 DoS vulnerability +- Update bandit to version 1.13.1 resolving a stuck process and unvalidated HTTP/1 headers vulnerabilities +- Updated CAStore to version 1.0.22 with updated Mozilla certificate bundle after eMudhra Intermediate CA Compromise +- Update certifi to version 2.18.0, which removes the distrusted emSign roots +- Update plug_crypto to version 2.2.1, so a malformed shout socket token no longer writes the secret key base to the error log +- Update Cowboy to address multipart header parsing denial-of-service vulnerabilities. +- Update cowboy to version 2.19.0 which fixes a memory exhaustion DoS +- Update Cowlib to address HTTP parsing resource exhaustion and injection vulnerabilities. +- Update cowlib to version 2.20.0 which fixes HTTP splitting and header smuggling/injection vulnerabilities +- Update gun to version 2.7.0 and cowlib to version 2.21.0 resolving a header smuggling vulnerability +- Update Decimal and compatible dependencies to address a rounding vulnerability. +- Update Decimal to version 3.2.0 resolving an unbounded allocation vulnerability +- Replace retired Earmark rendering with the maintained EarmarkParser and an attribute-escaping renderer. +- Update gun to version 2.6.0 which fixes an HTTP request/response smuggling vulnerability +- Update Hackney and its HTTPoison integration to address SSRF, timeout, and injection vulnerabilities. +- Hackney, Finch, Mua and the Swoosh HTTP mail adapters now trust only the CAStore certificate bundle, which rejects the compromised eMudhra intermediate CAs +- Ensure Host header is present and matches instance URI +- Update mint to version 1.10.0 which fixes multiple DoS vulnerabilities +- Update mint to version 1.11.0 resolving multiple vulnerabilities +- Update Phoenix LiveView to version 1.2.9 to fix an open redirect vulnerability +- Update Phoenix to version 1.8.11 resolving a process-exhaustion DoS vulnerability +- Update Phoenix to address long-poll NDJSON request denial-of-service vulnerabilities. +- Update Plug to address multipart header parsing denial-of-service vulnerabilities. +- Update Plug.Cowboy to address HTTP/2 scheme atom exhaustion denial-of-service vulnerabilities. +- Update Postgrex to address notification channel-name SQL injection vulnerabilities. +- Update postgrex to version 0.22.4 to fix an SQL injection vulnerability +- Remove unused QUIC lockfile entry with vulnerable TLS verification behavior. +- Update swoosh to version 1.28.0 which fixes a url injection vulnerability for the Microsoft Graph adapter +- Scrubbers: remove links and images whose scheme is hidden with tabs, newlines or leading control characters (for example javascript:), which browsers still run as javascript:. Custom scrubbers should use Pleroma.HTML.URIAttributes.allow_tag_with_uri_attributes instead of the FastSanitize macro + +### Changed + +- Update bundled Pleroma-FE to 2.11.4 +- Allow disabling Chats feature +- Pleroma now requires Elixir 1.17 or newer and Erlang/OTP 27 or newer +- Update Debian and Ubuntu installation requirements and OTP dependencies +- Update Jason to support Decimal 3 dependency resolution. +- Changed how Logger is configured due to changes in Elixir and added deprecation warnings for the old configuration format +- Update majic to 1.2.0. +- Avoid a duplicate user query during OAuth authentication. +- Use the Hex package for oban_plugins_lazarus. +- Docs OpenBSD: Increased recommended `datasize` from 1536M to 4096M to avoid Pleroma crashes +- OpenRC: Automatically run migrations before starting pleroma +- Optimize orphaned OAuth apps cleanup by deleting records in batches instead of one by one. +- Switch patched Phoenix 1.7.14 back to upstream with Phoenix 1.8.0+ +- Add optional migrations for rollback from Rebased +- Use upstream Hex releases for the `remote_ip` dependency and expose client IP ranges for remote IP resolution. +- Improve user search / autocompletion ordering. +- Gun is now the default HTTP client instead of Hackney + +### Added + +- Allow creating accounts without a password in the AdminAPI +- Implemented ConfigDB configuration blacklist and accompanying Mix task +- Accept more real-world ActivityPub object shapes collected by fedidev.fun +- Add Mastodon-compatible grouped notifications API endpoints under `/api/v2/notifications` for newly-created notifications +- Docs: Added documentation for installing newer version of Elixir/Erlang than what distro ships +- Allow attaching emojis to user lists +- Implement basics of Mastodon admin API +- Add backend support for Misskey Markdown (MFM) posts +- Add mix_audit for dependency vulnerability checks. +- Oban: Enabled the Reindexer plugin to reduce oban_jobs index DB bloat +- Add ParadeDB (pg_search) as an optional external search backend, indexing status content, content warnings, and attachment descriptions. +- Add support for Mastodon-compatible webhooks + +### Fixed + +- Fixed `/api/v1/pleroma/accounts/:id/favourites` ordering and pagination to use the favourite date. +- Reject unsupported or ambiguous inbox signatures instead of queueing a doomed failed-signature retry, allowing clients to fall back to legacy HTTP Signatures. +- Switch native captcha to the published pleroma_captcha Hex package. +- Updated the captcha package to fix compilation failure on BSD-based systems +- Dockerfile: update base images to fix problems with SSL handshakes +- Docs: Fixed OTP install links in documentation, broke up wall of install commands into smaller chunks, fixed sudo commands execution +- Fix notification and timeline queries for users with domain blocks choosing a plan that scans every active user per row; the followed-accounts exception is now an `ARRAY(SELECT ...)` InitPlan that the planner costs once. +- Fixed Elixir 1.19 warnings +- Support fetching featured collection items from the first collection page +- Fix preferred frontend not working due to InstanceStatic plug prematurely returning HTTP 404 +- Fixed Gun HTTP/1 concurrency, forwarding and authenticated proxy handling, tunnel requests, and streamed connection cleanup. Updated Gun and Cowlib for HTTP client security fixes. +- Stop leaking a Hackney connection process, and the response bytes it read, for every unpooled request that ends without a reusable socket +- Fix hashtag timeline query for additional tag handling +- Fix compatibility with timestamped HTTP Signatures used by GoToSocial +- Handle reports with just actor ap id as the object +- Fix Swoosh HTTP mail adapters failing with `cacerts: :undefined` on OTP 26+ +- Echo Mastodon-style `Sec-WebSocket-Protocol` tokens in streaming WebSocket handshakes. +- Enforce the max_body HTTP option again, which Hackney 4 silently stopped honouring because it always returns a fully read body +- Abort incomplete media proxy responses instead of caching partial bodies. +- Updated Pleroma-FE build URL after Forgejo migration +- Fix votersCount inflation when same voter picks multiple options +- Status: Fill all MastoAPI required values in preview cards +- Avoid creating atoms from serialized PublisherWorker parameters. +- Reject incoming reports when both the reporter and reported account are remote +- Fixed account status and media views exposing activities restricted from unauthenticated users. +- Stop leaking a Hackney connection process for every media proxy response that was streamed to the end +- Fix RSS feeds to be W3C compliant and render better in RSS readers +- Keep followed users visible in streams when their domain is blocked. +- Fix intermittent failure to compile syslog package +- Fix inconsistent User cache keys and invalidate numeric ID aliases. +- Include arm32v7 in multi-arch Docker images +- RichMedia: Fix backfill causing old posts to show up on timelines by disabling it in MastoAPI StatusView +- Fix Admin-FE installation by downloading builds from the package registry instead of GitLab job artifacts. +- Hackney: use HTTP/1.1 only, so closing one media proxy response no longer breaks other downloads from the same host +- Hackney: use the configured proxy when its host is an IP address; Hackney 4 ignored such proxies and connected directly + ## 2.10.2 ### Security diff --git a/Dockerfile b/Dockerfile index 5fd1cc0a1..4018d9bc9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,8 +1,8 @@ # https://hub.docker.com/r/hexpm/elixir/tags ARG ELIXIR_IMG=hexpm/elixir ARG ELIXIR_VER=1.17.3 -ARG ERLANG_VER=26.2.5.6 -ARG ALPINE_VER=3.17.9 +ARG ERLANG_VER=27.3.4.17 +ARG ALPINE_VER=3.23.5 ARG ELIXIR_TAG=${ELIXIR_VER}-erlang-${ERLANG_VER}-alpine-${ALPINE_VER} ARG ALPINE_IMG=alpine diff --git a/README.md b/README.md index 982a7249a..bccb39bdf 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ - + ## About diff --git a/changelog.d/captcha-package.fix b/changelog.d/captcha-package.fix deleted file mode 100644 index 98c2f6525..000000000 --- a/changelog.d/captcha-package.fix +++ /dev/null @@ -1 +0,0 @@ -Switch native captcha to the published pleroma_captcha Hex package. diff --git a/changelog.d/context-cleanup.skip b/changelog.d/context-cleanup.skip deleted file mode 100644 index ae609602d..000000000 --- a/changelog.d/context-cleanup.skip +++ /dev/null @@ -1 +0,0 @@ -litepub-0.1.jsonld cleanup diff --git a/changelog.d/featured-collection-page.fix b/changelog.d/featured-collection-page.fix deleted file mode 100644 index b611a4137..000000000 --- a/changelog.d/featured-collection-page.fix +++ /dev/null @@ -1 +0,0 @@ -Support fetching featured collection items from the first collection page diff --git a/changelog.d/host-header-verification.security b/changelog.d/host-header-verification.security deleted file mode 100644 index 11a0784b2..000000000 --- a/changelog.d/host-header-verification.security +++ /dev/null @@ -1 +0,0 @@ -Ensure Host header is present and matches instance URI diff --git a/changelog.d/http-signatures-0.1.3.fix b/changelog.d/http-signatures-0.1.3.fix deleted file mode 100644 index 3be1b9e9f..000000000 --- a/changelog.d/http-signatures-0.1.3.fix +++ /dev/null @@ -1 +0,0 @@ -Fix compatibility with timestamped HTTP Signatures used by GoToSocial diff --git a/changelog.d/iceshrimpnet-reports-fix.fix b/changelog.d/iceshrimpnet-reports-fix.fix deleted file mode 100644 index 7487ae47e..000000000 --- a/changelog.d/iceshrimpnet-reports-fix.fix +++ /dev/null @@ -1 +0,0 @@ -Handle reports with just actor ap id as the object diff --git a/changelog.d/legacy-sequences.fix b/changelog.d/legacy-sequences.fix new file mode 100644 index 000000000..0ecd4842d --- /dev/null +++ b/changelog.d/legacy-sequences.fix @@ -0,0 +1 @@ +Delete legacy Postgres sequences for activities, chats, and users which were converted to Flake IDs diff --git a/changelog.d/majic-1.2.0.change b/changelog.d/majic-1.2.0.change deleted file mode 100644 index bc8d5eb3c..000000000 --- a/changelog.d/majic-1.2.0.change +++ /dev/null @@ -1 +0,0 @@ -Update majic to 1.2.0. diff --git a/changelog.d/mastodon-websocket-protocol.fix b/changelog.d/mastodon-websocket-protocol.fix deleted file mode 100644 index 66dab16ed..000000000 --- a/changelog.d/mastodon-websocket-protocol.fix +++ /dev/null @@ -1 +0,0 @@ -Echo Mastodon-style `Sec-WebSocket-Protocol` tokens in streaming WebSocket handshakes. diff --git a/changelog.d/mfm-backend.add b/changelog.d/mfm-backend.add deleted file mode 100644 index f815c6828..000000000 --- a/changelog.d/mfm-backend.add +++ /dev/null @@ -1 +0,0 @@ -Add backend support for Misskey Markdown (MFM) posts diff --git a/changelog.d/mfm-extend.skip b/changelog.d/mfm-extend.skip deleted file mode 100644 index e69de29bb..000000000 diff --git a/changelog.d/oban-plugins-lazarus-hex.change b/changelog.d/oban-plugins-lazarus-hex.change deleted file mode 100644 index 9259df0f5..000000000 --- a/changelog.d/oban-plugins-lazarus-hex.change +++ /dev/null @@ -1 +0,0 @@ -Use the Hex package for oban_plugins_lazarus. diff --git a/changelog.d/phoenix-sec-websocket-headers.change b/changelog.d/phoenix-sec-websocket-headers.change deleted file mode 100644 index b3cb4ad3f..000000000 --- a/changelog.d/phoenix-sec-websocket-headers.change +++ /dev/null @@ -1 +0,0 @@ -Switch patched Phoenix 1.7.14 back to upstream with Phoenix 1.8.0+ diff --git a/changelog.d/pleroma-fe-link.fix b/changelog.d/pleroma-fe-link.fix deleted file mode 100644 index de93f86dd..000000000 --- a/changelog.d/pleroma-fe-link.fix +++ /dev/null @@ -1 +0,0 @@ -Updated Pleroma-FE build URL after Forgejo migration diff --git a/changelog.d/poll-voters-count-inflation.fix b/changelog.d/poll-voters-count-inflation.fix deleted file mode 100644 index 7eae41f13..000000000 --- a/changelog.d/poll-voters-count-inflation.fix +++ /dev/null @@ -1 +0,0 @@ -Fix votersCount inflation when same voter picks multiple options diff --git a/changelog.d/reject-third-party-reports.fix b/changelog.d/reject-third-party-reports.fix deleted file mode 100644 index 7d4e87b94..000000000 --- a/changelog.d/reject-third-party-reports.fix +++ /dev/null @@ -1 +0,0 @@ -Reject incoming reports when both the reporter and reported account are remote diff --git a/changelog.d/docker-e2e-image.skip b/changelog.d/release-2.11.0.skip similarity index 100% rename from changelog.d/docker-e2e-image.skip rename to changelog.d/release-2.11.0.skip diff --git a/changelog.d/remote-ip-hex.change b/changelog.d/remote-ip-hex.change deleted file mode 100644 index 44f6bc1bf..000000000 --- a/changelog.d/remote-ip-hex.change +++ /dev/null @@ -1 +0,0 @@ -Use upstream Hex releases for the `remote_ip` dependency and expose client IP ranges for remote IP resolution. diff --git a/changelog.d/user-search-sorting.change b/changelog.d/user-search-sorting.change deleted file mode 100644 index def27e955..000000000 --- a/changelog.d/user-search-sorting.change +++ /dev/null @@ -1 +0,0 @@ -Improve user search / autocompletion ordering. diff --git a/changelog.d/weblate-ci.skip b/changelog.d/weblate-ci.skip deleted file mode 100644 index e69de29bb..000000000 diff --git a/changelog.d/wss-necroposts.fix b/changelog.d/wss-necroposts.fix deleted file mode 100644 index 743f9087c..000000000 --- a/changelog.d/wss-necroposts.fix +++ /dev/null @@ -1 +0,0 @@ -RichMedia: Fix backfill causing old posts to show up on timelines by disabling it in MastoAPI StatusView diff --git a/config/config.exs b/config/config.exs index 3f880af53..1b8baed1f 100644 --- a/config/config.exs +++ b/config/config.exs @@ -133,13 +133,17 @@ config :pleroma, Pleroma.Web.Endpoint, ] # Configures Elixir's Logger -config :logger, backends: [:console] +# Primary config +config :logger, level: :debug -config :logger, :console, - level: :debug, +# Console config +config :logger, :default_handler, level: :debug + +config :logger, :default_formatter, format: "\n$time $metadata[$level] $message\n", metadata: [:actor, :path, :type, :user] +# Syslog config config :logger, :ex_syslogger, level: :debug, ident: "pleroma", @@ -157,7 +161,11 @@ config :mime, :types, %{ "image/apng" => ["apng"] } -config :tesla, adapter: Tesla.Adapter.Hackney +config :tesla, adapter: Tesla.Adapter.Gun + +# Hackney 4 shares one HTTP/2 connection per host, and closing one response +# (as the media proxy does) stops it for every other stream on it. +config :hackney, default_protocols: [:http1] # Configures http settings, upstream proxy etc. config :pleroma, :http, @@ -603,7 +611,11 @@ config :pleroma, Oban, search_indexing: [limit: 10, paused: true], slow: 5 ], - plugins: [Oban.Plugins.Lazarus, {Oban.Plugins.Pruner, max_age: 900}], + plugins: [ + Oban.Plugins.Lazarus, + {Oban.Plugins.Pruner, max_age: 900}, + {Oban.Plugins.Reindexer, schedule: "@weekly"} + ], crontab: [ {"0 0 * * 0", Pleroma.Workers.Cron.DigestEmailsWorker}, {"0 0 * * *", Pleroma.Workers.Cron.NewUsersDigestWorker}, @@ -794,7 +806,7 @@ config :pleroma, :frontends, "name" => "admin-fe", "git" => "https://git.pleroma.social/pleroma/admin-fe", "build_url" => - "https://git.pleroma.social/pleroma/admin-fe/-/jobs/artifacts/${ref}/download?job=build", + "https://git.pleroma.social/api/packages/pleroma/generic/admin-fe-builds/${ref}/latest.zip", "ref" => "develop" }, "soapbox" => %{ @@ -928,7 +940,8 @@ config :pleroma, Pleroma.User.Backup, timeout: :timer.minutes(30) config :pleroma, ConcurrentLimiter, [ - {Pleroma.Search, [max_running: 30, max_waiting: 50]} + {Pleroma.Search, [max_running: 30, max_waiting: 50]}, + {Pleroma.Webhook.Notify, [max_running: 5, max_waiting: 200]} ] config :pleroma, Pleroma.Web.WebFinger, domain: nil, update_nickname_on_user_fetch: true @@ -940,6 +953,15 @@ config :pleroma, Pleroma.Search.Meilisearch, private_key: nil, initial_indexing_chunk_size: 100_000 +config :pleroma, Pleroma.Search.ParadeDB, + url: nil, + table: "pleroma_search_documents", + fuzzy_distance: 0 + +config :pleroma, Pleroma.Search.ParadeDB.Repo, + pool_size: 2, + prepare: :unnamed + config :pleroma, Pleroma.Application, background_migrators: true, internal_fetch: true, @@ -962,6 +984,10 @@ config :pleroma, Pleroma.Search.QdrantSearch, vectors: %{size: 384, distance: "Cosine"} } +config :pleroma, :database_config_blacklist, [ + {:pleroma, :logger} +] + config :pleroma, :database_config_whitelist, [ {:pleroma}, {:cors_plug}, @@ -971,6 +997,8 @@ config :pleroma, :database_config_whitelist, [ {:web_push_encryption, :vapid_details} ] +config :pleroma, Pleroma.Chat, enabled: true + # Import environment specific config. This must remain at the bottom # of this file so it overrides the configuration defined above. import_config "#{Mix.env()}.exs" diff --git a/config/description.exs b/config/description.exs index 7c377588c..f435628aa 100644 --- a/config/description.exs +++ b/config/description.exs @@ -2726,6 +2726,19 @@ config :pleroma, :config_description, [ } ] }, + %{ + group: :pleroma, + key: Pleroma.Chat, + type: :group, + description: "Pleroma Chat settings", + children: [ + %{ + key: :enabled, + type: :boolean, + description: "Enables the chats API." + } + ] + }, %{ group: :pleroma, key: :http, @@ -3085,9 +3098,16 @@ config :pleroma, :config_description, [ %{ key: :max_connections, type: :integer, - description: "Maximum number of connections in the pool. Default: 250 connections.", + description: + "Maximum total number of connections in the pool. HTTP/1 origins may use multiple connections within this limit, while HTTP/2 connections are multiplexed. Default: 250 connections.", suggestions: [250] }, + %{ + key: :max_idle_time, + type: :integer, + description: "Time before an unused connection is closed. Default: 30000ms.", + suggestions: [30_000] + }, %{ key: :connect_timeout, type: :integer, @@ -3503,6 +3523,26 @@ config :pleroma, :config_description, [ suggestion: [5] } ] + }, + %{ + key: Pleroma.Webhook.Notify, + type: :keyword, + description: "Concurrent limits configuration for webhooks.", + suggestions: [max_running: 5, max_waiting: 200], + children: [ + %{ + key: :max_running, + type: :integer, + description: "Max running concurrently jobs.", + suggestion: [5] + }, + %{ + key: :max_waiting, + type: :integer, + description: "Max waiting jobs.", + suggestion: [200] + } + ] } ] }, @@ -3516,7 +3556,12 @@ config :pleroma, :config_description, [ key: :module, type: :keyword, description: "Selected search module.", - suggestion: [Pleroma.Search.DatabaseSearch, Pleroma.Search.Meilisearch] + suggestion: [ + Pleroma.Search.DatabaseSearch, + Pleroma.Search.Meilisearch, + Pleroma.Search.QdrantSearch, + Pleroma.Search.ParadeDB + ] } ] }, @@ -3549,6 +3594,34 @@ config :pleroma, :config_description, [ } ] }, + %{ + group: :pleroma, + key: Pleroma.Search.ParadeDB, + type: :group, + description: "ParadeDB settings.", + children: [ + %{ + key: :url, + type: :string, + description: + "ParadeDB database URL (separate Postgres instance). " <> + "Can also be set via `PARADEDB_DATABASE_URL`.", + suggestion: ["postgres://postgres:postgres@127.0.0.1:5432/paradedb"] + }, + %{ + key: :table, + type: :string, + description: "Table name used for search documents.", + suggestion: ["pleroma_search_documents"] + }, + %{ + key: :fuzzy_distance, + type: :integer, + description: "Edit-distance used for fuzzy token matching (0 disables fuzziness, max 2).", + suggestion: [0, 1, 2] + } + ] + }, %{ group: :pleroma, key: Pleroma.Language.LanguageDetector, diff --git a/config/dev.exs b/config/dev.exs index 14cf4a6dc..e86bdb7fd 100644 --- a/config/dev.exs +++ b/config/dev.exs @@ -36,7 +36,7 @@ config :pleroma, Pleroma.Emails.Mailer, adapter: Swoosh.Adapters.Local # different ports. # Do not include timestamps in development logs -config :logger, Logger.Backends.Console, format: "$metadata[$level] $message\n" +config :logger, :default_formatter, format: "$metadata[$level] $message\n" # Set a higher stacktrace during development. Avoid configuring such # in production as building large stacktraces may be expensive. diff --git a/config/prod.exs b/config/prod.exs index 2d252bf02..1dfbbe937 100644 --- a/config/prod.exs +++ b/config/prod.exs @@ -20,8 +20,7 @@ config :pleroma, Pleroma.Web.Endpoint, config :phoenix, serve_endpoints: true # Do not print debug messages in production -config :logger, Logger.Backends.Console, level: :info -config :logger, :console, level: :info +config :logger, :default_handler, level: :info config :logger, :ex_syslogger, level: :info # ## SSL Support diff --git a/config/test.exs b/config/test.exs index 9652b7d4b..ed5283a26 100644 --- a/config/test.exs +++ b/config/test.exs @@ -15,9 +15,11 @@ config :pleroma, Pleroma.Captcha, method: Pleroma.Captcha.Mock # Print only warnings and errors during test -config :logger, :console, - level: :warning, - format: "\n[$level] $message\n" +config :logger, :default_handler, level: :warning + +config :logger, :default_formatter, + format: "\n[$level] $message\n", + colors: [enabled: false] config :pleroma, :auth, oauth_consumer_strategies: [] diff --git a/docs/administration/CLI_tasks/frontend.md b/docs/administration/CLI_tasks/frontend.md index 4e9d9eecb..404d016ed 100644 --- a/docs/administration/CLI_tasks/frontend.md +++ b/docs/administration/CLI_tasks/frontend.md @@ -76,6 +76,22 @@ The resulting frontend will always be installed into a folder of this template: Careful: This folder will be completely replaced on installation. +## Installing Admin-FE + +=== "OTP" + + ```sh + ./bin/pleroma_ctl frontend install admin-fe --ref stable + ``` + +=== "From Source" + + ```sh + mix pleroma.frontend install admin-fe --ref stable + ``` + +Builds come from the package registry: `https://git.pleroma.social/api/packages/pleroma/generic/admin-fe-builds/${ref}/latest.zip`. `stable` and `develop` are published; `stable` is the better choice for production. A release tag works as `--ref` once a build for it is published. + ## Example installation for an unknown frontend The installation process is the same, but you will have to give all the needed options on the command line. For example: diff --git a/docs/administration/frontends-management.md b/docs/administration/frontends-management.md index f982c4bca..9a3bec37d 100644 --- a/docs/administration/frontends-management.md +++ b/docs/administration/frontends-management.md @@ -56,7 +56,7 @@ Due to aforementioned bug, preferred way of installing frontends in AdminFE is b ![screenshot of admin-fe with instructions on how to install a frontend](../assets/way_to_install_frontends.png) and filling in the fields. Unfortunately AdminFE does not provide the raw data necessary for you to fill those fields, so your best bet is to see what backend returns in browser's devtools or refer to the [source code](https://git.pleroma.social/pleroma/pleroma/-/blob/develop/config/config.exs?ref_type=heads#L742-791). For the most part, only **Name**, **Ref** (i.e. version) and **Build URL** fields are required, although some frontends might also require **Build Directory** to work. -For pleroma-fe you can use either `master` or `develop` refs, or potentially any ref in GitLab that has artifacts for `build` job, but that's outside scope of this document. +Pleroma-FE and Admin-FE builds come from the package registry, not from GitLab job artifacts. For Admin-FE, set **Name** to `admin-fe`, **Ref** to `stable` (or `develop`) and **Build URL** to `https://git.pleroma.social/api/packages/pleroma/generic/admin-fe-builds/${ref}/latest.zip`. Only refs with a published build can be installed: `stable`, `develop`, and release tags once they are published. ### Selecting default frontend diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md index 9efa1c8b3..0460caf26 100644 --- a/docs/configuration/cheatsheet.md +++ b/docs/configuration/cheatsheet.md @@ -541,6 +541,15 @@ Available caches: ## HTTP client +Pleroma uses [Gun](https://github.com/ninenines/gun) for outgoing HTTP requests. To use Hackney instead: + +```elixir +config :tesla, adapter: Tesla.Adapter.Hackney +``` + +!!! note + With Hackney, size limits for fetched responses (for example for link previews) are checked only after Hackney has read the whole response. They limit what Pleroma keeps, not how much it downloads or holds in memory. Gun stops reading at the limit. + ### :http * `proxy_url`: an upstream proxy to fetch posts and/or media with, (default: `nil`) @@ -550,6 +559,8 @@ Available caches: ### :hackney_pools +*For `hackney` adapter* + Advanced. Tweaks Hackney (http client) connections pools. There's three pools used: @@ -573,7 +584,8 @@ Settings for HTTP connection pool. * `:connection_acquisition_wait` - Timeout to acquire a connection from pool.The total max time is this value multiplied by the number of retries. * `connection_acquisition_retries` - Number of attempts to acquire the connection from the pool if it is overloaded. Each attempt is timed `:connection_acquisition_wait` apart. -* `:max_connections` - Maximum number of connections in the pool. +* `:max_connections` - Maximum total number of connections in the pool. HTTP/1 origins may use multiple connections within this limit, while HTTP/2 connections are multiplexed. +* `:max_idle_time` - Time before an unused connection is closed. * `:connect_timeout` - Timeout to connect to the host. * `:reclaim_multiplier` - Multiplied by `:max_connections` this will be the maximum number of idle connections that will be reclaimed in case the pool is overloaded. @@ -858,21 +870,23 @@ Web Push Notifications configuration. You can use the mix task `mix web_push.gen * ``private_key``: VAPID private key ## :logger -* `backends`: `:console` is used to send logs to stdout, `{ExSyslogger, :ex_syslogger}` to log to syslog +* Logging to console/stdout is done by default, use `{ExSyslogger, :ex_syslogger}` to log to syslog An example to enable ONLY ExSyslogger (f/ex in ``prod.secret.exs``) with info and debug suppressed: ```elixir -config :logger, +config :pleroma, :logger, backends: [{ExSyslogger, :ex_syslogger}] +config :logger, default_handler: false + config :logger, :ex_syslogger, level: :warning ``` Another example, keeping console output and adding the pid to syslog output: ```elixir -config :logger, - backends: [:console, {ExSyslogger, :ex_syslogger}] +config :pleroma, :logger, + backends: [{ExSyslogger, :ex_syslogger}] config :logger, :ex_syslogger, level: :warning, @@ -883,23 +897,22 @@ See: [logger’s documentation](https://hexdocs.pm/logger/Logger.html) and [ex_s An example of logging info to local syslog, but debug to console: ```elixir -config :logger, - backends: [ {ExSyslogger, :ex_syslogger}, :console ], - level: :info +config :pleroma, :logger, + backends: [{ExSyslogger, :ex_syslogger}] config :logger, :ex_syslogger, level: :info, ident: "pleroma", format: "$metadata[$level] $message" -config :logger, :console, - level: :debug, +config :logger, :default_handler, + level: :debug + +config :logger, :default_formatter, format: "\n$time $metadata[$level] $message\n", metadata: [:request_id] ``` - - ## Database options ### RUM indexing for full text search diff --git a/docs/configuration/howto_proxy.md b/docs/configuration/howto_proxy.md index 10a635266..d805ff965 100644 --- a/docs/configuration/howto_proxy.md +++ b/docs/configuration/howto_proxy.md @@ -10,3 +10,6 @@ The other way to do it, for example, with Tor you would most likely add somethin ``` config :pleroma, :http, proxy_url: {:socks5, :localhost, 9050} ``` + +The Gun HTTP adapter supports SOCKS5. SOCKS4 proxies require another HTTP adapter. +For regular HTTP proxies, Gun forwards plain HTTP requests directly and uses CONNECT tunnels for HTTPS requests. diff --git a/docs/configuration/search.md b/docs/configuration/search.md index d34f84d4f..5820cdbd9 100644 --- a/docs/configuration/search.md +++ b/docs/configuration/search.md @@ -85,7 +85,7 @@ You will see a "Default Admin API Key", this is the key you actually put into yo ### Initial indexing -After setting up the configuration, you'll want to index all of your already existing posts. Only public posts are indexed. You'll only +After setting up the configuration, you'll want to index all of your already existing posts. Only public/unlisted posts are indexed. You'll only have to do it one time, but it might take a while, depending on the amount of posts your instance has seen. This is also a fairly RAM consuming process for `meilisearch`, and it will take a lot of RAM when running if you have a lot of posts (seems to be around 5G for ~1.2 million posts while idle and up to 7G while indexing initially, but your experience may be different). @@ -145,3 +145,155 @@ This will clear **all** the posts from the search index. Note, that deleted post there is no need to actually clear the whole index, unless you want **all** of it gone. That said, the index does not hold any information that cannot be re-created from the database, it should also generally be a lot smaller than the size of your database. Still, the size depends on the amount of text in posts. + +## ParadeDB + +[ParadeDB](https://www.paradedb.com/) is a Postgres extension that provides BM25 full-text search. In Pleroma, it can be used as an +external search backend by pointing Pleroma at a separate Postgres instance with the `pg_search` extension installed. +Pleroma's integration suite tests `pg_search` 0.24.3 on PostgreSQL 15. + +Pleroma will maintain a small `pleroma_search_documents` table in that database (via the existing search indexing queue) and run search +queries against it. +The indexed text includes status content, content warnings, and attachment descriptions. + +### Configuration + +Set the search module to `Pleroma.Search.ParadeDB` and configure the ParadeDB database URL. These settings must be in static/runtime config +(or `PARADEDB_DATABASE_URL`) so the dedicated ParadeDB Repo is started with the application. They cannot be migrated to or changed through +ConfigDB. + +> config :pleroma, Pleroma.Search, module: Pleroma.Search.ParadeDB +> +> config :pleroma, Pleroma.Search.ParadeDB, +> url: System.get_env("PARADEDB_DATABASE_URL"), +> table: "pleroma_search_documents", +> fuzzy_distance: 0 +> +> config :pleroma, Pleroma.Search.ParadeDB.Repo, +> pool_size: 2, +> prepare: :unnamed + +When connecting over an untrusted network, enable certificate verification in the Repo configuration. For example: + +> config :pleroma, Pleroma.Search.ParadeDB.Repo, +> ssl: [ +> verify: :verify_peer, +> cacertfile: "/etc/pleroma/paradedb-ca.pem", +> server_name_indication: ~c"paradedb.example.com" +> ] + +### Initial rollout + +The `pg_search` extension must already be available in the database. Creating the extension normally requires an administrative database +role. The Pleroma runtime role requires `CONNECT` on the database, `USAGE` on the table's schema and the `pdb` schema, and `SELECT`, `INSERT`, `UPDATE`, +and `DELETE` on the generated table. For example, after creating the table as its owner: + +```sql +GRANT CONNECT ON DATABASE paradedb TO pleroma_search; +GRANT USAGE ON SCHEMA public, pdb TO pleroma_search; +GRANT SELECT, INSERT, UPDATE, DELETE ON TABLE pleroma_search_documents TO pleroma_search; +``` + +Use this rollout order to prevent jobs from being handled by the previous backend: + +1. Install `pg_search` 0.24.3 on PostgreSQL 15 or another deliberately tested combination. +2. Stop Pleroma so no search indexing jobs are running. +3. Put the ParadeDB backend in static configuration. +4. Create the table and BM25 index in a one-shot offline process using `PARADEDB_DATABASE_URL` with the table owner/setup credential. +5. Apply the runtime grants shown above, set the service's `PARADEDB_DATABASE_URL` to the restricted runtime credential, and start Pleroma. +6. Start the initial backfill and monitor the `search_indexing` queue. + +Create the table and BM25 index once with: + +=== "OTP" + ```sh + PARADEDB_DATABASE_URL='OWNER_URL' PLEROMA_CTL_RPC_DISABLED=true \ + ./bin/pleroma_ctl search.indexer create_index + ``` + +=== "From Source" + ```sh + PARADEDB_DATABASE_URL='OWNER_URL' mix pleroma.search.indexer create_index + ``` + +The task currently prints an error without returning a failing process status. Verify that it prints `Index created` before continuing. + +### Backfill + +Run an initial indexing pass to enqueue existing posts: + +=== "OTP" + ```sh + ./bin/pleroma_ctl search.indexer index + ``` + +=== "From Source" + ```sh + PARADEDB_DATABASE_URL='RUNTIME_URL' mix pleroma.search.indexer index + ``` + +The command enqueues up to 100,000 Create activities by default. It uses keyset pagination, so new activities arriving during the backfill +do not shift the remaining pages. The defaults and accepted options are: + +* `--limit`: maximum activities to enqueue in this run; defaults to `100000` and must be positive +* `--step`: activities fetched from Pleroma's database per page; defaults to `1000` and must be positive +* `--chunk`: jobs inserted into Oban per batch; defaults to `100` and must be positive +* `--before`: resume before the checkpoint activity ID printed by an earlier run; the cursor is exclusive + +The checkpoint is the last activity enqueued, not confirmation that its job succeeded. Repeat the command with its last `--before` +checkpoint only when it reports that more activities remain. Inspect failed or discarded Oban jobs as well as waiting jobs before declaring +the backfill complete. + +The same operation can be run from an attached console while Pleroma is online: + +```elixir +Pleroma.Search.Backfill.run(limit: 10_000) +``` + +The returned `next_cursor` can be supplied as `before` when `exhausted` is `false`. The attached API also accepts `limit: :infinity`, but this +can create an unbounded number of Oban rows; bounded runs are safer on large instances. An `on_page` callback may be supplied for progress +reporting. + +Note: Like the other external search backends, only public/unlisted Notes are indexed. + +Set `fuzzy_distance` to `1` or `2` to allow typo-tolerant matching (`2` is the maximum ParadeDB supports). + +Indexing runs via the Oban `search_indexing` queue. Its stock configuration starts paused so the search health monitor can control it; verify +that the queue is running before backfilling. A manually managed queue can be enabled with `search_indexing: 10` in the Oban `queues` +configuration instead of a `paused: true` entry. +Monitor that queue until the backfill jobs have drained before evaluating search completeness. + +### Rebuilding + +Stop Pleroma, or otherwise ensure no search indexing jobs are running, before rebuilding. While ParadeDB is still the configured backend, +run `drop_index`, then `create_index` with the table owner/setup credential. Reapply runtime grants after recreating the table, restart +Pleroma, and begin a new backfill. The DDL tasks print failures without returning a failing process status, so verify the exact `Index +dropped` and `Index created` success messages. Dropping the table removes only derived search data; Pleroma activities remain in the primary +database. + +### Rolling back + +To retain the ParadeDB table for a later retry, configure `Pleroma.Search.DatabaseSearch` (or the previous external backend) in static config +and restart Pleroma. + +Before switching backends, inspect the `search_indexing` queue. Drain or cancel pending backfill jobs, or explicitly accept that they will run +against the newly configured backend after restart. + +To remove the table during rollback, first stop Pleroma and run `drop_index` with the table owner/setup credential while ParadeDB is still +configured: + +=== "OTP" + ```sh + PARADEDB_DATABASE_URL='OWNER_URL' PLEROMA_CTL_RPC_DISABLED=true \ + ./bin/pleroma_ctl search.indexer drop_index + ``` + +=== "From Source" + ```sh + PARADEDB_DATABASE_URL='OWNER_URL' mix pleroma.search.indexer drop_index + ``` + +Verify that the task prints `Index dropped`. Then change the backend in static config and restart. The ParadeDB environment variable may be +removed after removing the ParadeDB stanza or switching to a backend that does not start its Repo. After switching away from ParadeDB, the +generic `drop_index` task targets the newly configured backend and will not remove the ParadeDB table; remove it directly with the database +owner instead. diff --git a/docs/development/API/admin_api.md b/docs/development/API/admin_api.md index 3719ceeb9..5f8a886be 100644 --- a/docs/development/API/admin_api.md +++ b/docs/development/API/admin_api.md @@ -80,10 +80,24 @@ The `/api/v1/pleroma/admin/*` path is backwards compatible with `/api/pleroma/ad { `nickname`, `email`, - `password` + `password` (optional; when provided, must not be empty) } ] -- Response: Array of user objects +- Response: Array of user objects. When a user is created without a `password`, `data` + object contains a `password_reset_link` the user can follow to set their own password. + +```json +[ + { + "data": { + "nickname": string, + "email": string, + "password_reset_link": string (optional) + }, + "type": "success" + } +] +``` ## `POST /api/v1/pleroma/admin/users/follow` @@ -1839,3 +1853,92 @@ Note that this differs from the Mastodon API variant: Mastodon API only returns ```json {} ``` + +## `GET /api/v1/pleroma/admin/webhooks` + +### List webhooks + +- Method: `GET` +- Response: + +```json +[ + { + "enabled": true, + "id": "2", + "events": ["account.created"], + "url": "https://webhook.example/", + "secret": "eb85d4ccd8510e78f912743949dc354e8146987d", + "updated_at": "2022-10-29T17:44:16.000Z", + "created_at": "2022-10-29T17:44:13.000Z" + } +] +``` + +## `GET /api/v1/pleroma/admin/webhooks/:id` + +### Get an individual webhook + +- Method: `GET` +- Params: + - `id`: **string** Webhook ID +- Response: A webhook + +## `POST /api/v1/pleroma/admin/webhooks` + +### Create a webhook + +- Method: `POST` +- Params: + - `url`: **string** Webhook URL + - `events`: **[string]** Types of events to trigger on (`account.created`, `report.created`) + - *optional* `enabled`: **boolean** Whether webhook is enabled +- Response: A webhook + +## `PATCH /api/v1/pleroma/admin/webhooks/:id` + +### Update a webhook + +- Method: `PATCH` +- Params: + - `id`: **string** Webhook ID + - *optional* `url`: **string** Webhook URL + - *optional* `events`: **[string]** Types of events to trigger on (`account.created`, `report.created`) + - *optional* `enabled`: **boolean** Whether webhook is enabled +- Response: A webhook + +## `DELETE /api/v1/pleroma/admin/webhooks/:id` + +### Delete a webhook + +- Method: `DELETE` +- Params: + - `id`: **string** Webhook ID +- Response: A webhook + +## `POST /api/v1/pleroma/admin/webhooks/:id/enable` + +### Activate a webhook + +- Method: `POST` +- Params: + - `id`: **string** Webhook ID +- Response: A webhook + +## `POST /api/v1/pleroma/admin/webhooks/:id/disable` + +### Deactivate a webhook + +- Method: `POST` +- Params: + - `id`: **string** Webhook ID +- Response: A webhook + +## `POST /api/v1/pleroma/admin/webhooks/:id/rotate_secret` + +### Rotate webhook signing secret + +- Method: `POST` +- Params: + - `id`: **string** Webhook ID +- Response: A webhook diff --git a/docs/installation/debian_based_en.md b/docs/installation/debian_based_en.md index 30f48792d..f9086d2f9 100644 --- a/docs/installation/debian_based_en.md +++ b/docs/installation/debian_based_en.md @@ -4,7 +4,7 @@ ## Installation -This guide will assume you are on Debian 12 (“bookworm”) or later. This guide should also work with Ubuntu 22.04 (“jammy”) and later. It also assumes that you have administrative rights, either as root or a user with [sudo permissions](https://www.digitalocean.com/community/tutorials/how-to-add-delete-and-grant-sudo-privileges-to-users-on-a-debian-vps). If you want to run this guide with root, ignore the `sudo` at the beginning of the lines, unless it calls a user like `sudo -Hu pleroma`; in this case, use `su -s $SHELL -c 'command'` instead. +This guide will assume you are on Debian 13 (“Trixie”) or later. This guide should also work with Ubuntu 26.04 (“resolute”) and later. It also assumes that you have administrative rights, either as root or a user with [sudo permissions](https://www.digitalocean.com/community/tutorials/how-to-add-delete-and-grant-sudo-privileges-to-users-on-a-debian-vps). If you want to run this guide with root, ignore the `sudo` at the beginning of the lines, unless it calls a user like `sudo -Hu pleroma`; in this case, use `su -s $SHELL -c 'command'` instead. {! backend/installation/generic_dependencies.include !} diff --git a/docs/installation/debian_based_jp.md b/docs/installation/debian_based_jp.md index 5b261296d..c853840c2 100644 --- a/docs/installation/debian_based_jp.md +++ b/docs/installation/debian_based_jp.md @@ -14,7 +14,7 @@ Note: This article is potentially outdated because at this time we may not have - PostgreSQL 11.0以上 (Ubuntu16.04では9.5しか提供されていないので，[](https://www.postgresql.org/download/linux/ubuntu/)こちらから新しいバージョンを入手してください) - `postgresql-contrib` 11.0以上 (同上) -- Elixir 1.15 以上 ([Debianのリポジトリからインストールしないこと！！！ ここからインストールすること！](https://elixir-lang.org/install.html#unix-and-unix-like)。または [asdf](https://github.com/asdf-vm/asdf) をpleromaユーザーでインストールしてください) +- Elixir 1.17 以上 (Erlang/OTP 27 以上) ([Debianのリポジトリからインストールしないこと！！！ ここからインストールすること！](https://elixir-lang.org/install.html#unix-and-unix-like)。または [asdf](https://github.com/asdf-vm/asdf) をpleromaユーザーでインストールしてください) - `erlang-dev` - `erlang-nox` - `git` diff --git a/docs/installation/erlang_elixir_requirements.include b/docs/installation/erlang_elixir_requirements.include new file mode 100644 index 000000000..395d0f9de --- /dev/null +++ b/docs/installation/erlang_elixir_requirements.include @@ -0,0 +1,2 @@ +* Elixir >=1.17.0, <1.20 +* Erlang OTP >=27.0.0 (supported: <29) diff --git a/docs/installation/generic_dependencies.include b/docs/installation/generic_dependencies.include index 588a6c019..86dbc6838 100644 --- a/docs/installation/generic_dependencies.include +++ b/docs/installation/generic_dependencies.include @@ -1,8 +1,7 @@ ## Required dependencies +{! backend/installation/erlang_elixir_requirements.include !} * PostgreSQL >=11.0 -* Elixir >=1.15.0 <1.19 -* Erlang OTP >=23.0.0 (supported: <28) * git * file / libmagic * gcc or clang @@ -14,3 +13,7 @@ * ImageMagick * FFmpeg * exiftool + +!!!info + **Source installs only** + If your operating system repositories don't meet the Elixir/Erlang requirements, check [Installing newer versions of Elixir and Erlang](./installing_newer_elixir_erlang.md). diff --git a/docs/installation/installing_newer_elixir_erlang.md b/docs/installation/installing_newer_elixir_erlang.md new file mode 100644 index 000000000..0220996d8 --- /dev/null +++ b/docs/installation/installing_newer_elixir_erlang.md @@ -0,0 +1,333 @@ +# Installing newer versions of Elixir and Erlang + +This guide is for installing newer versions of Elixir and/or Erlang on your system when installing Pleroma from source and your operating system repositories don't meet the minimum requirements listed below. +This guide assumes you are using Debian or Ubuntu. +**Using the system repositories is recommended if possible.** +**If a newer release of your operating system would meet the requirements, consider upgrading instead.** + +Shell commands prepended with `#` in this page are supposed to be ran with root privileges, commands prepended with `$` are supposed to be ran as the Pleroma user unless noted otherwise. + +!!!warning + If you are, or will be, following the source install guide, make sure to switch to the Pleroma user before running any of the `mix` commands using `sudo -Hu pleroma /bin/bash`. + Not doing so will result in `command not found` errors. + +!!!info + Most of this guide can be used to install Elixir and/or Erlang on operating systems that don't package either one. + Installing Pleroma on such systems is largely untested. + +## Requirements + +{! backend/installation/erlang_elixir_requirements.include !} + +## Install methods + +Choose one of the following install methods: + +* User level: + * mise (Erlang and Elixir install methods are built-in[^1]) + * asdf (plugin-based, plugins are shells scripts; builds Erlang from source) + * pkgsrc (for advanced users, harder to install and maintain; builds Elixir and Erlang from source) +* System-wide: + * pkgsrc (for advanced users, harder to install and maintain; builds Elixir and Erlang from source) + +[^1]: On Ubuntu downloads binaries both for Elixir and Erlang. On Debian builds Erlang from source and downloads binaries for Elixir. + +!!!warning + **Ignore if using mise on Ubuntu.** + You are about to compile Erlang on your server. + If you are planning to run Pleroma on a VPS virtual machine, make sure you have enough disk space and system resources available. + At least 2GB of RAM and 8GB of extra free disk space is required to continue in this guide. + If your system does not meet these requirements, use [OTP install](./otp_en.md). + **This will take a considerable amount of time.** + +!!!warning + Before proceeding, stop Pleroma if already running and uninstall both Elixir and Erlang. + Keeping Elixir will likely result in a non-functional Elixir install and keeping Erlang can result in various failures to compile Pleroma and its dependencies. + +## mise + +To start, create the Pleroma user according to the [source install guide](./debian_based_en.md#install-pleromabe). + +### Debian-only preparation + +If you are using Ubuntu, skip to the [next section](./installing_newer_elixir_erlang.md#elixirerlang-installation). + +Install dependencies: +``` +# apt install build-essential libssl-dev libncurses-dev git unzip +``` + +Add Erlang autotools configure options to Pleroma user's shell config (`.bashrc` for bash): +```sh +export KERL_CONFIGURE_OPTIONS="--without-javac --without-odbc --without-wx --without-ssh" +``` + +### Elixir/Erlang installation + +Install curl (for the mise installer): +``` +# apt install curl +``` + +Switch to the Pleroma user using `sudo -Hu pleroma /bin/bash` and follow the install steps from mise documentation and install it for the user Pleroma: [https://mise.jdx.dev/getting-started.html](https://mise.jdx.dev/getting-started.html) + +Add the following to activate mise in the Pleroma user's shell configuration (`.bashrc` for bash): +```sh +# Modify to match the mise installation location +eval "$($HOME/.local/bin/mise activate bash)" +``` + +Source new profile settings: +```sh +$ source ~/.bashrc +``` + +If installing Pleroma, follow the [Pleroma source installation instructions](./debian_based_en.md) up to the point before downloading mix dependencies for Pleroma, ignoring the instructions to download Elixir/Erlang. + +Install Erlang (replace `` with a supported release meeting Pleroma's requirements, OTP 27 is recommended): +```sh +$ mise install erlang@ +``` + +!!!info + To get a list of available versions, use `mise ls-remote `. + +Then run the following in the Pleroma source code repository: +```sh +$ mise use erlang@ +``` + +Install Elixir (replace `` with a supported release and `` with the mise-installed Erlang/OTP version, Elixir 1.18 is recommended): +```sh +$ mise install elixir@-otp- +``` + +Then run the following in the Pleroma source code directory: +```sh +$ mise use elixir@ +``` + +If installing Pleroma, continue with the source install guide up to the point of installing the systemd Pleroma service. +Don't attempt to start the Pleroma service yet. + +Create a `pleroma.sh` startup script owned by the Pleroma user in Pleroma user's home directory: +```sh +#!/bin/bash +# Modify to match the mise installation location +eval "$($HOME/.local/bin/mise activate bash)" + +exec mix phx.server +``` + +Make the startup script executable: +```sh +chmod u+x pleroma.sh +``` + +Override the Pleroma service file to point to the new start up script, run `# systemctl edit pleroma` and add the following: +```ini +[Service] +ExecStart= +ExecStart=/var/lib/pleroma/pleroma.sh +``` + +You can now start the Pleroma service. +If you are installing Pleroma, continue with creating your first user according to the install guide. + +## asdf + +To start, create the Pleroma user according to the [source install guide](./debian_based_en.md#install-pleromabe) and switch to the user using `sudo -Hu pleroma /bin/bash`. +Then follow the install and configure steps from asdf documentation and install it for the user Pleroma: [https://asdf-vm.com/guide/getting-started.html](https://asdf-vm.com/guide/getting-started.html) + +Install dependencies: +``` +# apt install build-essential libssl-dev libncurses-dev git unzip curl +``` + +Add Erlang autotools configure options to Pleroma user's shell config (`.bashrc` for bash): +```sh +export KERL_CONFIGURE_OPTIONS="--without-javac --without-odbc --without-wx --without-ssh" +``` + +Source new profile settings: +```sh +$ source ~/.bashrc +``` + +Install the Erlang plugin: +```sh +$ asdf plugin add erlang https://github.com/asdf-vm/asdf-erlang.git +``` + +Build and install Erlang (replace `` with a supported release meeting Pleroma's requirements, OTP 27 is recommended): +```sh +$ asdf install erlang +``` + +!!!info + To get a list of available versions, use `asdf list all `. + +Install the Elixir plugin: +```sh +$ asdf plugin add elixir https://github.com/asdf-vm/asdf-elixir.git +``` + +Install Elixir (replace `` with a supported release and `` with the asdf-installed Erlang/OTP version, Elixir 1.18 is recommended): +```sh +$ asdf install elixir -otp- +``` + +If installing Pleroma, follow the [Pleroma source installation instructions](./debian_based_en.md) up to the point before downloading mix dependencies for Pleroma, ignoring the instructions to download Elixir/Erlang. + +Then run the following in the Pleroma source code directory: +```sh +$ asdf set erlang +$ asdf set elixir +``` + +If installing Pleroma, continue with the source install guide up to the point of installing the systemd Pleroma service. +Don't attempt to start the Pleroma service yet. + +Create a `pleroma.sh` startup script owned by the Pleroma user in Pleroma user's home directory: +```sh +#!/bin/sh +# Modify to match your .bashrc settings used when installing asdf +export ASDF_DATA_DIR="/your/custom/data/dir" +export PATH="${ASDF_DATA_DIR:-$HOME/.asdf}/shims:$PATH" + +exec mix phx.server +``` + +Make the startup script executable: +```sh +chmod u+x pleroma.sh +``` + +Override the Pleroma service file to point to the new start up script, run `# systemctl edit pleroma` and add the following: +```ini +[Service] +ExecStart= +ExecStart=/var/lib/pleroma/pleroma.sh +``` + +You can now start the Pleroma service. +If you are installing Pleroma, continue with creating your first user according to the install guide. + +## pkgsrc + +pkgsrc is NetBSD's portable package building framework also supporting Linux. +Its use is discouraged for novice users as it is harder to maintain and keep up-to-date compared to other options in this guide. + +Before installing pkgsrc, choose which install method (tarball or CVS) you want to use. +Both update differently and have their caveats, check NetBSD documentation for more info: [https://www.netbsd.org/docs/pkgsrc/getting.html#uptodate](https://www.netbsd.org/docs/pkgsrc/getting.html#uptodate) +Using the stable branch is recommended. + +!!!warning + For CVS install, `cvs` might not be installed on your system, in which case it can be installed with `# apt install cvs`. + +pkgsrc supports both user-level and system-wide package installation, hence why most commands below don't have a `#`/`$` prepended to them. +For user-level installs, use the Pleroma user created in the [source install guide](./debian_based_en.md#install-pleromabe). +To start, follow the `quickstart -> install source packages` portion of the guide to acquire pkgsrc: [https://pkgsrc.org](https://pkgsrc.org) + +### Bootstrap + +Install build dependencies for bootstrap: +``` +# apt install build-essential +``` + +For user-level installation run: +```sh +$ cd pkgsrc/bootstrap +$ ./bootstrap --prefer-pkgsrc yes --unprivileged --make-jobs +``` + +For system-wide installation run: +``` +# cd pkgsrc/bootstrap +# ./bootstrap --prefer-pkgsrc yes --make-jobs +``` + +!!!info + For more bootstrap options, run `./bootstrap --help` + +Add pgksrc packages to `$PATH` in shell configuration (`.bashrc` for bash), default prefix is `$HOME/pkg` for user-level install and `/usr/pkg` for system-wide install: +```sh +export PATH="$PATH:/bin" +``` + +Source new profile settings: +```sh +source ~/.bashrc +``` + +!!!info + For system-wide installs, both the root user and Pleroma user need to have the `PATH` set accordingly. + +### Package installation + +Erlang: +```sh +cd pkgsrc/lang/erlang +bmake install clean +``` + +Elixir: +```sh +cd ../elixir +bmake install clean +``` + +### Pleroma startup script + +If installing Pleroma, first follow the [Pleroma source installation instructions](./debian_based_en.md) up to the point of installing the systemd Pleroma service, ignoring the instruction to install Elixir/Erlang. Don't attempt to start the service yet. + +Create a `pleroma.sh` startup script owned by the Pleroma user in Pleroma user's home directory: +```sh +#!/bin/sh +# Modify to match your .bashrc settings used when bootsrapping pkgsrc +export PATH="$PATH:/bin" + +exec mix phx.server +``` + +Make the startup script executable: +```sh +chmod u+x pleroma.sh +``` + +Override the Pleroma service file to point to the new start up script, run `# systemctl edit pleroma` and add the following: +```ini +[Service] +ExecStart= +ExecStart=/var/lib/pleroma/pleroma.sh +``` + +You can now start the Pleroma service. +If you are installing Pleroma, continue with creating your first user according to the install guide. + +## Maintenance + +Since Elixir and Erlang is no longer managed by the system package manager after following this guide, it is recommended to periodically check their respective websites/repositories for newer versions with security bug fixes. +When updating Elixir, Erlang or both, stop Pleroma before proceeding with the update. + +Pleroma will likely need to be recompiled after the update by running the following command in the Pleroma source code directory as the Pleroma user: +```sh +MIX_ENV=prod mix compile +``` + +If Pleroma fails to compile or start after upgrading Elixir/Erlang, run the following as a troubleshooting step in the Pleroma source code directory as the Pleroma user: +```sh +mix clean +mix deps.clean --all +mix deps.get +MIX_ENV=prod mix compile +``` + +Then try to start Pleroma again. If that doesn't help, join the Matrix room or IRC channel mentioned below, or make an issue on [git.pleroma.social/pleroma/pleroma](https://git.pleroma.social/pleroma/pleroma/issues/new/choose) explaining the problem. + +## Questions + +Questions about the guide or it didn’t work as it should, ask in [#pleroma:libera.chat](https://matrix.to/#/#pleroma:libera.chat) via Matrix or **#pleroma** on **libera.chat** via IRC. + + diff --git a/docs/installation/migrating_from_rebased.md b/docs/installation/migrating_from_rebased.md new file mode 100644 index 000000000..254f2884a --- /dev/null +++ b/docs/installation/migrating_from_rebased.md @@ -0,0 +1,130 @@ +# Migrating from Rebased + +Rebased and Pleroma currently have compatible database schemas, so removing +Rebased-specific database objects is optional. The cleanup described below is +destructive and can take a long time on large databases. + +This guide requires a Pleroma release that contains this document and native +webhook support. If you are reading the development version of this guide +before those changes reach `stable`, wait for the corresponding stable release. + +## Prepare and back up + +Plan a maintenance window and read the [backup guide](../administration/backup.md) +before starting. + +Stop Rebased and confirm it is stopped before making the backup. Omit the +backup guide's final restart step and leave Rebased stopped: + +```bash +sudo systemctl stop pleroma +sudo systemctl is-active pleroma +``` + +Make a verified database backup and a copy of the Rebased installation, +including its configuration, uploads, and `instance` directory. Also save the +current systemd unit if you plan to replace it. + +## Install Pleroma + +Do not use `git pull --rebase` to change an existing Rebased checkout into a +Pleroma checkout. The forks use different branches, and rebasing can replay +Rebased-specific commits over Pleroma. Move the old checkout aside and clone a +clean copy of Pleroma instead. The paths below assume the default source +installation in `/opt/pleroma`: + +```bash +sudo mv /opt/pleroma /opt/pleroma.rebased +sudo mkdir /opt/pleroma +sudo chown pleroma:pleroma /opt/pleroma +sudo -Hu pleroma git clone -b stable https://git.pleroma.social/pleroma/pleroma /opt/pleroma +``` + +Restore the instance-specific files from the old checkout. This usually +includes `config/prod.secret.exs`, `config/prod.exported_from_db.secret.exs` if +present, `uploads`, and `instance`. Diff and restore any other non-stock +configuration files, preserve their ownership, and review them before starting +Pleroma. + +Pleroma requires Elixir 1.17 or newer and Erlang/OTP 27 or newer. Install supported Erlang and Elixir +versions as described in the installation guide for your distribution before +running any Mix command. Do not reuse Rebased's `.tool-versions`. If you keep +using [`asdf`](https://asdf-vm.com/), select supported versions in the +`pleroma` user's home and use the asdf Mix shim for every command below. + +Set `MIX` to the appropriate binary, verify the selected runtime, then install +the dependencies and compile Pleroma: + +```bash +cd /opt/pleroma +MIX=/usr/bin/mix +# If using asdf instead: +# MIX=/var/lib/pleroma/.asdf/shims/mix +sudo -Hu pleroma "$MIX" --version +sudo -Hu pleroma MIX_ENV=prod "$MIX" deps.get +sudo -Hu pleroma MIX_ENV=prod "$MIX" compile +``` + +Compare your existing systemd unit with `installation/pleroma.service`. If you +replace it, review its user, paths, and environment first. Reload systemd after +making changes: + +```bash +sudo systemctl daemon-reload +``` + +Rebased does not bundle a frontend, so `instance/static` may contain an +installed frontend. Pleroma includes a bundled frontend. You can remove the +separately installed frontend after the migration, but do not remove other +instance files such as custom emojis. + +## Database migration + +Rebased's extra columns, index, and enum labels do not prevent Pleroma from +starting. Existing notifications with Rebased-only event types are not +supported by Pleroma, however, and can cause notification API errors. Run the +cleanup below, or archive and remove those notification rows separately before +starting Pleroma. Skipping the cleanup preserves them if you need to restore +Rebased. + +Pleroma supports Rebased's webhook schema. The cleanup does not roll back the +shared webhook migrations, so webhook configuration is preserved. + +To perform the optional cleanup, run the rollback while the service is stopped: + +```bash +MIX=/usr/bin/mix +# If using asdf instead: +# MIX=/var/lib/pleroma/.asdf/shims/mix +sudo -Hu pleroma MIX_ENV=prod "$MIX" ecto.rollback --migrations-path priv/repo/optional_migrations/rebased_rollbacks --all +``` + +This deletes the Rebased-only `accepts_email_list`, `location`, and +`last_move_at` user fields and the `pleroma:participation_accepted`, +`pleroma:participation_request`, `pleroma:event_reminder`, and +`pleroma:event_update` notifications. It also rewrites the notifications table +and can require substantial time, free disk space, and write-ahead log capacity +on a large instance. + +Apply all pending Pleroma migrations: + +```bash +sudo -Hu pleroma MIX_ENV=prod "$MIX" ecto.migrate +``` + +## Start and verify Pleroma + +Start Pleroma and inspect its status and logs: + +```bash +sudo systemctl start pleroma +sudo systemctl status pleroma +sudo journalctl -u pleroma -n 100 --no-pager +``` + +Verify that the instance, uploads, frontend, and webhook configuration work as +expected before deleting `/opt/pleroma.rebased` or the database backup. If the +migration fails, keep the service stopped and restore both the database backup +and the previous checkout before restarting Rebased. If you changed the +systemd unit or runtime setup, restore those too and run +`systemctl daemon-reload` before restarting the service. diff --git a/docs/installation/migrating_from_source_otp_en.md b/docs/installation/migrating_from_source_otp_en.md index 798862566..450d33804 100644 --- a/docs/installation/migrating_from_source_otp_en.md +++ b/docs/installation/migrating_from_source_otp_en.md @@ -76,35 +76,51 @@ $EDITOR /etc/pleroma/config.exs ``` ## Installing the release Before proceeding, get the flavour from [Detecting flavour](otp_en.md#detecting-flavour) section in OTP installation guide. + + +Delete all files in pleroma user's directory: ```sh -# Delete all files in pleroma user's directory rm -r ~pleroma/* +``` -# Set the flavour environment variable to the string you got in Detecting flavour section. -# For example if the flavour is `amd64-musl` the command will be +Set the flavour environment variable to the string you got in Detecting flavour section. +For example if the flavour is `amd64-musl` the command will be: + +```sh export FLAVOUR="amd64-musl" +``` -# Clone the release build into a temporary directory and unpack it -# Replace `stable` with `unstable` if you want to run the unstable branch -sudo -Hu pleroma " -curl 'https://git.pleroma.social/api/v4/projects/2/jobs/artifacts/stable/download?job=$FLAVOUR' -o /tmp/pleroma.zip -unzip /tmp/pleroma.zip -d /tmp/ -" +Clone the release build into a temporary directory and unpack it. +Replace `stable` with `develop`, if you want to run the develop branch. -# Move the release to the home directory and delete temporary files -sudo -Hu pleroma " -mv /tmp/release/* ~pleroma/ -rmdir /tmp/release -rm /tmp/pleroma.zip -" +```sh +sudo -Hu pleroma curl "https://git.pleroma.social/api/packages/pleroma/generic/pleroma-otp-stable-$FLAVOUR/latest/pleroma.zip" -o /tmp/pleroma.zip +sudo -Hu pleroma unzip /tmp/pleroma.zip -d /tmp/ +``` -# Start the instance to verify that everything is working as expected +Move the release to the home directory and delete temporary files: + +```sh +sudo -Hu pleroma mv /tmp/release/* ~pleroma/ +sudo -Hu pleroma rmdir /tmp/release +sudo -Hu pleroma rm /tmp/pleroma.zip +``` + +Start the instance to verify that everything is working as expected: + +```sh sudo -Hu pleroma "./bin/pleroma daemon" +``` -# Wait for about 20 seconds and query the instance endpoint, if it shows your uri, name and email correctly, you are configured correctly +Wait for about 20 seconds and query the instance endpoint, if it shows your uri, name and email correctly, you are configured correctly: + +```sh sleep 20 && curl http://localhost:4000/api/v1/instance +``` -# Stop the instance +Stop the instance: + +```sh sudo -Hu pleroma "./bin/pleroma stop" ``` diff --git a/docs/installation/openbsd_en.md b/docs/installation/openbsd_en.md index 1de016cdd..82d137e96 100644 --- a/docs/installation/openbsd_en.md +++ b/docs/installation/openbsd_en.md @@ -67,13 +67,19 @@ Pleroma will be run by a dedicated \_pleroma user. Before creating it, insert th ``` pleroma:\ - :datasize=1536M:\ + :datasize=4096M:\ :openfiles-max=4096:\ :openfiles-cur=1024:\ :setenv=LC_ALL=en_US.UTF-8,VIX_COMPILATION_MODE=PLATFORM_PROVIDED_LIBVIPS,MIX_ENV=prod:\ :tc=daemon: ``` +Then run: + +``` +# cap_mkdb /etc/login.conf +``` + This creates a "pleroma" login class and sets higher values than default for datasize and openfiles (see [login.conf(5)](https://man.openbsd.org/login.conf)), this is required to avoid having Pleroma crash some time after starting. Create the \_pleroma user, assign it the pleroma login class and create its home directory (/home/\_pleroma/): diff --git a/docs/installation/otp_en.md b/docs/installation/otp_en.md index 123dccf3a..0f296a9f6 100644 --- a/docs/installation/otp_en.md +++ b/docs/installation/otp_en.md @@ -32,6 +32,7 @@ Other than things bundled in the OTP release Pleroma depends on: * curl (to download the release build) * unzip (needed to unpack release builds) * ncurses (ERTS won't run without it) +* libvips (for image processing) * PostgreSQL (also utilizes extensions in postgresql-contrib) * nginx (could be swapped with another reverse proxy but this guide covers only it) * certbot (for Let's Encrypt certificates, could be swapped with another ACME client, but this guide covers only it) @@ -46,7 +47,13 @@ Other than things bundled in the OTP release Pleroma depends on: === "Debian/Ubuntu" ``` - apt install curl unzip libncurses5 postgresql postgresql-contrib nginx certbot libmagic-dev + apt install curl unzip libncurses6 libvips42t64 postgresql postgresql-contrib nginx certbot libmagic-dev + ``` + + On Debian 12 and Ubuntu 22.04, replace `libvips42t64` with `libvips42`: + + ``` + apt install curl unzip libncurses6 libvips42 postgresql postgresql-contrib nginx certbot libmagic-dev ``` ### Installing optional packages @@ -88,9 +95,23 @@ RUM indexes are an alternative indexing scheme that is not included in PostgreSQ ``` === "Debian/Ubuntu" + Install the package matching your distribution's PostgreSQL version: + ``` - # Available only on Buster/19.04 - apt install postgresql-11-rum + # Debian 13 + apt install postgresql-17-rum + + # Debian 12 + apt install postgresql-15-rum + + # Ubuntu 26.04 + apt install postgresql-18-rum + + # Ubuntu 24.04 + apt install postgresql-16-rum + + # Ubuntu 22.04 + apt install postgresql-14-rum ``` #### (Optional) Performance configuration @@ -109,60 +130,90 @@ Restart PostgreSQL to apply configuration changes: ``` ### Installing Pleroma +Create a Pleroma user: + ```sh -# Create a Pleroma user adduser --system --shell /bin/false --home /opt/pleroma pleroma +``` -# Set the flavour environment variable to the string you got in Detecting flavour section. -# For example if the flavour is `amd64-musl` the command will be +Set the flavour environment variable to the string you got in Detecting flavour section. For example if the flavour is `amd64-musl` the command will be: + +```sh export FLAVOUR="amd64-musl" +``` -# Clone the release build into a temporary directory and unpack it -sudo -Hu pleroma " -curl 'https://git.pleroma.social/api/v4/projects/2/jobs/artifacts/stable/download?job=$FLAVOUR' -o /tmp/pleroma.zip -unzip /tmp/pleroma.zip -d /tmp/ -" +Clone the release build into a temporary directory and unpack it: -# Move the release to the home directory and delete temporary files -sudo -Hu pleroma " -mv /tmp/release/* /opt/pleroma -rmdir /tmp/release -rm /tmp/pleroma.zip -" -# Create uploads directory and set proper permissions (skip if planning to use a remote uploader) -# Note: It does not have to be `/var/lib/pleroma/uploads`, the config generator will ask about the upload directory later +```sh +sudo -Hu pleroma curl "https://git.pleroma.social/api/packages/pleroma/generic/pleroma-otp-stable-$FLAVOUR/latest/pleroma.zip" -o /tmp/pleroma.zip +sudo -Hu pleroma unzip /tmp/pleroma.zip -d /tmp/ +``` +Move the release to the home directory and delete temporary files: + +```sh +sudo -Hu pleroma mv /tmp/release/* /opt/pleroma +sudo -Hu pleroma rmdir /tmp/release +sudo -Hu pleroma rm /tmp/pleroma.zip +``` + +Create uploads directory and set proper permissions (skip if planning to use a remote uploader). +Note: It does not have to be `/var/lib/pleroma/uploads`, the config generator will ask about the upload directory later. + +```sh mkdir -p /var/lib/pleroma/uploads chown -R pleroma /var/lib/pleroma +``` -# Create custom public files directory (custom emojis, frontend bundle overrides, robots.txt, etc.) -# Note: It does not have to be `/var/lib/pleroma/static`, the config generator will ask about the custom public files directory later +Create custom public files directory (custom emojis, frontend bundle overrides, robots.txt, etc.) +Note: It does not have to be `/var/lib/pleroma/static`, the config generator will ask about the custom public files directory later. + +```sh mkdir -p /var/lib/pleroma/static chown -R pleroma /var/lib/pleroma +``` -# Create a config directory +Create a config directory: + +```sh mkdir -p /etc/pleroma chown -R pleroma /etc/pleroma +``` -# Run the config generator +Run the config generator: + +```sh sudo -Hu pleroma "./bin/pleroma_ctl instance gen --output /etc/pleroma/config.exs --output-psql /tmp/setup_db.psql" +``` -# Create the postgres database +Create the postgres database and schema: + +```sh sudo -u postgres -s $SHELL -lc "psql -f /tmp/setup_db.psql" - -# Create the database schema sudo -Hu pleroma "./bin/pleroma_ctl migrate" +``` -# If you have installed RUM indexes uncommend and run -# sudo -Hu pleroma "./bin/pleroma_ctl migrate --migrations-path priv/repo/optional_migrations/rum_indexing/" +(Optional) If you have installed RUM indexes: -# Start the instance to verify that everything is working as expected +```sh +sudo -Hu pleroma "./bin/pleroma_ctl migrate --migrations-path priv/repo/optional_migrations/rum_indexing/" +``` + +Start the instance to verify that everything is working as expected: + +```sh sudo -Hu pleroma "./bin/pleroma daemon" +``` -# Wait for about 20 seconds and query the instance endpoint, if it shows your uri, name and email correctly, you are configured correctly +Wait for about 20 seconds and query the instance endpoint, if it shows your uri, name and email correctly, you are configured correctly: + +```sh sleep 20 && curl http://localhost:4000/api/v1/instance +``` -# Stop the instance +Stop the instance: + +```sh sudo -Hu pleroma "./bin/pleroma stop" ``` diff --git a/installation/init.d/pleroma b/installation/init.d/pleroma index cb6635a0b..3b584bd45 100755 --- a/installation/init.d/pleroma +++ b/installation/init.d/pleroma @@ -37,6 +37,13 @@ depend() { need nginx postgresql } +start_pre() { + ebegin "running migrations if any" + su -s /bin/sh -c "cd \"${directory}\" && exec /usr/bin/mix ecto.migrate" \ + "${command_user%%:*}" + eend "$?" +} + healthcheck() { # put pleroma_health=YES in /etc/conf.d/pleroma if you want healthchecking # and make sure you have curl installed diff --git a/lib/mix/pleroma.ex b/lib/mix/pleroma.ex index e7869919c..aebad6984 100644 --- a/lib/mix/pleroma.ex +++ b/lib/mix/pleroma.ex @@ -62,7 +62,8 @@ defmodule Mix.Pleroma do {Majic.Pool, [name: Pleroma.MajicPool, pool_size: Pleroma.Config.get([:majic_pool, :size], 2)]} ] ++ - http_children(adapter) + http_children(adapter) ++ + search_children() cachex_children = Enum.map(@cachex_children, &Pleroma.Application.build_cachex(&1, [])) @@ -141,4 +142,14 @@ defmodule Mix.Pleroma do end defp http_children(_), do: [] + + defp search_children do + case Pleroma.Config.get([Pleroma.Search, :module]) do + Pleroma.Search.ParadeDB -> + [Pleroma.Search.ParadeDB.Repo] + + _ -> + [] + end + end end diff --git a/lib/mix/tasks/pleroma/config.ex b/lib/mix/tasks/pleroma/config.ex index c5f2e9b0a..81c41d967 100644 --- a/lib/mix/tasks/pleroma/config.ex +++ b/lib/mix/tasks/pleroma/config.ex @@ -252,40 +252,43 @@ defmodule Mix.Tasks.Pleroma.Config do if whitelisted_configs in [nil, false] do shell_error("No unwanted settings in ConfigDB. No changes made.") else - whitelisted_groups = - whitelisted_configs - |> Enum.filter(fn - {_group} -> true - _ -> false - end) - |> Enum.map(fn {group} -> group end) - - whitelisted_keys = - whitelisted_configs - |> Enum.filter(fn - {_group, _key} -> true - _ -> false - end) + {whitelisted_groups, whitelisted_keys} = get_filters(whitelisted_configs) filtered = from(c in ConfigDB) |> Repo.all() |> Enum.filter(¬_whitelisted?(&1, whitelisted_groups, whitelisted_keys)) - if not Enum.empty?(filtered) do - shell_info("The following settings will be removed from ConfigDB:\n") - Enum.each(filtered, &dump(&1)) + do_config_filter(filtered, force) + end + end - if force or shell_prompt("Are you sure you want to continue?", "n") in ~w(Yn Y y) do - filtered_ids = Enum.map(filtered, fn %{id: id} -> id end) + # Removes blacklisted configuration sections + def run(["filter_blacklisted" | rest]) do + {options, [], []} = + OptionParser.parse( + rest, + strict: [force: :boolean], + aliases: [f: :force] + ) - Repo.delete_all(from(c in ConfigDB, where: c.id in ^filtered_ids)) - else - shell_error("No changes made.") - end - else - shell_error("No unwanted settings in ConfigDB. No changes made.") - end + force = Keyword.get(options, :force, false) + + start_pleroma() + + blacklisted_configs = Pleroma.Config.get(:database_config_blacklist) + + if blacklisted_configs in [nil, false] do + shell_error("No unwanted settings in ConfigDB. No changes made.") + else + {blacklisted_groups, blacklisted_keys} = get_filters(blacklisted_configs) + + filtered = + from(c in ConfigDB) + |> Repo.all() + |> Enum.filter(&blacklisted?(&1, blacklisted_groups, blacklisted_keys)) + + do_config_filter(filtered, force) end end @@ -332,9 +335,12 @@ defmodule Mix.Tasks.Pleroma.Config do group |> Pleroma.Config.Loader.filter_group(settings) |> Enum.each(fn {key, value} -> - {:ok, _} = ConfigDB.update_or_create(%{group: group, key: key, value: value}) - - shell_info("Settings for key #{key} migrated.") + if ConfigDB.static?(group, key) do + shell_info("Settings for key #{key} must remain in static configuration; skipping.") + else + {:ok, _} = ConfigDB.update_or_create(%{group: group, key: key, value: value}) + shell_info("Settings for key #{key} migrated.") + end end) shell_info("Settings for group #{inspect(group)} migrated.") @@ -490,8 +496,49 @@ defmodule Mix.Tasks.Pleroma.Config do Ecto.Adapters.SQL.query!(Repo, "ALTER SEQUENCE config_id_seq RESTART;") end + defp get_filters(config) do + groups = + config + |> Enum.filter(fn + {_group} -> true + _ -> false + end) + |> Enum.map(fn {group} -> group end) + + keys = + config + |> Enum.filter(fn + {_group, _key} -> true + _ -> false + end) + + {groups, keys} + end + defp not_whitelisted?(%{group: group, key: key}, whitelisted_groups, whitelisted_keys) do not Enum.member?(whitelisted_groups, group) and not Enum.member?(whitelisted_keys, {group, key}) end + + defp blacklisted?(%{group: group, key: key}, blacklisted_groups, blacklisted_keys) do + Enum.member?(blacklisted_groups, group) or + Enum.member?(blacklisted_keys, {group, key}) + end + + defp do_config_filter(filtered, _) when length(filtered) == 0 do + shell_error("No unwanted settings in ConfigDB. No changes made.") + end + + defp do_config_filter(filtered, force) do + shell_info("The following settings will be removed from ConfigDB:\n") + Enum.each(filtered, &dump(&1)) + + if force or shell_prompt("Are you sure you want to continue?", "n") in ~w(Yn Y y) do + filtered_ids = Enum.map(filtered, fn %{id: id} -> id end) + + Repo.delete_all(from(c in ConfigDB, where: c.id in ^filtered_ids)) + else + shell_error("No changes made.") + end + end end diff --git a/lib/mix/tasks/pleroma/search/indexer.ex b/lib/mix/tasks/pleroma/search/indexer.ex index 2a52472f9..62cbf2e50 100644 --- a/lib/mix/tasks/pleroma/search/indexer.ex +++ b/lib/mix/tasks/pleroma/search/indexer.ex @@ -4,9 +4,6 @@ defmodule Mix.Tasks.Pleroma.Search.Indexer do import Mix.Pleroma - import Ecto.Query - - alias Pleroma.Workers.SearchIndexingWorker def run(["create_index"]) do start_pleroma() @@ -35,49 +32,30 @@ defmodule Mix.Tasks.Pleroma.Search.Indexer do strict: [ chunk: :integer, limit: :integer, - step: :integer + step: :integer, + before: :string ] ) start_pleroma() - chunk_size = Keyword.get(options, :chunk, 100) - limit = Keyword.get(options, :limit, 100_000) - per_step = Keyword.get(options, :step, 1000) + result = + Pleroma.Search.Backfill.run( + options ++ + [ + on_page: fn page -> + IO.puts( + "Queued #{page.count} activities (#{page.enqueued} total), " <> + "checkpoint: #{page.next_cursor}" + ) + end + ] + ) - chunks = max(div(limit, per_step), 1) + IO.puts("Queued #{result.enqueued} activities") - 1..chunks - |> Enum.each(fn step -> - q = - from(a in Pleroma.Activity, - limit: ^per_step, - offset: ^per_step * (^step - 1), - select: [:id], - order_by: [desc: :id] - ) - - {:ok, ids} = - Pleroma.Repo.transaction(fn -> - Pleroma.Repo.stream(q, timeout: :infinity) - |> Enum.map(fn a -> - a.id - end) - end) - - IO.puts("Got #{length(ids)} activities, adding to indexer") - - ids - |> Enum.chunk_every(chunk_size) - |> Enum.each(fn chunk -> - IO.puts("Adding #{length(chunk)} activities to indexing queue") - - chunk - |> Enum.map(fn id -> - SearchIndexingWorker.new(%{"op" => "add_to_index", "activity" => id}) - end) - |> Oban.insert_all() - end) - end) + unless result.exhausted do + IO.puts("Resume with --before #{result.next_cursor}") + end end end diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 8e1c5de0d..adc054f3b 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -43,6 +43,7 @@ defmodule Pleroma.Application do # every time the application is restarted, so we disable module # conflicts at runtime Code.compiler_options(ignore_module_conflict: true) + configure_logger() Pleroma.Telemetry.Logger.attach() Config.Holder.save_default() Pleroma.HTML.compile_scrubbers() @@ -65,6 +66,7 @@ defmodule Pleroma.Application do if match?({Tesla.Adapter.Finch, _}, adapter) do Logger.info("Starting Finch") + use_castore_for_otp_cacerts() Finch.start_link(name: MyFinch) end @@ -80,12 +82,15 @@ defmodule Pleroma.Application do [ Pleroma.PromEx, Pleroma.LDAP, - Pleroma.Repo, - Config.TransferTask, - Pleroma.Emoji, - Pleroma.Web.Plugs.RateLimiter.Supervisor, - {Task.Supervisor, name: Pleroma.TaskSupervisor} + Pleroma.Repo ] ++ + search_children() ++ + [ + Config.TransferTask, + Pleroma.Emoji, + Pleroma.Web.Plugs.RateLimiter.Supervisor, + {Task.Supervisor, name: Pleroma.TaskSupervisor} + ] ++ cachex_children() ++ http_children(adapter) ++ [ @@ -113,6 +118,44 @@ defmodule Pleroma.Application do Supervisor.start_link(children, opts) end + def configure_logger do + Config.get([:logger, :backends], []) + |> Enum.each(fn backend -> + case backend_to_logger(backend) do + {:ok, logger} -> + add_logger(logger) + + {:error, :console_backend} -> + Logger.warning(":console is no longer considered a backend and is enabled by default") + end + end) + end + + defp add_logger(backend) do + case LoggerBackends.add(backend) do + {:ok, _} -> + Logger.debug("Successfully added logger backend: #{inspect(backend)}") + + {:error, :already_present} -> + :ok + + {:error, reason} -> + Logger.error("Failed to add logger backend #{inspect(backend)}: #{inspect(reason)}") + end + end + + defp backend_to_logger(:console), do: {:error, :console_backend} + + defp backend_to_logger({:ex_syslogger = backend, name}) do + Logger.warning( + "Configuration {:#{backend}, :#{name}} is incorrect. Use {ExSyslogger, :#{name}} instead!" + ) + + {:ok, {ExSyslogger, name}} + end + + defp backend_to_logger(backend), do: {:ok, backend} + def load_custom_modules do dir = Config.get([:modules, :runtime_dir]) @@ -159,6 +202,16 @@ defmodule Pleroma.Application do ] end + defp search_children do + case Config.get([Pleroma.Search, :module]) do + Pleroma.Search.ParadeDB -> + [Pleroma.Search.ParadeDB.Repo] + + _ -> + [] + end + end + defp emoji_packs_expiration, do: expiration(default: :timer.seconds(5 * 60), interval: :timer.seconds(60)) @@ -252,6 +305,15 @@ defmodule Pleroma.Application do end end + # Mint trusts the OTP CA store, which comes from the OS. Make it the CAStore + # bundle that the other adapters trust. OTP 24 has no such store and Mint + # uses CAStore there anyway. + defp use_castore_for_otp_cacerts do + if Code.ensure_loaded?(:public_key) and function_exported?(:public_key, :cacerts_load, 1) do + :ok = :public_key.cacerts_load(CAStore.file_path()) + end + end + defp http_children_hackney do pools = [:federation, :media] @@ -278,7 +340,8 @@ defmodule Pleroma.Application do config = Config.get(ConcurrentLimiter, []) [ - Pleroma.Search + Pleroma.Search, + Pleroma.Webhook.Notify ] |> Enum.each(fn module -> mod_config = Keyword.get(config, module, []) diff --git a/lib/pleroma/chat.ex b/lib/pleroma/chat.ex index 926176bfa..6ac1b24b8 100644 --- a/lib/pleroma/chat.ex +++ b/lib/pleroma/chat.ex @@ -108,4 +108,6 @@ defmodule Pleroma.Chat do |> cast(%{pinned: false}, [:pinned]) |> Repo.update() end + + def enabled?, do: Pleroma.Config.get([__MODULE__, :enabled], true) end diff --git a/lib/pleroma/config/deprecation_warnings.ex b/lib/pleroma/config/deprecation_warnings.ex index 58d164dc7..934a4a02e 100644 --- a/lib/pleroma/config/deprecation_warnings.ex +++ b/lib/pleroma/config/deprecation_warnings.ex @@ -20,6 +20,8 @@ defmodule Pleroma.Config.DeprecationWarnings do "\n* `config :pleroma, :instance, mrf_transparency_exclusions` is now `config :pleroma, :mrf, transparency_exclusions`"} ] + @logger_formatter_config_knobs [:colors, :format, :metadata, :truncate, :utc_log] + def check_exiftool_filter do filters = Config.get([Pleroma.Upload]) |> Keyword.get(:filters, []) @@ -218,7 +220,8 @@ defmodule Pleroma.Config.DeprecationWarnings do check_quarantined_instances_tuples(), check_transparency_exclusions_tuples(), check_simple_policy_tuples(), - check_exiftool_filter() + check_exiftool_filter(), + check_deprecated_logger_config() ] |> Enum.reduce(:ok, fn :ok, :ok -> :ok @@ -415,4 +418,72 @@ defmodule Pleroma.Config.DeprecationWarnings do :ok end end + + defp merge_deprecated_logger_config(config) do + formatter_config = Enum.filter(config, fn {k, _} -> k in @logger_formatter_config_knobs end) + formatter = Logger.default_formatter(formatter_config) + log_level = Keyword.get(config, :level, nil) + + Logger.debug(""" + Reconfiguring console Logger with deprecated configuration syntax. + Handler configuration: + log_level: #{inspect(log_level)} + + Formatter: + #{inspect(formatter)} + """) + + if Config.get(:env) != :test do + if log_level, do: :logger.update_handler_config(:default, :level, log_level) + :logger.set_handler_config(:default, :formatter, formatter) + end + end + + @spec check_deprecated_logger_config() :: :ok | :error + def check_deprecated_logger_config do + backends_config = Application.get_env(:logger, :backends) + console_config = Application.get_env(:logger, :console) + + # NOTE: No need to merge the old backends config since it still works. + # And new configuration will just add new Logger backends. + backend = + if backends_config do + Logger.warning(""" + !!!DEPRECATION WARNING!!! + Your configuration is using deprecated syntax for configuring backends of Elixir's logger. + `config :logger, backends: [...]` is deprecated syntax due to changes in Elixir. + Please update your configuration at your earliest convenience to use: + `config :pleroma, :logger, backends: [...]` + + Note: `:console` is no longer considered a backend and is used by default, you can disable it using: + `config :logger, :default_handler: false` + """) + + true + else + false + end + + console = + if console_config do + Logger.warning(""" + !!!DEPRECATION WARNING!!! + Your configuration is using deprecated syntax for configuring logging to console. + `config :logger, :console` is deprecated syntax due to changes in Elixir. + Please update your configuration at your earliest convenience to use + `config :logger, default_handler` and `config :logger, :default_formatter`. + + Note: `:default_handler` is used only for the `level` setting. All other configurations go under + `:default_formatter`. For more info visit: https://hexdocs.pm/logger/Logger.html#module-backends-and-backwards-compatibility + """) + + merge_deprecated_logger_config(console_config) + + true + else + false + end + + if backend or console, do: :error, else: :ok + end end diff --git a/lib/pleroma/config/transfer_task.ex b/lib/pleroma/config/transfer_task.ex index 140dd7711..48cb158ca 100644 --- a/lib/pleroma/config/transfer_task.ex +++ b/lib/pleroma/config/transfer_task.ex @@ -47,11 +47,12 @@ defmodule Pleroma.Config.TransferTask do settings = (Repo.all(ConfigDB) ++ deleted_settings) + |> Enum.reject(&ConfigDB.static?/1) |> Enum.map(&merge_with_default/1) started_applications = Application.started_applications() - reject = [nil, :postgrex] + reject = [nil, :postgrex, :logger] reject = if restart_pleroma? do diff --git a/lib/pleroma/config_db.ex b/lib/pleroma/config_db.ex index 2c3df773b..9aa52f7d6 100644 --- a/lib/pleroma/config_db.ex +++ b/lib/pleroma/config_db.ex @@ -23,6 +23,12 @@ defmodule Pleroma.ConfigDB do {:pleroma, :mrf_keyword, :replace} ] + @static_config [ + {:pleroma, Pleroma.Search}, + {:pleroma, Pleroma.Search.ParadeDB}, + {:pleroma, Pleroma.Search.ParadeDB.Repo} + ] + schema "config" do field(:key, Pleroma.EctoType.Config.Atom) field(:group, Pleroma.EctoType.Config.Atom) @@ -32,16 +38,32 @@ defmodule Pleroma.ConfigDB do timestamps() end - @spec get_all_as_keyword() :: keyword() - def get_all_as_keyword do + @spec get_all_as_keyword(keyword()) :: keyword() + def get_all_as_keyword(options \\ []) do ConfigDB |> select([c], {c.group, c.key, c.value}) |> Repo.all() + |> maybe_reject_static(options) |> Enum.reduce([], fn {group, key, value}, acc -> Keyword.update(acc, group, [{key, value}], &Keyword.merge(&1, [{key, value}])) end) end + def static?(%{group: group, key: key}), do: static?(group, key) + def static?(_config), do: false + + def static?(group, key) do + Enum.any?(@static_config, fn {static_group, static_key} -> + group in [static_group, inspect(static_group)] and key in [static_key, inspect(static_key)] + end) + end + + defp maybe_reject_static(configs, exclude_static: true) do + Enum.reject(configs, fn {group, key, _value} -> static?(group, key) end) + end + + defp maybe_reject_static(configs, _options), do: configs + @spec get_all_by_group(atom() | String.t()) :: [t()] def get_all_by_group(group) do from(c in ConfigDB, where: c.group == ^group) |> Repo.all() diff --git a/lib/pleroma/docs/generator.ex b/lib/pleroma/docs/generator.ex index 93b19484f..70413837a 100644 --- a/lib/pleroma/docs/generator.ex +++ b/lib/pleroma/docs/generator.ex @@ -12,13 +12,27 @@ defmodule Pleroma.Docs.Generator do @spec list_behaviour_implementations(behaviour :: module()) :: [module()] def list_behaviour_implementations(behaviour) do - :code.all_loaded() - |> Enum.filter(fn {module, _} -> + # :code.all_loaded() no longer works on Elixir 1.19 and newer since modules are lazily loaded. + # We can get a complete list of modules by interating over Application.loaded_applications/0, + # but that includes every module which would get loaded below, which is very slow. + # Hardcode the Pleroma application instead, which is good enough for now. + application_modules = + case :application.get_key(:pleroma, :modules) do + {:ok, modules} -> modules + :undefined -> [] + end + + # At compile time or in tests some modules will be outside of the Pleroma application. + loaded_modules = Enum.map(:code.all_loaded(), fn {module, _} -> module end) + + (application_modules ++ loaded_modules) + |> Enum.uniq() + |> Enum.filter(fn module -> + Code.ensure_loaded(module) + # This shouldn't be needed as all modules are expected to have module_info/1, # but in test environments some transient modules `:elixir_compiler_XX` # are loaded for some reason (where XX is a random integer). - Code.ensure_loaded(module) - if function_exported?(module, :module_info, 1) do module.module_info(:attributes) |> Keyword.get_values(:behaviour) @@ -26,7 +40,6 @@ defmodule Pleroma.Docs.Generator do |> Enum.member?(behaviour) end end) - |> Enum.map(fn {module, _} -> module end) end @doc """ diff --git a/lib/pleroma/emails/mailer.ex b/lib/pleroma/emails/mailer.ex index 2a80f8547..e6abf4456 100644 --- a/lib/pleroma/emails/mailer.ex +++ b/lib/pleroma/emails/mailer.ex @@ -36,10 +36,11 @@ defmodule Pleroma.Emails.Mailer do def deliver(email, config \\ []) def deliver(email, config) do - # temporary hackney fix until hackney max_connections bug is fixed - # https://git.pleroma.social/pleroma/pleroma/-/issues/2101 + # Swoosh HTTP adapters use Hackney, which otherwise trusts its certifi bundle email = - Swoosh.Email.put_private(email, :hackney_options, ssl_options: [versions: [:"tlsv1.2"]]) + Swoosh.Email.put_private(email, :hackney_options, + ssl_options: Pleroma.HTTP.TLS.ssl_options() + ) case enabled?() do true -> Swoosh.Mailer.deliver(email, parse_config(config)) @@ -67,6 +68,21 @@ defmodule Pleroma.Emails.Mailer do end defp parse_config(config) do - Swoosh.Mailer.parse_config(@otp_app, __MODULE__, @mailer_config, config) + @otp_app + |> Swoosh.Mailer.parse_config(__MODULE__, @mailer_config, config) + |> put_mua_cacertfile() + end + + @doc false + # Mua otherwise trusts the OS CA store. Keep a CA that the admin set. + def put_mua_cacertfile(config) do + ssl = Keyword.get(config, :ssl, []) + + if config[:adapter] == Swoosh.Adapters.Mua and + not Keyword.has_key?(ssl, :cacertfile) and not Keyword.has_key?(ssl, :cacerts) do + Keyword.put(config, :ssl, [{:cacertfile, CAStore.file_path()} | ssl]) + else + config + end end end diff --git a/lib/pleroma/following_relationship.ex b/lib/pleroma/following_relationship.ex index e27f78d33..b803c07b4 100644 --- a/lib/pleroma/following_relationship.ex +++ b/lib/pleroma/following_relationship.ex @@ -258,10 +258,16 @@ defmodule Pleroma.FollowingRelationship do [_, user_actor: user_actor], fragment( # "(actor's domain NOT in domain_blocks) OR (actor IS in followed AP IDs)" + # + # ARRAY(SELECT ...) makes the followed-ids lookup an InitPlan that is + # evaluated and costed once. As a plain subselect, the planner charges + # its startup cost on every rescan of the actor lookup, which inflates + # the cost of the index plan enough for it to lose to a plan that + # materialises every active user for every candidate row. """ NOT (substring(? from '.*://([^/]*)') = ANY(?)) OR - ? = ANY(SELECT ap_id FROM users AS u INNER JOIN following_relationships AS fr - ON u.id = fr.following_id WHERE fr.follower_id = ? AND fr.state = ?) + ? = ANY(ARRAY(SELECT ap_id FROM users AS u INNER JOIN following_relationships AS fr + ON u.id = fr.following_id WHERE fr.follower_id = ? AND fr.state = ?)) """, user_actor.ap_id, ^user.domain_blocks, diff --git a/lib/pleroma/formatter.ex b/lib/pleroma/formatter.ex index 4bf2f6b95..59df4d20a 100644 --- a/lib/pleroma/formatter.ex +++ b/lib/pleroma/formatter.ex @@ -3,12 +3,15 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Formatter do + require Logger + alias Pleroma.HTML alias Pleroma.User @safe_mention_regex ~r/^(\s*(?(@.+?\s+){1,})+)(?.*)/s @link_regex ~r"((?:http(s)?:\/\/)?[\w.-]+(?:\.[\w\.-]+)+[\w\-\._~%:/?#[\]@!\$&'\(\)\*\+,;=.]+)|[0-9a-z+\-\.]+:[0-9a-z$-_.+!*'(),]+"ui @markdown_characters_regex ~r/(`|\*|_|{|}|[|]|\(|\)|#|\+|-|\.|!)/ + @markdown_void_elements ~w[area base br col command embed hr img input keygen link meta param source track wbr] defp linkify_opts do Pleroma.Config.get(Pleroma.Formatter) ++ @@ -124,14 +127,157 @@ defmodule Pleroma.Formatter do end def markdown_to_html(text) do - Earmark.as_html!(text, %Earmark.Options{compact_output: true, smartypants: false}) + markdown_to_html(text, %{}) end def markdown_to_html(text, opts) do - Earmark.as_html!( - text, - %Earmark.Options{compact_output: true, smartypants: false} |> Map.merge(opts) - ) + breaks = Map.get(opts, :breaks, false) + + {_status, ast, _messages} = EarmarkParser.as_ast(text, breaks: breaks, gfm: true) + + ast + |> maybe_strip_email_autolinks(text, Map.get(opts, :strip_email_autolinks, false)) + |> render_markdown_nodes(false) + |> IO.iodata_to_binary() + rescue + exception -> + Logger.warning("Markdown rendering failed in #{inspect(exception.__struct__)}") + markdown_fallback(text, Map.get(opts, :breaks, false)) + end + + defp markdown_fallback(text, breaks) do + text = text |> escape_html() |> IO.iodata_to_binary() |> maybe_render_markdown_breaks(breaks) + "

" <> text <> "

" + end + + defp maybe_render_markdown_breaks(text, true), do: String.replace(text, ~r/\r?\n/, "
") + defp maybe_render_markdown_breaks(text, false), do: text + + defp maybe_strip_email_autolinks(ast, _text, false), do: ast + + defp maybe_strip_email_autolinks(ast, text, true) do + emails = + ~r/<([^ >]+@[^ >]+)>/ + |> Regex.scan(text, capture: :all_but_first) + |> Enum.map(fn [email] -> String.replace_prefix(email, "mailto:", "") end) + |> MapSet.new() + + strip_email_autolinks(ast, emails) + end + + defp strip_email_autolinks(nodes, emails) when is_list(nodes) do + Enum.map(nodes, &strip_email_autolinks(&1, emails)) + end + + defp strip_email_autolinks({"a", attrs, [email], _meta} = node, emails) + when is_binary(email) do + if List.keyfind(attrs, "href", 0) == {"href", "mailto:" <> email} and + MapSet.member?(emails, email) do + email + else + node + end + end + + defp strip_email_autolinks({tag, attrs, children, meta}, emails) do + {tag, attrs, strip_email_autolinks(children, emails), meta} + end + + defp strip_email_autolinks(text, _emails) when is_binary(text), do: text + + defp render_markdown_nodes(nodes, verbatim) when is_list(nodes) do + Enum.map(nodes, &render_markdown_node(&1, verbatim)) + end + + defp render_markdown_node(text, true) when is_binary(text), + do: strip_dangerous_markdown_html(text) + + defp render_markdown_node(text, false) when is_binary(text) do + text + |> strip_dangerous_markdown_html() + |> escape_markdown_html() + end + + defp render_markdown_node({:comment, _attrs, _children, _meta}, _verbatim), do: [] + + defp render_markdown_node({tag, attrs, children, meta}, verbatim) do + normalized_tag = String.downcase(tag) + + if normalized_tag in ["script", "style"] do + [] + else + render_markdown_element(tag, normalized_tag, attrs, children, meta, verbatim) + end + end + + defp render_markdown_element("code", _normalized_tag, attrs, children, _meta, _verbatim) do + ["", render_markdown_code(children), ""] + end + + defp render_markdown_element(tag, "hr", attrs, _children, _meta, _verbatim) do + attrs = Enum.reject(attrs, &match?({"class", class} when class in ~w[thin medium thick], &1)) + ["<", tag, render_markdown_attrs(attrs), "/>"] + end + + defp render_markdown_element(tag, normalized_tag, attrs, _children, _meta, _verbatim) + when normalized_tag in @markdown_void_elements do + ["<", tag, render_markdown_attrs(attrs), ">"] + end + + defp render_markdown_element(tag, _normalized_tag, attrs, children, meta, verbatim) do + children_verbatim = verbatim or Map.get(meta, :verbatim, false) + + [ + "<", + tag, + render_markdown_attrs(attrs), + ">", + render_markdown_nodes(children, children_verbatim), + "" + ] + end + + defp render_markdown_attrs(attrs) do + Enum.map(attrs, fn {name, value} -> + value = value |> to_string() |> HtmlEntities.decode() |> sanitize_markdown_url(name) + [" ", to_string(name), "=\"", escape_html(value), "\""] + end) + end + + defp render_markdown_code(children) do + Enum.map(children, &escape_html/1) + end + + defp sanitize_markdown_url(value, name) do + if String.downcase(to_string(name)) in ["href", "src"] do + # Percent-encode instead of cutting, so the URL stays complete but can + # never close the attribute or open a tag. + String.replace(value, ~r/["<>]/, fn + "\"" -> "%22" + "<" -> "%3C" + ">" -> "%3E" + end) + else + value + end + end + + defp strip_dangerous_markdown_html(text) do + text + |> String.replace(~r/<(script|style)\b[^>]*>.*?<\/\1>/is, "") + |> String.replace(~r/<(script|style)\b[^>]*>.*\z/is, "") + |> String.replace(~r/<\/(script|style)\s*>/i, "") + end + + defp escape_markdown_html(text) do + text |> HtmlEntities.decode() |> escape_html() + end + + defp escape_html(text) do + {:safe, escaped} = Phoenix.HTML.html_escape(text) + escaped end def html_escape({text, mentions, hashtags}, type) do diff --git a/lib/pleroma/gun.ex b/lib/pleroma/gun.ex index c2d6b4bf2..764eb4b72 100644 --- a/lib/pleroma/gun.ex +++ b/lib/pleroma/gun.ex @@ -6,9 +6,12 @@ defmodule Pleroma.Gun do @callback open(charlist(), pos_integer(), map()) :: {:ok, pid()} @callback info(pid()) :: map() @callback close(pid()) :: :ok + @callback cancel(pid(), reference() | [reference()]) :: :ok @callback await_up(pid, pos_integer()) :: {:ok, atom()} | {:error, atom()} @callback connect(pid(), map()) :: reference() - @callback await(pid(), reference()) :: {:response, :fin, 200, []} + @callback await(pid(), reference()) :: tuple() + @callback await_tunnel_up(pid(), reference() | :undefined, timeout()) :: + {:ok, atom()} | {:error, term()} @callback set_owner(pid(), pid()) :: :ok defp api, do: Pleroma.Config.get([Pleroma.Gun], Pleroma.Gun.API) @@ -19,11 +22,15 @@ defmodule Pleroma.Gun do def close(pid), do: api().close(pid) + def cancel(pid, stream), do: api().cancel(pid, stream) + def await_up(pid, timeout \\ 5_000), do: api().await_up(pid, timeout) def connect(pid, opts), do: api().connect(pid, opts) def await(pid, ref), do: api().await(pid, ref) + def await_tunnel_up(pid, ref, timeout), do: api().await_tunnel_up(pid, ref, timeout) + def set_owner(pid, owner), do: api().set_owner(pid, owner) end diff --git a/lib/pleroma/gun/api.ex b/lib/pleroma/gun/api.ex index ff2100623..8f6387069 100644 --- a/lib/pleroma/gun/api.ex +++ b/lib/pleroma/gun/api.ex @@ -32,6 +32,9 @@ defmodule Pleroma.Gun.API do @impl Gun defdelegate close(pid), to: :gun + @impl Gun + defdelegate cancel(pid, stream), to: :gun + @impl Gun defdelegate await_up(pid, timeout \\ 5_000), to: :gun @@ -41,6 +44,28 @@ defmodule Pleroma.Gun.API do @impl Gun defdelegate await(pid, ref), to: :gun + @impl Gun + def await_tunnel_up(pid, stream_ref, timeout) do + monitor_ref = Process.monitor(pid) + + result = + receive do + {:gun_tunnel_up, ^pid, ^stream_ref, protocol} -> + {:ok, protocol} + + {:gun_down, ^pid, _protocol, reason, _killed_streams} -> + {:error, {:down, reason}} + + {:DOWN, ^monitor_ref, :process, ^pid, reason} -> + {:error, {:down, reason}} + after + timeout -> {:error, :timeout} + end + + Process.demonitor(monitor_ref, [:flush]) + result + end + @impl Gun defdelegate set_owner(pid, owner), to: :gun end diff --git a/lib/pleroma/gun/conn.ex b/lib/pleroma/gun/conn.ex index 804cd11c7..85a2fb0e8 100644 --- a/lib/pleroma/gun/conn.ex +++ b/lib/pleroma/gun/conn.ex @@ -42,18 +42,34 @@ defmodule Pleroma.Gun.Conn do Map.put(opts, :tls_opts, tls_opts) end + defp do_open(%URI{scheme: "http"} = uri, %{proxy: {proxy_host, proxy_port}} = opts) do + with open_opts <- opts |> proxy_open_opts() |> Map.put(:protocols, [:http]), + {:ok, conn} <- Gun.open(proxy_host, proxy_port, open_opts), + {:ok, protocol} <- await_up(conn, opts[:connect_timeout]) do + {:ok, conn, protocol, :forward_proxy} + else + error -> + Logger.warning( + "Opening forward proxy connection to #{compose_uri_log(uri)} failed with error #{inspect(error)}" + ) + + error + end + end + defp do_open(uri, %{proxy: {proxy_host, proxy_port}} = opts) do connect_opts = uri |> destination_opts() |> add_http2_opts(uri.scheme, Map.get(opts, :tls_opts, [])) + |> add_proxy_auth(opts) - with open_opts <- Map.delete(opts, :tls_opts), + with open_opts <- proxy_open_opts(opts), {:ok, conn} <- Gun.open(proxy_host, proxy_port, open_opts), - {:ok, protocol} <- Gun.await_up(conn, opts[:connect_timeout]), - stream <- Gun.connect(conn, connect_opts), - {:response, :fin, 200, _} <- Gun.await(conn, stream) do - {:ok, conn, protocol} + {:ok, _proxy_protocol} <- await_up(conn, opts[:connect_timeout]) do + conn + |> Gun.connect(connect_opts) + |> await_proxy_connect(conn, opts[:connect_timeout]) else error -> Logger.warning( @@ -64,30 +80,29 @@ defmodule Pleroma.Gun.Conn do end end - defp do_open(uri, %{proxy: {proxy_type, proxy_host, proxy_port}} = opts) do - version = - proxy_type - |> to_string() - |> String.last() - |> case do - "4" -> 4 - _ -> 5 - end + defp do_open(_uri, %{proxy: {:socks4, _proxy_host, _proxy_port}}) do + {:error, :socks4_unsupported} + end + defp do_open(uri, %{proxy: {proxy_type, proxy_host, proxy_port}} = opts) + when proxy_type in [:socks, :socks5] do socks_opts = uri |> destination_opts() |> add_http2_opts(uri.scheme, Map.get(opts, :tls_opts, [])) - |> Map.put(:version, version) + |> Map.put(:version, 5) + |> add_socks_proxy_auth(opts) - opts = + open_opts = opts + |> proxy_open_opts() |> Map.put(:protocols, [:socks]) |> Map.put(:socks_opts, socks_opts) - with {:ok, conn} <- Gun.open(proxy_host, proxy_port, opts), - {:ok, protocol} <- Gun.await_up(conn, opts[:connect_timeout]) do - {:ok, conn, protocol} + with {:ok, conn} <- Gun.open(proxy_host, proxy_port, open_opts), + {:ok, :socks} <- await_up(conn, opts[:connect_timeout]), + {:ok, protocol} <- await_tunnel_up(conn, :undefined, opts[:connect_timeout]) do + {:ok, conn, protocol, nil} else error -> Logger.warning( @@ -100,10 +115,11 @@ defmodule Pleroma.Gun.Conn do defp do_open(%URI{host: host, port: port} = uri, opts) do host = Pleroma.HTTP.AdapterHelper.parse_host(host) + opts = Map.put(opts, :transport, transport(uri.scheme)) with {:ok, conn} <- Gun.open(host, port, opts), - {:ok, protocol} <- Gun.await_up(conn, opts[:connect_timeout]) do - {:ok, conn, protocol} + {:ok, protocol} <- await_up(conn, opts[:connect_timeout]) do + {:ok, conn, protocol, nil} else error -> Logger.warning( @@ -120,11 +136,90 @@ defmodule Pleroma.Gun.Conn do end defp add_http2_opts(opts, "https", tls_opts) do - Map.merge(opts, %{protocols: [:http2], transport: :tls, tls_opts: tls_opts}) + Map.merge(opts, %{protocols: [:http2, :http], transport: :tls, tls_opts: tls_opts}) end defp add_http2_opts(opts, _, _), do: opts + defp add_proxy_auth(connect_opts, %{proxy_auth: {username, password}}) + when is_binary(username) and is_binary(password) do + Map.merge(connect_opts, %{username: username, password: password}) + end + + defp add_proxy_auth(connect_opts, _), do: connect_opts + + defp add_socks_proxy_auth(socks_opts, %{proxy_auth: {username, password}}) + when is_binary(username) and is_binary(password) do + Map.put(socks_opts, :auth, [{:username_password, username, password}]) + end + + defp add_socks_proxy_auth(socks_opts, _), do: socks_opts + + defp proxy_open_opts(opts) do + proxy_tls_opts = Map.get(opts, :proxy_tls_opts, []) + + opts = + opts + |> Map.drop([:proxy, :proxy_auth, :proxy_tls_opts, :tls_opts]) + |> Map.put_new(:transport, :tcp) + + if opts.transport == :tls do + Map.put(opts, :tls_opts, proxy_tls_opts) + else + opts + end + end + + defp await_up(conn, timeout) do + case Gun.await_up(conn, timeout) do + {:ok, protocol} -> + {:ok, protocol} + + error -> + Gun.close(conn) + error + end + end + + defp await_proxy_connect(stream, conn, timeout) do + case Gun.await(conn, stream) do + {:response, :fin, 200, _headers} -> + case await_tunnel_up(conn, stream, timeout) do + {:ok, protocol} -> {:ok, conn, protocol, stream} + error -> error + end + + {:response, _fin, 407, _headers} -> + close_with_error(conn, :proxy_auth_failed) + + {:response, _fin, status, _headers} when is_integer(status) -> + close_with_error(conn, {:proxy_connect_failed, status}) + + error -> + Gun.close(conn) + error + end + end + + defp close_with_error(conn, reason) do + Gun.close(conn) + {:error, reason} + end + + defp transport("https"), do: :tls + defp transport(_), do: :tcp + + defp await_tunnel_up(conn, stream_ref, timeout) do + case Gun.await_tunnel_up(conn, stream_ref, timeout) do + {:ok, protocol} -> + {:ok, protocol} + + error -> + Gun.close(conn) + error + end + end + def compose_uri_log(%URI{scheme: scheme, host: host, path: path}) do "#{scheme}://#{host}#{path}" end diff --git a/lib/pleroma/gun/connection_pool.ex b/lib/pleroma/gun/connection_pool.ex index 2e851de19..5e4fc5c3f 100644 --- a/lib/pleroma/gun/connection_pool.ex +++ b/lib/pleroma/gun/connection_pool.ex @@ -4,53 +4,132 @@ defmodule Pleroma.Gun.ConnectionPool do @registry __MODULE__ + @locks Module.concat(__MODULE__, Locks) alias Pleroma.Gun.ConnectionPool.WorkerSupervisor + @connection_option_keys [ + :http_opts, + :http2_opts, + :protocols, + :proxy, + :proxy_auth, + :proxy_tls_opts, + :retry, + :retry_timeout, + :socks_opts, + :tcp_opts, + :tls_opts, + :transport, + :ws_opts + ] + def children do [ - {Registry, keys: :unique, name: @registry}, + {Registry, keys: :duplicate, name: @registry}, + {Registry, keys: :unique, name: @locks}, Pleroma.Gun.ConnectionPool.WorkerSupervisor ] end @spec get_conn(URI.t(), keyword()) :: {:ok, pid()} | {:error, term()} def get_conn(uri, opts) do - key = "#{uri.scheme}:#{uri.host}:#{uri.port}" + key = connection_key(uri, opts) - case Registry.lookup(@registry, key) do - # The key has already been registered, but connection is not up yet - [{worker_pid, nil}] -> - get_gun_pid_from_worker(worker_pid, true) - - [{worker_pid, {gun_pid, _used_by, _crf, _last_reference}}] -> - GenServer.call(worker_pid, :add_client) + case checkout_existing(Registry.lookup(@registry, key)) do + {:ok, gun_pid} -> {:ok, gun_pid} - [] -> - # :gun.set_owner fails in :connected state for whatevever reason, - # so we open the connection in the process directly and send it's pid back - # We trust gun to handle timeouts by itself - case WorkerSupervisor.start_worker([key, uri, opts, self()]) do - {:ok, worker_pid} -> - get_gun_pid_from_worker(worker_pid, false) + :not_available -> + provision_or_wait(key, uri, opts) + end + end - {:error, {:already_started, worker_pid}} -> - get_gun_pid_from_worker(worker_pid, true) + defp checkout_existing([]), do: :not_available - err -> - err + defp checkout_existing([{_worker_pid, nil} | workers]), do: checkout_existing(workers) + + defp checkout_existing([{worker_pid, _value} | workers]) do + case checkout_worker(worker_pid) do + {:ok, gun_pid} -> {:ok, gun_pid} + _ -> checkout_existing(workers) + end + end + + defp provision_or_wait(key, uri, opts) do + result = with_route_lock(key, fn -> provision(key, uri, opts) end) + + case result do + {:ok, gun_pid} -> + {:ok, gun_pid} + + {:new, worker_pid} -> + get_gun_pid_from_worker(worker_pid) + + {:wait, worker_pid} -> + case checkout_worker(worker_pid, :infinity) do + {:ok, gun_pid} -> {:ok, gun_pid} + :busy -> get_conn(uri, opts) + error -> error + end + + error -> + error + end + end + + defp provision(key, uri, opts) do + workers = Registry.lookup(@registry, key) + + case checkout_existing(workers) do + {:ok, gun_pid} -> + {:ok, gun_pid} + + :not_available -> + case Enum.find(workers, fn {_worker_pid, value} -> is_nil(value) end) do + {worker_pid, nil} -> {:wait, worker_pid} + nil -> start_worker(key, uri, opts) end end end - defp get_gun_pid_from_worker(worker_pid, register) do + defp with_route_lock(key, function) do + case Registry.register(@locks, key, nil) do + {:ok, _} -> + try do + function.() + after + Registry.unregister(@locks, key) + end + + {:error, {:already_registered, _owner}} -> + Process.sleep(1) + with_route_lock(key, function) + end + end + + defp start_worker(key, uri, opts) do + # :gun.set_owner fails in :connected state, so the worker opens the connection itself. + case WorkerSupervisor.start_worker([key, uri, opts, self()]) do + {:ok, worker_pid} -> {:new, worker_pid} + error -> error + end + end + + defp checkout_worker(worker_pid, timeout \\ 5_000) do + try do + GenServer.call(worker_pid, :checkout, timeout) + catch + :exit, reason -> {:error, {:connection_worker_exit, reason}} + end + end + + defp get_gun_pid_from_worker(worker_pid) do # GenServer.call will block the process for timeout length if # the server crashes on startup (which will happen if gun fails to connect) # so instead we use cast + monitor ref = Process.monitor(worker_pid) - if register, do: GenServer.cast(worker_pid, {:add_client, self()}) receive do {:conn_pid, pid} -> @@ -68,19 +147,115 @@ defmodule Pleroma.Gun.ConnectionPool do @spec release_conn(pid()) :: :ok def release_conn(conn_pid) do - # :ets.fun2ms(fn {_, {worker_pid, {gun_pid, _, _, _}}} when gun_pid == conn_pid -> + # :ets.fun2ms(fn {_, {worker_pid, {gun_pid, _tunnel}}} when gun_pid == conn_pid -> # worker_pid end) query_result = Registry.select(@registry, [ - {{:_, :"$1", {:"$2", :_, :_, :_}}, [{:==, :"$2", conn_pid}], [:"$1"]} + {{:_, :"$1", {:"$2", :_}}, [{:==, :"$2", conn_pid}], [:"$1"]} ]) case query_result do [worker_pid] -> - GenServer.call(worker_pid, :remove_client) + try do + GenServer.call(worker_pid, :remove_client) + catch + :exit, _ -> :ok + end [] -> :ok end end + + @spec discard_conn(pid()) :: :ok + def discard_conn(conn_pid) do + case worker_for_conn(conn_pid) do + nil -> + :ok + + worker_pid -> + DynamicSupervisor.terminate_child(WorkerSupervisor, worker_pid) + end + end + + @spec cancel_stream(pid(), reference() | [reference()]) :: :ok + def cancel_stream(conn_pid, stream) do + case worker_for_conn(conn_pid) do + nil -> + Pleroma.Gun.cancel(conn_pid, stream) + + worker_pid -> + try do + GenServer.call(worker_pid, {:cancel_stream, stream}) + catch + :exit, _ -> :ok + end + end + end + + @spec register_stream(pid(), reference() | [reference()]) :: :ok + def register_stream(conn_pid, stream) do + call_worker(conn_pid, {:register_stream, stream}) + end + + @spec finish_stream(pid(), reference() | [reference()]) :: :ok + def finish_stream(conn_pid, stream) do + call_worker(conn_pid, {:finish_stream, stream}) + end + + @spec release_stream(pid(), reference() | [reference()]) :: :ok + def release_stream(conn_pid, stream) do + call_worker(conn_pid, {:release_stream, stream}) + end + + @spec tunnel_ref(pid()) :: reference() | :forward_proxy | nil + def tunnel_ref(conn_pid) do + case Registry.select(@registry, [ + {{:_, :_, {:"$1", :"$2"}}, [{:==, :"$1", conn_pid}], [:"$2"]} + ]) do + [tunnel] -> tunnel + [] -> nil + end + end + + defp worker_for_conn(conn_pid) do + case Registry.select(@registry, [ + {{:_, :"$1", {:"$2", :_}}, [{:==, :"$2", conn_pid}], [:"$1"]} + ]) do + [worker_pid] -> worker_pid + [] -> nil + end + end + + defp call_worker(conn_pid, message) do + case worker_for_conn(conn_pid) do + nil -> + :ok + + worker_pid -> + try do + GenServer.call(worker_pid, message) + catch + :exit, _ -> :ok + end + end + end + + defp connection_key(uri, opts) do + origin = { + String.downcase(uri.scheme), + String.downcase(uri.host), + uri.port + } + + profile = + opts + |> Map.new() + |> Map.take(@connection_option_keys) + |> :erlang.term_to_binary() + |> then(&:crypto.hash(:sha256, &1)) + + fingerprint = Base.encode16(profile, case: :lower) + Enum.join(Tuple.to_list(origin) ++ [fingerprint], ":") + end end diff --git a/lib/pleroma/gun/connection_pool/reclaimer.ex b/lib/pleroma/gun/connection_pool/reclaimer.ex index 3580d38f5..bd7de7e31 100644 --- a/lib/pleroma/gun/connection_pool/reclaimer.ex +++ b/lib/pleroma/gun/connection_pool/reclaimer.ex @@ -9,11 +9,11 @@ defmodule Pleroma.Gun.ConnectionPool.Reclaimer do def start_monitor do pid = - case GenServer.start(__MODULE__, [], name: {:via, Registry, {registry(), "reclaimer"}}) do + case GenServer.start(__MODULE__, [], name: __MODULE__) do {:ok, pid} -> pid - {:error, {:already_registered, pid}} -> + {:error, {:already_started, pid}} -> pid end @@ -41,17 +41,16 @@ defmodule Pleroma.Gun.ConnectionPool.Reclaimer do reclaim_max: reclaim_max }) - # :ets.fun2ms( - # fn {_, {worker_pid, {_, used_by, crf, last_reference}}} when used_by == [] -> - # {worker_pid, crf, last_reference} end) - unused_conns = + workers = Registry.select( registry(), [ - {{:_, :"$1", {:_, :"$2", :"$3", :"$4"}}, [{:==, :"$2", []}], [{{:"$1", :"$3", :"$4"}}]} + {{:_, :"$1", {:_, :_}}, [], [:"$1"]} ] ) + unused_conns = Enum.flat_map(workers, &reclaim_info/1) + case unused_conns do [] -> :telemetry.execute( @@ -65,25 +64,43 @@ defmodule Pleroma.Gun.ConnectionPool.Reclaimer do {:stop, :no_unused_conns, nil} unused_conns -> - reclaimed = + reclaimed_count = unused_conns - |> Enum.sort(fn {_pid1, crf1, last_reference1}, {_pid2, crf2, last_reference2} -> - crf1 <= crf2 and last_reference1 <= last_reference2 - end) - |> Enum.take(reclaim_max) + |> Enum.sort_by(fn {_pid, crf, last_reference} -> {crf, last_reference} end) + |> Enum.reduce_while(0, fn + _worker, count when count == reclaim_max -> + {:halt, count} - reclaimed - |> Enum.each(fn {pid, _, _} -> - DynamicSupervisor.terminate_child(Pleroma.Gun.ConnectionPool.WorkerSupervisor, pid) - end) + {worker_pid, _crf, _last_reference}, count -> + if reclaim(worker_pid), do: {:cont, count + 1}, else: {:cont, count} + end) :telemetry.execute( [:pleroma, :connection_pool, :reclaim, :stop], - %{reclaimed_count: Enum.count(reclaimed)}, + %{reclaimed_count: reclaimed_count}, %{max_connections: max_connections} ) {:stop, :normal, nil} end end + + defp reclaim_info(worker_pid) do + try do + case GenServer.call(worker_pid, :reclaim_info) do + {:idle, crf, last_reference} -> [{worker_pid, crf, last_reference}] + :in_use -> [] + end + catch + :exit, _ -> [] + end + end + + defp reclaim(worker_pid) do + try do + GenServer.call(worker_pid, :reclaim) == :reclaimed + catch + :exit, _ -> false + end + end end diff --git a/lib/pleroma/gun/connection_pool/worker.ex b/lib/pleroma/gun/connection_pool/worker.ex index 38527ec1d..24e067e87 100644 --- a/lib/pleroma/gun/connection_pool/worker.ex +++ b/lib/pleroma/gun/connection_pool/worker.ex @@ -8,25 +8,22 @@ defmodule Pleroma.Gun.ConnectionPool.Worker do defp registry, do: Pleroma.Gun.ConnectionPool - def start_link([key | _] = opts) do - GenServer.start_link(__MODULE__, opts, name: {:via, Registry, {registry(), key}}) - end + def start_link(opts), do: GenServer.start_link(__MODULE__, opts) @impl true - def init([_key, _uri, _opts, _client_pid] = opts) do + def init([key, _uri, _opts, _client_pid] = opts) do + {:ok, _} = Registry.register(registry(), key, nil) {:ok, nil, {:continue, {:connect, opts}}} end @impl true def handle_continue({:connect, [key, uri, opts, client_pid]}, _) do - with {:ok, conn_pid, protocol} <- Gun.Conn.open(uri, opts), + with {:ok, conn_pid, protocol, tunnel} <- Gun.Conn.open(uri, opts), Process.link(conn_pid) do time = :erlang.monotonic_time(:millisecond) - {_, _} = - Registry.update_value(registry(), key, fn _ -> - {conn_pid, [client_pid], 1, time} - end) + Registry.unregister(registry(), key) + {:ok, _} = Registry.register(registry(), key, {conn_pid, tunnel}) send(client_pid, {:conn_pid, conn_pid}) @@ -34,8 +31,14 @@ defmodule Pleroma.Gun.ConnectionPool.Worker do %{ key: key, timer: nil, - client_monitors: %{client_pid => Process.monitor(client_pid)}, - protocol: protocol + clients: %{ + client_pid => %{count: 1, monitor: Process.monitor(client_pid), streams: MapSet.new()} + }, + conn_pid: conn_pid, + protocol: protocol, + proxied: Keyword.has_key?(opts, :proxy), + crf: 1, + last_reference: time }, :hibernate} else err -> @@ -44,106 +47,225 @@ defmodule Pleroma.Gun.ConnectionPool.Worker do end @impl true - def handle_cast({:add_client, client_pid}, state) do - case handle_call(:add_client, {client_pid, nil}, state) do - {:reply, conn_pid, state, :hibernate} -> - send(client_pid, {:conn_pid, conn_pid}) - {:noreply, state, :hibernate} - end + def handle_call(:checkout, _from, %{protocol: protocol, clients: clients} = state) + when protocol != :http2 and map_size(clients) > 0 do + {:reply, :busy, state, :hibernate} end - @impl true - def handle_cast({:remove_client, client_pid}, state) do - case handle_call(:remove_client, {client_pid, nil}, state) do - {:reply, _, state, :hibernate} -> - {:noreply, state, :hibernate} - end - end - - @impl true - def handle_call(:add_client, {client_pid, _}, %{key: key, protocol: protocol} = state) do + def handle_call(:checkout, {client_pid, _}, %{key: key, protocol: protocol} = state) do time = :erlang.monotonic_time(:millisecond) - {{conn_pid, used_by, _, _}, _} = - Registry.update_value(registry(), key, fn {conn_pid, used_by, crf, last_reference} -> - {conn_pid, [client_pid | used_by], crf(time - last_reference, crf), time} - end) + [{_, {conn_pid, _tunnel}}] = + Registry.lookup(registry(), key) |> Enum.filter(&(elem(&1, 0) == self())) + + state = + state + |> cancel_idle_timer() + |> add_client(client_pid) + |> Map.put(:crf, crf(time - state.last_reference, state.crf)) + |> Map.put(:last_reference, time) :telemetry.execute( [:pleroma, :connection_pool, :client, :add], - %{client_pid: client_pid, clients: used_by}, + %{client_pid: client_pid, clients: Map.keys(state.clients)}, %{key: state.key, protocol: protocol} ) - state = - if state.timer != nil do - Process.cancel_timer(state[:timer]) - %{state | timer: nil} - else - state - end - - ref = Process.monitor(client_pid) - - state = put_in(state.client_monitors[client_pid], ref) - {:reply, conn_pid, state, :hibernate} + {:reply, {:ok, conn_pid}, state, :hibernate} end @impl true - def handle_call(:remove_client, {client_pid, _}, %{key: key} = state) do - {{_conn_pid, used_by, _crf, _last_reference}, _} = - Registry.update_value(registry(), key, fn {conn_pid, used_by, crf, last_reference} -> - {conn_pid, List.delete(used_by, client_pid), crf, last_reference} - end) + def handle_call(:remove_client, {client_pid, _}, state) do + case active_streams(state, client_pid) do + [] -> + {:reply, :ok, remove_client(state, client_pid, true), :hibernate} - {ref, state} = pop_in(state.client_monitors[client_pid]) + streams when state.protocol == :http2 -> + Enum.each(streams, &(:ok = Gun.cancel(state.conn_pid, &1))) + {:reply, :ok, remove_client(state, client_pid, true), :hibernate} - Process.demonitor(ref, [:flush]) + _streams -> + {:stop, :normal, :ok, state} + end + end - timer = - if used_by == [] do - max_idle = Pleroma.Config.get([:connections_pool, :max_idle_time], 30_000) - Process.send_after(self(), :idle_close, max_idle) - else - nil - end + def handle_call({:register_stream, stream}, {client_pid, _}, state) do + {:reply, :ok, update_stream(state, client_pid, &MapSet.put(&1, stream)), :hibernate} + end - {:reply, :ok, %{state | timer: timer}, :hibernate} + def handle_call({:finish_stream, stream}, {client_pid, _}, state) do + {:reply, :ok, update_stream(state, client_pid, &MapSet.delete(&1, stream)), :hibernate} + end + + def handle_call({:release_stream, stream}, {client_pid, _}, state) do + state = update_stream(state, client_pid, &MapSet.delete(&1, stream)) + {:reply, :ok, remove_client(state, client_pid, true), :hibernate} + end + + def handle_call({:cancel_stream, stream}, {client_pid, _}, %{protocol: :http2} = state) do + :ok = Gun.cancel(state.conn_pid, stream) + state = update_stream(state, client_pid, &MapSet.delete(&1, stream)) + {:reply, :ok, remove_client(state, client_pid, true), :hibernate} + end + + def handle_call({:cancel_stream, _stream}, _from, state) do + {:stop, :normal, :ok, state} + end + + def handle_call(:reclaim_info, _from, %{clients: clients} = state) + when map_size(clients) == 0 do + {:reply, {:idle, state.crf, state.last_reference}, state, :hibernate} + end + + def handle_call(:reclaim_info, _from, state) do + {:reply, :in_use, state, :hibernate} + end + + def handle_call(:reclaim, _from, %{clients: clients} = state) when map_size(clients) == 0 do + {:stop, :normal, :reclaimed, state} + end + + def handle_call(:reclaim, _from, state) do + {:reply, :in_use, state, :hibernate} end @impl true - def handle_info(:idle_close, state) do - # Gun monitors the owner process, and will close the connection automatically - # when it's terminated + def handle_info({:idle_close, token}, %{timer: {_timer_ref, token}, clients: clients} = state) + when map_size(clients) == 0 do + # Gun monitors the owner process, and will close the connection automatically. {:stop, :normal, state} end + def handle_info({:idle_close, _token}, state), do: {:noreply, state, :hibernate} + @impl true - def handle_info({:gun_up, _pid, _protocol}, state) do - {:noreply, state, :hibernate} + def handle_info({:gun_up, _pid, protocol}, state) do + {:noreply, %{state | protocol: protocol}, :hibernate} end - # Gracefully shutdown if the connection got closed without any streams left + @impl true + def handle_info({:gun_error, conn_pid, stream, _reason}, %{conn_pid: conn_pid} = state) do + {:noreply, delete_stream(state, stream), :hibernate} + end + + @impl true + def handle_info({:gun_down, _pid, _protocol, _reason, _streams}, %{proxied: true} = state) do + {:stop, :normal, state} + end + + # Gracefully shutdown if the connection got closed without any streams left. @impl true def handle_info({:gun_down, _pid, _protocol, _reason, []}, state) do {:stop, :normal, state} end - # Otherwise, wait for retry + # Otherwise, wait for retry. @impl true def handle_info({:gun_down, _pid, _protocol, _reason, _killed_streams}, state) do {:noreply, state, :hibernate} end @impl true - def handle_info({:DOWN, _ref, :process, pid, reason}, state) do - :telemetry.execute( - [:pleroma, :connection_pool, :client, :dead], - %{client_pid: pid, reason: reason}, - %{key: state.key} - ) + def handle_info({:DOWN, ref, :process, pid, reason}, state) do + case state.clients[pid] do + %{monitor: ^ref} -> + :telemetry.execute( + [:pleroma, :connection_pool, :client, :dead], + %{client_pid: pid, reason: reason}, + %{key: state.key} + ) - handle_cast({:remove_client, pid}, state) + handle_dead_client(state, pid) + + _ -> + {:noreply, state, :hibernate} + end + end + + defp add_client(state, client_pid) do + update_in(state.clients, fn clients -> + case clients[client_pid] do + nil -> + Map.put(clients, client_pid, %{ + count: 1, + monitor: Process.monitor(client_pid), + streams: MapSet.new() + }) + + client -> + Map.put(clients, client_pid, %{client | count: client.count + 1}) + end + end) + end + + defp handle_dead_client(state, client_pid) do + streams = active_streams(state, client_pid) + + cond do + streams == [] -> + {:noreply, remove_client(state, client_pid, false, true), :hibernate} + + state.protocol == :http2 -> + Enum.each(streams, &(:ok = Gun.cancel(state.conn_pid, &1))) + {:noreply, remove_client(state, client_pid, false, true), :hibernate} + + true -> + {:stop, :normal, state} + end + end + + defp active_streams(state, client_pid) do + case state.clients[client_pid] do + %{streams: streams} -> MapSet.to_list(streams) + nil -> [] + end + end + + defp update_stream(state, client_pid, update) do + update_in(state.clients, fn clients -> + case clients[client_pid] do + nil -> clients + client -> Map.put(clients, client_pid, %{client | streams: update.(client.streams)}) + end + end) + end + + defp delete_stream(state, stream) do + update_in(state.clients, fn clients -> + Map.new(clients, fn {pid, client} -> + {pid, %{client | streams: MapSet.delete(client.streams, stream)}} + end) + end) + end + + defp remove_client(state, client_pid, demonitor, all \\ false) do + case state.clients[client_pid] do + nil -> + state + + %{count: count} = client when count > 1 and not all -> + put_in(state.clients[client_pid], %{client | count: count - 1}) + + client -> + if demonitor, do: Process.demonitor(client.monitor, [:flush]) + state = update_in(state.clients, &Map.delete(&1, client_pid)) + + if map_size(state.clients) == 0, do: schedule_idle_close(state), else: state + end + end + + defp cancel_idle_timer(%{timer: nil} = state), do: state + + defp cancel_idle_timer(%{timer: {timer_ref, _token}} = state) do + Process.cancel_timer(timer_ref) + %{state | timer: nil} + end + + defp schedule_idle_close(state) do + max_idle = Pleroma.Config.get([:connections_pool, :max_idle_time], 30_000) + token = make_ref() + timer_ref = Process.send_after(self(), {:idle_close, token}, max_idle) + %{state | timer: {timer_ref, token}} end # LRFU policy: https://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.55.1478 diff --git a/lib/pleroma/gun/connection_pool/worker_supervisor.ex b/lib/pleroma/gun/connection_pool/worker_supervisor.ex index b9dedf61e..2c60bea8f 100644 --- a/lib/pleroma/gun/connection_pool/worker_supervisor.ex +++ b/lib/pleroma/gun/connection_pool/worker_supervisor.ex @@ -26,7 +26,14 @@ defmodule Pleroma.Gun.ConnectionPool.WorkerSupervisor do def start_worker(opts, true) do case DynamicSupervisor.start_child(__MODULE__, {Worker, opts}) do {:error, :max_children} -> - :telemetry.execute([:pleroma, :connection_pool, :provision_failure], %{opts: opts}) + [key | _] = opts + + :telemetry.execute( + [:pleroma, :connection_pool, :provision_failure], + %{}, + %{key: key} + ) + {:error, :pool_full} res -> diff --git a/lib/pleroma/html/uri_attributes.ex b/lib/pleroma/html/uri_attributes.ex new file mode 100644 index 000000000..491f43e3d --- /dev/null +++ b/lib/pleroma/html/uri_attributes.ex @@ -0,0 +1,40 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.HTML.URIAttributes do + @moduledoc """ + URI attribute rules for the scrubbers. + + Works like `FastSanitize.Sanitizer.Meta.allow_tag_with_uri_attributes/3`, but + checks the scheme the way a browser reads the URL. Browsers remove tabs and + newlines anywhere and control characters and spaces at the start, so + `java\\tscript:` is a `javascript:` link to them, while `URI.parse/1` finds no + scheme and FastSanitize lets it through. + """ + + defmacro allow_tag_with_uri_attributes(tag, attrs, valid_schemes) do + for attr <- attrs do + quote do + def scrub_attribute(unquote(tag), {unquote(attr), "&" <> _}), do: nil + + def scrub_attribute(unquote(tag), {unquote(attr), uri} = attribute) do + if Pleroma.HTML.URIAttributes.valid_scheme?(uri, unquote(valid_schemes)), + do: attribute + end + end + end + end + + @spec valid_scheme?(String.t(), [String.t()]) :: boolean() + def valid_scheme?(uri, valid_schemes) do + scheme = + uri + |> String.replace(~r/[\t\n\r]/, "") + |> String.replace(~r/\A[\x00-\x20]+/, "") + |> URI.parse() + |> Map.get(:scheme) + + is_nil(scheme) or scheme in valid_schemes + end +end diff --git a/lib/pleroma/http.ex b/lib/pleroma/http.ex index bdeb2171e..c230cea46 100644 --- a/lib/pleroma/http.ex +++ b/lib/pleroma/http.ex @@ -130,5 +130,9 @@ defmodule Pleroma.HTTP do end defp default_middleware, - do: [Tesla.Middleware.FollowRedirects, Pleroma.Tesla.Middleware.EncodeUrl] + do: [ + Tesla.Middleware.FollowRedirects, + Pleroma.Tesla.Middleware.EncodeUrl, + Pleroma.Tesla.Middleware.MaxBody + ] end diff --git a/lib/pleroma/http/adapter/gun.ex b/lib/pleroma/http/adapter/gun.ex new file mode 100644 index 000000000..dea1798a1 --- /dev/null +++ b/lib/pleroma/http/adapter/gun.ex @@ -0,0 +1,253 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.HTTP.Adapter.Gun do + @behaviour Tesla.Adapter + + alias Pleroma.Gun.ConnectionPool + alias Tesla.Multipart + + @default_timeout 1_000 + + @impl Tesla.Adapter + def call(env, opts) do + adapter_opts = + Tesla.Adapter.opts( + [close_conn: true, body_as: :plain, send_body: :at_once, receive: true], + env, + opts + ) + |> Map.new() + + with {:ok, status, headers, body} <- request(env, adapter_opts) do + env = %{env | status: status, headers: format_headers(headers), body: body} + {:ok, put_stream_ref(env, body)} + end + end + + defp put_stream_ref(env, %{stream: stream}) do + put_in(env.opts[:adapter][:stream], stream) + end + + defp put_stream_ref(env, _body), do: env + + defp request(env, %{conn: conn, tunnel: tunnel} = opts) do + uri = URI.parse(Tesla.build_url(env)) + path = request_path(uri, tunnel) + method = Tesla.Adapter.Shared.format_method(env.method) + {headers, body, send_body} = prepare_body(env.headers, env.body, opts[:send_body]) + headers = maybe_add_forward_proxy_headers(headers, uri, opts, tunnel) + request_opts = request_opts(opts[:reply_to], tunnel) + + stream = + try do + open_stream(conn, method, path, headers, body, request_opts, send_body) + catch + kind, reason -> + ConnectionPool.release_conn(conn) + :erlang.raise(kind, reason, __STACKTRACE__) + end + + :ok = ConnectionPool.register_stream(conn, stream) + :ok = send_stream_body(conn, stream, body, send_body) + response = read_response(conn, stream, opts) + + case response do + {:error, reason} -> + ConnectionPool.cancel_stream(conn, stream) + {:error, {:gun_stream_error, reason}} + + response -> + unless opts[:body_as] in [:chunks, :stream] and stream_body?(response) do + :ok = ConnectionPool.finish_stream(conn, stream) + end + + if opts[:close_conn] and opts[:body_as] not in [:stream, :chunks] do + :gun.close(conn) + end + + response + end + end + + defp stream_body?({:ok, _status, _headers, %{stream: _stream}}), do: true + defp stream_body?({:ok, _status, _headers, %Stream{}}), do: true + defp stream_body?(_response), do: false + + defp request_opts(reply_to, nil), do: %{reply_to: reply_to || self()} + + defp request_opts(reply_to, :forward_proxy), do: %{reply_to: reply_to || self()} + + defp request_opts(reply_to, tunnel) do + %{reply_to: reply_to || self(), tunnel: tunnel} + end + + defp request_path(uri, :forward_proxy) do + path = Tesla.Adapter.Shared.prepare_path(uri.path, uri.query) + "#{uri.scheme}://#{authority(uri)}#{path}" + end + + defp request_path(uri, _tunnel) do + Tesla.Adapter.Shared.prepare_path(uri.path, uri.query) + end + + defp maybe_add_forward_proxy_headers(headers, uri, opts, :forward_proxy) do + headers + |> put_header_new("host", authority(uri)) + |> maybe_put_proxy_authorization(opts) + end + + defp maybe_add_forward_proxy_headers(headers, _uri, _opts, _tunnel), do: headers + + defp maybe_put_proxy_authorization(headers, %{proxy_auth: {username, password}}) + when is_binary(username) and is_binary(password) do + value = "Basic " <> Base.encode64(username <> ":" <> password) + List.keystore(headers, "proxy-authorization", 0, {"proxy-authorization", value}) + end + + defp maybe_put_proxy_authorization(headers, _opts), do: headers + + defp put_header_new(headers, name, value) do + if List.keymember?(headers, name, 0), do: headers, else: [{name, value} | headers] + end + + defp authority(%URI{scheme: scheme, host: host, port: port}) do + host = if String.contains?(host, ":"), do: "[#{host}]", else: host + if port == URI.default_port(scheme), do: host, else: "#{host}:#{port}" + end + + defp prepare_body(headers, %Multipart{} = multipart, _send_body) do + {format_headers(headers ++ Multipart.headers(multipart)), Multipart.body(multipart), :stream} + end + + defp prepare_body(headers, body, _send_body) + when is_struct(body, Stream) or is_function(body) do + {format_headers(headers), body, :stream} + end + + defp prepare_body(headers, body, send_body) do + {format_headers(headers), body || "", send_body} + end + + defp open_stream(conn, method, path, headers, _body, request_opts, :stream) do + :gun.headers(conn, method, path, headers, request_opts) + end + + defp open_stream(conn, method, path, headers, body, request_opts, :at_once) do + :gun.request(conn, method, path, headers, body, request_opts) + end + + defp send_stream_body(conn, stream, body, :stream) do + try do + for data <- body, do: :ok = :gun.data(conn, stream, :nofin, data) + :gun.data(conn, stream, :fin, "") + catch + kind, reason -> + ConnectionPool.cancel_stream(conn, stream) + :erlang.raise(kind, reason, __STACKTRACE__) + end + end + + defp send_stream_body(_conn, _stream, _body, :at_once), do: :ok + + defp read_response(conn, stream, opts) do + receive? = opts[:receive] + + receive do + {:gun_response, ^conn, ^stream, :fin, status, headers} -> + {:ok, status, headers, ""} + + {:gun_response, ^conn, ^stream, :nofin, status, headers} -> + format_response(conn, stream, opts, status, headers, opts[:body_as]) + + {:gun_up, ^conn, _protocol} when receive? -> + read_response(conn, stream, opts) + + {:gun_error, ^conn, reason} -> + {:error, reason} + + {:gun_error, ^conn, ^stream, reason} -> + {:error, reason} + + {:gun_down, ^conn, _protocol, _reason, _killed_streams} when receive? -> + read_response(conn, stream, opts) + + {:DOWN, _ref, :process, ^conn, reason} -> + {:error, reason} + after + opts[:timeout] || @default_timeout -> {:error, :recv_response_timeout} + end + end + + defp format_response(conn, stream, opts, status, headers, :plain) do + case read_body(conn, stream, opts) do + {:ok, body} -> + {:ok, status, headers, body} + + {:error, error} -> + :ok = :gun.flush(stream) + {:error, error} + end + end + + defp format_response(conn, stream, opts, status, headers, :stream) do + body = + Stream.resource( + fn -> :reading end, + fn + :reading -> + case Tesla.Adapter.Gun.read_chunk(conn, stream, opts) do + {:nofin, part} -> {[part], :reading} + {:fin, part} -> {[part], :done} + {:error, reason} -> raise "Gun stream failed: #{inspect(reason)}" + end + + :done -> + {:halt, :done} + end, + fn _state -> + if opts[:close_conn], do: :gun.close(conn) + end + ) + + {:ok, status, headers, body} + end + + defp format_response(conn, stream, opts, status, headers, :chunks) do + {:ok, status, headers, %{pid: conn, stream: stream, opts: Map.to_list(opts)}} + end + + defp read_body(conn, stream, opts, acc \\ "") do + receive do + {:gun_data, ^conn, ^stream, :fin, body} -> + append_body(acc, body, opts[:max_body]) + + {:gun_data, ^conn, ^stream, :nofin, body} -> + with {:ok, acc} <- append_body(acc, body, opts[:max_body]) do + read_body(conn, stream, opts, acc) + end + + {:gun_error, ^conn, ^stream, reason} -> + {:error, reason} + + {:DOWN, _ref, :process, ^conn, reason} -> + {:error, reason} + after + opts[:timeout] || @default_timeout -> {:error, :recv_body_timeout} + end + end + + defp append_body(acc, part, nil), do: {:ok, acc <> part} + + defp append_body(acc, part, limit) do + body = acc <> part + if byte_size(body) <= limit, do: {:ok, body}, else: {:error, :body_too_large} + end + + defp format_headers(headers) do + Enum.map(headers, fn {key, value} -> + {String.downcase(to_string(key)), to_string(value)} + end) + end +end diff --git a/lib/pleroma/http/adapter_helper.ex b/lib/pleroma/http/adapter_helper.ex index 32c1080f7..4f0e23f20 100644 --- a/lib/pleroma/http/adapter_helper.ex +++ b/lib/pleroma/http/adapter_helper.ex @@ -46,6 +46,24 @@ defmodule Pleroma.HTTP.AdapterHelper do |> adapter_helper().options(uri) end + @doc """ + The configured Tesla adapter with TLS options, for clients that do not go + through `Pleroma.HTTP`. + """ + @spec tesla_adapter() :: module() | {module(), keyword()} + def tesla_adapter do + case adapter() do + Tesla.Adapter.Hackney -> + {Tesla.Adapter.Hackney, Pleroma.HTTP.TLS.put_hackney_ssl_options([])} + + Tesla.Adapter.Gun -> + {Tesla.Adapter.Gun, certificates_verification: true} + + adapter -> + adapter + end + end + defp adapter, do: Application.get_env(:tesla, :adapter) defp adapter_helper do diff --git a/lib/pleroma/http/adapter_helper/gun.ex b/lib/pleroma/http/adapter_helper/gun.ex index 30ba26765..11efaca88 100644 --- a/lib/pleroma/http/adapter_helper/gun.ex +++ b/lib/pleroma/http/adapter_helper/gun.ex @@ -28,6 +28,7 @@ defmodule Pleroma.HTTP.AdapterHelper.Gun do @defaults |> Keyword.merge(config_opts) + |> ssl_options_to_tls_opts() |> add_scheme_opts(uri) |> AdapterHelper.maybe_add_proxy(proxy) |> Keyword.merge(incoming_opts) @@ -35,6 +36,15 @@ defmodule Pleroma.HTTP.AdapterHelper.Gun do |> maybe_stream() end + # [:http, :adapter, :ssl_options] is Hackney's name for the TLS options, + # and AdminFE sets it. Keep those settings when Gun is the adapter. + defp ssl_options_to_tls_opts(opts) do + case Keyword.pop(opts, :ssl_options) do + {nil, opts} -> opts + {ssl_options, opts} -> Keyword.put_new(opts, :tls_opts, ssl_options) + end + end + defp add_scheme_opts(opts, %{scheme: "http"}), do: opts defp add_scheme_opts(opts, %{scheme: "https"}) do @@ -48,10 +58,10 @@ defmodule Pleroma.HTTP.AdapterHelper.Gun do Keyword.put(opts, :timeout, recv_timeout) end - # Gun uses [body_as: :stream] + # Keep the pool lease until the caller consumes or cancels the body. defp maybe_stream(opts) do case Keyword.pop(opts, :stream, nil) do - {true, opts} -> Keyword.put(opts, :body_as, :stream) + {true, opts} -> Keyword.put(opts, :body_as, :chunks) {_, opts} -> opts end end diff --git a/lib/pleroma/http/adapter_helper/hackney.ex b/lib/pleroma/http/adapter_helper/hackney.ex index 2b1502001..b9e1d020f 100644 --- a/lib/pleroma/http/adapter_helper/hackney.ex +++ b/lib/pleroma/http/adapter_helper/hackney.ex @@ -28,6 +28,25 @@ defmodule Pleroma.HTTP.AdapterHelper.Hackney do |> add_scheme_opts(uri) |> maybe_add_with_body() |> Pleroma.HTTP.AdapterHelper.maybe_add_proxy(proxy) + |> Keyword.put(:follow_redirect, false) + |> Keyword.put(:force_redirect, false) + |> normalize_proxy() + |> Pleroma.HTTP.TLS.put_hackney_ssl_options() + end + + # Hackney 4 ignores a proxy whose host is an IP tuple and connects directly. + # ConfigDB turns a numeric proxy host into such a tuple. + defp normalize_proxy(opts) do + case Keyword.fetch(opts, :proxy) do + {:ok, {type, host, port}} when is_tuple(host) -> + Keyword.put(opts, :proxy, {type, :inet.ntoa(host), port}) + + {:ok, {host, port}} when is_tuple(host) -> + Keyword.put(opts, :proxy, {:inet.ntoa(host), port}) + + _ -> + opts + end end defp add_scheme_opts(opts, _), do: opts diff --git a/lib/pleroma/http/tls.ex b/lib/pleroma/http/tls.ex new file mode 100644 index 000000000..bb90f8e4e --- /dev/null +++ b/lib/pleroma/http/tls.ex @@ -0,0 +1,49 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.HTTP.TLS do + @moduledoc """ + TLS client options that trust the CAStore bundle. + + Hackney otherwise trusts its own certifi bundle. Hackney 1 drops all of its + default TLS options, including certificate checks, as soon as any + `ssl_options` are given. Hackney 4 merges them, so `partial_chain` must be + set too: Hackney's own one trusts any certificate in the peer chain that is + in certifi. + """ + + @spec ssl_options(keyword()) :: keyword() + def ssl_options(overrides \\ []) do + defaults = [ + verify: :verify_peer, + cacertfile: CAStore.file_path(), + depth: 20, + customize_hostname_check: [match_fun: :public_key.pkix_verify_hostname_match_fun(:https)], + partial_chain: &__MODULE__.reject_partial_chain/1 + ] + + # Hackney drops cacerts when cacertfile is set too. + defaults = + if Keyword.has_key?(overrides, :cacerts), + do: Keyword.delete(defaults, :cacertfile), + else: defaults + + Keyword.merge(defaults, overrides) + end + + @doc false + def reject_partial_chain(_certs), do: :unknown_ca + + @spec put_hackney_ssl_options(keyword()) :: keyword() + def put_hackney_ssl_options(opts) do + # Hackney 4 ignores the top-level insecure option and falls back to certifi. + if opts[:insecure] do + Keyword.put(opts, :ssl_options, verify: :verify_none) + else + opts + |> Keyword.put(:ssl_options, ssl_options(Keyword.get(opts, :ssl_options, []))) + |> Keyword.put(:proxy_ssl_options, ssl_options(Keyword.get(opts, :proxy_ssl_options, []))) + end + end +end diff --git a/lib/pleroma/list.ex b/lib/pleroma/list.ex index cd23bd0a9..3e8a83342 100644 --- a/lib/pleroma/list.ex +++ b/lib/pleroma/list.ex @@ -9,6 +9,7 @@ defmodule Pleroma.List do import Ecto.Changeset alias Pleroma.Activity + alias Pleroma.Emoji alias Pleroma.Repo alias Pleroma.User @@ -18,14 +19,53 @@ defmodule Pleroma.List do field(:following, {:array, :string}, default: []) field(:ap_id, :string) field(:exclusive, :boolean, default: false) + field(:emoji, :string) timestamps() end def update_changeset(list, attrs \\ %{}) do list - |> cast(attrs, [:title, :exclusive]) + |> cast(attrs, [:title, :exclusive, :emoji]) |> validate_required([:title]) + |> fix_emoji() + |> validate_emoji() + end + + defp fix_emoji(changeset) do + with {:emoji_field, emoji} when is_binary(emoji) <- + {:emoji_field, get_field(changeset, :emoji)}, + {:fixed_emoji, emoji} <- + {:fixed_emoji, + emoji + |> Emoji.fully_qualify_emoji() + |> Emoji.maybe_quote()} do + put_change(changeset, :emoji, emoji) + else + {:emoji_field, _} -> changeset + end + end + + defp validate_emoji(changeset) do + validate_change(changeset, :emoji, fn + :emoji, nil -> + [] + + :emoji, emoji -> + if Emoji.unicode?(emoji) or valid_local_custom_emoji?(emoji) do + [] + else + [emoji: "Invalid emoji"] + end + end) + end + + defp valid_local_custom_emoji?(emoji) do + with %{file: _path} <- Emoji.get(emoji) do + true + else + _ -> false + end end def follow_changeset(list, attrs \\ %{}) do diff --git a/lib/pleroma/moderation_log.ex b/lib/pleroma/moderation_log.ex index 90219312c..260c139ec 100644 --- a/lib/pleroma/moderation_log.ex +++ b/lib/pleroma/moderation_log.ex @@ -346,6 +346,16 @@ defmodule Pleroma.ModerationLog do "@#{actor_nickname} approved users: #{users_to_nicknames_string(users)}" end + def get_log_entry_message(%ModerationLog{ + data: %{ + "actor" => %{"nickname" => actor_nickname}, + "action" => "reject", + "subject" => users + } + }) do + "@#{actor_nickname} rejected users: #{users_to_nicknames_string(users)}" + end + def get_log_entry_message(%ModerationLog{ data: %{ "actor" => %{"nickname" => actor_nickname}, diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex index b8693c3a8..afd750823 100644 --- a/lib/pleroma/notification.ex +++ b/lib/pleroma/notification.ex @@ -35,6 +35,7 @@ defmodule Pleroma.Notification do # remember to add a migration to add it to the `notifications_type` enum # as well. field(:type, :string) + field(:group_key, :string) belongs_to(:user, User, type: FlakeId.Ecto.CompatType) belongs_to(:activity, Activity, type: FlakeId.Ecto.CompatType) @@ -49,7 +50,7 @@ defmodule Pleroma.Notification do |> type_from_activity() notification - |> changeset(%{type: type}) + |> changeset(%{type: type, group_key: grouped_notification_key(type, activity)}) |> Repo.update() end end @@ -62,6 +63,99 @@ defmodule Pleroma.Notification do |> Repo.aggregate(:count, :id) end + @groupable_notification_types ~w{favourite follow reblog} + @group_bucket_seconds 12 * 60 * 60 + + def groupable_notification_types, do: @groupable_notification_types + + def group_notifications(notifications, grouped_types \\ nil) do + grouped_types = normalize_grouped_types(grouped_types) + + {group_keys, grouped_notifications} = + Enum.reduce(notifications, {[], %{}}, fn notification, + {group_keys, grouped_notifications} -> + group_key = group_key(notification, grouped_types) + + if Map.has_key?(grouped_notifications, group_key) do + {group_keys, Map.update!(grouped_notifications, group_key, &[notification | &1])} + else + {[group_key | group_keys], Map.put(grouped_notifications, group_key, [notification])} + end + end) + + group_keys + |> Enum.reverse() + |> Enum.map(fn group_key -> + grouped_notifications + |> Map.fetch!(group_key) + |> Enum.reverse() + end) + end + + def group_key(notification, grouped_types \\ nil) + + def group_key(%Notification{} = notification, nil) do + group_key(notification, @groupable_notification_types) + end + + def group_key(%Notification{type: type, group_key: group_key} = notification, grouped_types) do + grouped_types = normalize_grouped_types(grouped_types) + + if type in @groupable_notification_types and type in grouped_types and is_binary(group_key) do + group_key + else + ungrouped_group_key(notification) + end + end + + def normalize_grouped_types(nil), do: @groupable_notification_types + def normalize_grouped_types(types) when is_list(types), do: types + def normalize_grouped_types(type), do: [type] + + defp ungrouped_group_key(%Notification{id: id}), do: "ungrouped-#{id}" + + defp grouped_notification_key(type, activity) when type in @groupable_notification_types do + with target_id when is_binary(target_id) <- group_target_id(type, activity) do + # Mastodon uses Redis to reuse a recent bucket for rolling 12h windows. Pleroma keeps keys + # deterministic and self-contained; clients must treat group_key as opaque either way. + "#{type}-#{target_id}-#{group_time_bucket(activity)}" + end + end + + defp grouped_notification_key(_type, _activity), do: nil + + defp group_time_bucket(%Activity{inserted_at: inserted_at}) when not is_nil(inserted_at) do + inserted_at + |> NaiveDateTime.to_erl() + |> :calendar.datetime_to_gregorian_seconds() + |> div(@group_bucket_seconds) + end + + defp group_time_bucket(_), + do: group_time_bucket(%Activity{inserted_at: NaiveDateTime.utc_now()}) + + defp group_target_id(type, activity) when type in ["favourite", "reblog"] do + with object_id when is_binary(object_id) <- object_id_for(activity), + %Activity{id: id} <- Activity.get_create_by_object_ap_id(object_id) do + to_string(id) + else + _ -> nil + end + end + + defp group_target_id("follow", %{data: %{"object" => ap_id}}) do + case User.get_cached_by_ap_id(ap_id) do + %User{id: id} -> to_string(id) + _ -> nil + end + end + + defp group_target_id(_, _), do: nil + + defp object_id_for(%{data: %{"object" => %{"id" => id}}}) when is_binary(id), do: id + defp object_id_for(%{data: %{"object" => id}}) when is_binary(id), do: id + defp object_id_for(_), do: nil + @notification_types ~w{ favourite follow @@ -79,7 +173,7 @@ defmodule Pleroma.Notification do def changeset(%Notification{} = notification, attrs) do notification - |> cast(attrs, [:seen, :type]) + |> cast(attrs, [:seen, :type, :group_key]) |> validate_inclusion(:type, @notification_types) end @@ -458,7 +552,8 @@ defmodule Pleroma.Notification do user_id: user.id, activity: activity, seen: mark_as_read?(activity, user), - type: type + type: type, + group_key: grouped_notification_key(type, activity) }) |> Marker.multi_set_last_read_id(user, "notifications") |> Repo.transaction() diff --git a/lib/pleroma/object/containment.ex b/lib/pleroma/object/containment.ex index 77fac12c0..c4cc1f014 100644 --- a/lib/pleroma/object/containment.ex +++ b/lib/pleroma/object/containment.ex @@ -15,12 +15,28 @@ defmodule Pleroma.Object.Containment do actor end + def get_actor(%{"actor" => nil, "attributedTo" => actor}) when not is_nil(actor) do + get_actor(%{"actor" => actor}) + end + + def get_actor(%{"actor" => nil}), do: nil + + def get_actor(%{"actor" => []}), do: nil + def get_actor(%{"actor" => actor}) when is_list(actor) do - if is_binary(Enum.at(actor, 0)) do - Enum.at(actor, 0) - else - Enum.find(actor, fn %{"type" => type} -> type in ["Person", "Service", "Application"] end) - |> Map.get("id") + case actor do + [first | _] when is_binary(first) -> + first + + _ -> + Enum.find_value(actor, fn + %{"type" => type, "id" => id} + when type in ["Person", "Service", "Application"] and is_binary(id) -> + id + + _ -> + nil + end) end end @@ -28,10 +44,6 @@ defmodule Pleroma.Object.Containment do id end - def get_actor(%{"actor" => nil, "attributedTo" => actor}) when not is_nil(actor) do - get_actor(%{"actor" => actor}) - end - def get_object(%{"object" => id}) when is_binary(id) do id end @@ -67,9 +79,11 @@ defmodule Pleroma.Object.Containment do def contain_origin(id, %{"actor" => _actor} = params) do id_uri = URI.parse(id) - actor_uri = URI.parse(get_actor(params)) - compare_uris(actor_uri, id_uri) + case get_actor(params) do + actor when is_binary(actor) -> compare_uris(URI.parse(actor), id_uri) + _ -> :error + end end def contain_origin(id, %{"attributedTo" => actor} = params), diff --git a/lib/pleroma/reverse_proxy.ex b/lib/pleroma/reverse_proxy.ex index c7ee47c6e..77ab95964 100644 --- a/lib/pleroma/reverse_proxy.ex +++ b/lib/pleroma/reverse_proxy.ex @@ -12,7 +12,7 @@ defmodule Pleroma.ReverseProxy do @keep_resp_headers @resp_cache_headers ++ ~w(content-length content-type content-disposition content-encoding) ++ ~w(content-range accept-ranges vary) - @default_cache_control_header "public, max-age=1209600, immutable" + @default_cache_control_header "public, max-age=1209600, immutable, no-transform" @valid_resp_codes [200, 206, 304] @max_read_duration :timer.seconds(30) @max_body_length :infinity @@ -23,6 +23,16 @@ defmodule Pleroma.ReverseProxy do @cachex Pleroma.Config.get([:cachex, :provider], Cachex) + defmodule StreamError do + @moduledoc false + defexception [:url, :reason] + + @impl true + def message(%{url: url, reason: reason}) do + "reverse proxy stream from #{url} failed: #{inspect(reason)}" + end + end + def max_read_duration_default, do: @max_read_duration def default_cache_control_header, do: @default_cache_control_header @@ -69,11 +79,15 @@ defmodule Pleroma.ReverseProxy do @default_options [pool: :media] @inline_content_types [ + "image/apng", + "image/avif", + "image/bmp", "image/gif", "image/jpeg", "image/jpg", "image/png", "image/svg+xml", + "image/webp", "audio/mpeg", "audio/mp3", "video/webm", @@ -98,7 +112,13 @@ defmodule Pleroma.ReverseProxy do def call(_conn, _url, _opts \\ []) def call(conn = %{method: method}, url, opts) when method in @methods do - client_opts = Keyword.merge(@default_options, Keyword.get(opts, :http, [])) + client_opts = + @default_options + |> Keyword.merge(Keyword.get(opts, :http, [])) + |> Keyword.put_new( + :recv_timeout, + receive_timeout(opts) + ) req_headers = build_req_headers(conn.req_headers, opts) @@ -110,12 +130,8 @@ defmodule Pleroma.ReverseProxy do end with {:ok, nil} <- @cachex.get(:failed_proxy_url_cache, url), - {:ok, code, headers, client} <- request(method, url, req_headers, client_opts), - :ok <- - header_length_constraint( - headers, - Keyword.get(opts, :max_body_length, @max_body_length) - ) do + {:ok, code, headers, client} <- + request_with_constraints(method, url, req_headers, client_opts, opts) do response(conn, client, url, code, headers, opts) else {:ok, true} -> @@ -170,7 +186,8 @@ defmodule Pleroma.ReverseProxy do {:ok, code, headers} when code in @valid_resp_codes -> {:ok, code, downcase_headers(headers)} - {:ok, code, _, _} -> + {:ok, code, _, client} -> + client().close(client) {:error, {:invalid_http_response, code}} {:ok, code, _} -> @@ -181,13 +198,38 @@ defmodule Pleroma.ReverseProxy do end end + defp request_with_constraints(method, url, headers, client_opts, opts) do + method + |> request(url, headers, client_opts) + |> constrain_response(Keyword.get(opts, :max_body_length, @max_body_length)) + end + + defp constrain_response({:ok, code, headers, client}, limit) do + case header_length_constraint(headers, limit) do + {:ok, headers} -> + {:ok, code, headers, client} + + error -> + client().close(client) + error + end + end + + defp constrain_response({:ok, code, headers}, limit) do + case header_length_constraint(headers, limit) do + {:ok, headers} -> {:ok, code, headers} + error -> error + end + end + + defp constrain_response(response, _limit), do: response + defp response(conn, client, url, status, headers, opts) do Logger.debug("#{__MODULE__} #{status} #{url} #{inspect(headers)}") result = conn |> put_resp_headers(build_resp_headers(headers, opts)) - |> streaming_compat |> send_chunked(status) |> chunk_reply(client, opts) @@ -195,17 +237,18 @@ defmodule Pleroma.ReverseProxy do {:ok, conn} -> halt(conn) - {:error, :closed, conn} -> + {:error, {:downstream, _error}, conn} -> client().close(client) halt(conn) - {:error, error, conn} -> + {:error, {:upstream, error}, conn} -> Logger.warning( "#{__MODULE__} request to #{url} failed while reading/chunking: #{inspect(error)}" ) client().close(client) - halt(conn) + track_failed_url(url, error, opts) + raise_stream_error(conn, url, error) end end @@ -227,11 +270,19 @@ defmodule Pleroma.ReverseProxy do sent_so_far, Keyword.get(opts, :max_body_length, @max_body_length) ), - {:ok, conn} <- chunk(conn, data) do + {:ok, conn} <- write_chunk(conn, data) do chunk_reply(conn, client, opts, sent_so_far, duration) else :done -> {:ok, conn} - {:error, error} -> {:error, error, conn} + {:error, {source, error}} -> {:error, {source, error}, conn} + {:error, error} -> {:error, {:upstream, error}, conn} + end + end + + defp write_chunk(conn, data) do + case chunk(conn, data) do + {:error, error} -> {:error, {:downstream, error}} + result -> result end end @@ -281,7 +332,7 @@ defmodule Pleroma.ReverseProxy do |> Enum.filter(fn {k, _} -> k in @keep_req_headers end) |> build_req_range_or_encoding_header(opts) |> build_req_user_agent_header(opts) - |> Keyword.merge(Keyword.get(opts, :req_headers, [])) + |> merge_headers(Keyword.get(opts, :req_headers, [])) end # Disable content-encoding if any @range_headers are requested (see #1823). @@ -310,7 +361,30 @@ defmodule Pleroma.ReverseProxy do |> build_resp_cache_headers(opts) |> sanitise_content_type() |> build_resp_content_disposition_header(opts) - |> Keyword.merge(Keyword.get(opts, :resp_headers, [])) + |> merge_headers(Keyword.get(opts, :resp_headers, [])) + |> ensure_no_transform() + end + + defp merge_headers(headers, extra_headers) do + Enum.reduce(extra_headers, headers, fn {key, value}, headers -> + List.keystore(headers, String.downcase(key), 0, {String.downcase(key), value}) + end) + end + + defp ensure_no_transform(headers) do + {_, cache_control} = + List.keyfind(headers, "cache-control", 0, {"cache-control", @default_cache_control_header}) + + directives = + cache_control + |> String.split(",") + |> Enum.map(&(&1 |> String.trim() |> String.downcase())) + + if "no-transform" in directives do + headers + else + replace_header(headers, "cache-control", cache_control <> ", no-transform") + end end defp sanitise_content_type(headers) do @@ -384,29 +458,46 @@ defmodule Pleroma.ReverseProxy do disposition = "attachment; filename=\"#{name}\"" - List.keystore(headers, "content-disposition", 0, {"content-disposition", disposition}) + replace_header(headers, "content-disposition", disposition) else headers end end - defp header_length_constraint(headers, limit) when is_integer(limit) and limit > 0 do - with {_, size} <- List.keyfind(headers, "content-length", 0), - {size, _} <- Integer.parse(size), - true <- size <= limit do - :ok - else - false -> - {:error, :body_too_large} + defp replace_header(headers, key, value) do + [{key, value} | Enum.reject(headers, fn {header, _} -> header == key end)] + end + + defp header_length_constraint(headers, limit) do + lengths = + for {"content-length", value} <- headers, + do: parse_content_length(value) + + case Enum.uniq(lengths) do + [] -> + {:ok, headers} + + [size] when is_integer(size) -> + case body_size_constraint(size, limit) do + :ok -> {:ok, replace_header(headers, "content-length", to_string(size))} + error -> error + end _ -> - :ok + {:error, :invalid_content_length} end end - defp header_length_constraint(_, _), do: :ok + defp parse_content_length(value) when is_binary(value) do + case Integer.parse(value) do + {size, ""} when size >= 0 -> size + _ -> :invalid + end + end - defp body_size_constraint(size, limit) when is_integer(limit) and limit > 0 and size >= limit do + defp parse_content_length(_value), do: :invalid + + defp body_size_constraint(size, limit) when is_integer(limit) and limit > 0 and size > limit do {:error, :body_too_large} end @@ -421,7 +512,16 @@ defmodule Pleroma.ReverseProxy do end end - defp check_read_duration(_, _), do: {:ok, :no_duration_limit, :no_duration_limit} + defp check_read_duration(_, _), do: {:ok, :no_duration_limit} + + defp receive_timeout(opts) do + case Keyword.get(opts, :max_read_duration, @max_read_duration) do + timeout when is_integer(timeout) and timeout > 0 -> timeout + _ -> :infinity + end + end + + defp increase_read_duration(:no_duration_limit), do: {:ok, :no_duration_limit} defp increase_read_duration({previous_duration, started}) when is_integer(previous_duration) and is_integer(started) do @@ -431,6 +531,40 @@ defmodule Pleroma.ReverseProxy do defp client, do: Pleroma.ReverseProxy.Client.Wrapper + # Neither Plug adapter exposes a public way to abort a committed response. + # Cowboy finalizes chunked HTTP/1 responses when the request process exits, + # so terminate the connection first to leave the response visibly incomplete. + defp raise_stream_error( + %Plug.Conn{adapter: {Plug.Cowboy.Conn, %{pid: connection_pid}}} = conn, + url, + error + ) do + if Plug.Conn.get_http_protocol(conn) in [:"HTTP/1.0", :"HTTP/1.1"] do + Process.exit(connection_pid, :kill) + end + + raise StreamError, url: url, reason: error + end + + # Bandit turns this exception into RST_STREAM; a regular exception sends an + # empty DATA frame with END_STREAM and makes the partial body look complete. + defp raise_stream_error( + %Plug.Conn{ + adapter: {Bandit.Adapter, %{transport: %{stream_id: stream_id}}} + }, + url, + error + ) do + raise Bandit.HTTP2.Errors.StreamError, + message: StreamError.message(%StreamError{url: url, reason: error}), + error_code: Bandit.HTTP2.Errors.internal_error(), + stream_id: stream_id + end + + defp raise_stream_error(_conn, url, error) do + raise StreamError, url: url, reason: error + end + defp track_failed_url(url, error, opts) do ttl = unless error in [:body_too_large, 400, 204] do @@ -442,19 +576,6 @@ defmodule Pleroma.ReverseProxy do @cachex.put(:failed_proxy_url_cache, url, true, ttl: ttl) end - # When Cowboy handles a chunked response with a content-length header it streams - # over HTTP 1.1 instead of chunking. Bandit cannot stream over HTTP 1.1 so the header - # must be stripped or it breaks RFC compliance for Transfer Encoding: Chunked. RFC9112§6.2 - # - # HTTP2 is always streamed for all adapters. - defp streaming_compat(conn) do - with Phoenix.Endpoint.Cowboy2Adapter <- Pleroma.Web.Endpoint.config(:adapter) do - conn - else - _ -> delete_resp_header(conn, "content-length") - end - end - # Only when Tesla adapter is Hackney or Finch does the URL # need encoding before Reverse Proxying as both end up # using the raw Hackney client and cannot leverage our diff --git a/lib/pleroma/reverse_proxy/client/hackney.ex b/lib/pleroma/reverse_proxy/client/hackney.ex index 4c81a6225..b4d2b5e01 100644 --- a/lib/pleroma/reverse_proxy/client/hackney.ex +++ b/lib/pleroma/reverse_proxy/client/hackney.ex @@ -28,17 +28,18 @@ defmodule Pleroma.ReverseProxy.Client.Hackney do @impl true def request(method, url, headers, body, opts \\ []) do opts = - Keyword.put_new(opts, :path_encode_fun, fn path -> - path - end) + opts + |> Keyword.put_new(:path_encode_fun, fn path -> path end) + |> Pleroma.HTTP.TLS.put_hackney_ssl_options() if opts[:follow_redirect] != false do {_state, req_opts} = Access.get_and_update(opts, :follow_redirect, fn a -> {a, false} end) env = %{method: method, headers: headers, body: body, req_opts: req_opts} - res = :hackney.request(method, url, headers, body, req_opts) + res = do_request(method, url, headers, body, req_opts) case res do - {:ok, code, resp_headers, _client} when code in @redirect_statuses -> + {:ok, code, resp_headers, client} when code in @redirect_statuses -> + close(client) redirect(url, resp_headers, env, @redirect_limit) {:ok, code, resp_headers} when code in @redirect_statuses -> @@ -48,11 +49,34 @@ defmodule Pleroma.ReverseProxy.Client.Hackney do res end else - :hackney.request(method, url, headers, body, opts) + do_request(method, url, headers, body, opts) end end @impl true + def stream_body(%{released: true}), do: :done + + def stream_body(%{ref: ref, recv_timeout: recv_timeout} = client) do + _ = :hackney.stream_next(ref) + + case receive_response(ref, recv_timeout) do + {:ok, data} when is_binary(data) -> + {:ok, data, client} + + # Reaching the end of the body does not free the connection: Hackney parks + # a finished conn instead of stopping it, and an unpooled one has neither + # an idle timeout nor a live owner monitor to reap it, so it would sit + # there forever pinning every binary it read. Release it here, mirroring + # what the gun client does, so callers only close what they abandon. + {:ok, :done} -> + close(client) + :done + + {:error, error} -> + {:error, error} + end + end + def stream_body(ref) do case :hackney.stream_body(ref) do :done -> :done @@ -62,8 +86,98 @@ defmodule Pleroma.ReverseProxy.Client.Hackney do end @impl true + def close(%{released: true}), do: :ok + + def close(%{ref: ref}), do: :hackney.close(ref) + def close(ref), do: :hackney.close(ref) + defp do_request(method, url, headers, body, opts) do + recv_timeout = Keyword.get(opts, :recv_timeout, :infinity) + + req_opts = + opts + |> Keyword.put(:async, :once) + |> Keyword.put(:stream_to, self()) + + with {:ok, ref} <- :hackney.request(method, url, headers, body, req_opts) do + result = + with {:ok, {:status, code, _reason}} <- receive_response(ref, recv_timeout), + {:ok, {:headers, resp_headers}} <- receive_response(ref, recv_timeout) do + {:ok, code, resp_headers, build_client(ref, recv_timeout, method, code, opts)} + end + + case result do + {:ok, _, _, _} = response -> + response + + error -> + close(ref) + flush_responses(ref) + error + end + end + end + + # A response that cannot carry a body (204, 304, or a reply to HEAD) is + # complete as soon as the headers are parsed. Hackney queues `done` for us and, + # if the connection is pooled, checks it straight back in, so the ref stops + # being ours at that point: pulling the stream or stopping the connection + # afterwards would disturb whichever request checks it out next. Mark it so + # `stream_body/1` and `close/1` leave it alone. + # + # This relies on hackney never running its own redirect handling for us -- + # `request/5` forces `follow_redirect: false` -- otherwise a redirect status + # would be checked back in too and would need the same treatment. + defp build_client(ref, recv_timeout, method, code, opts) do + client = %{ref: ref, recv_timeout: recv_timeout} + + if released?(method, code) and pooled?(opts) do + flush_responses(ref) + Map.put(client, :released, true) + else + client + end + end + + defp released?(method, _code) when method in [:head, :HEAD, "head", "HEAD"], do: true + defp released?(_method, code) when code in [204, 304], do: true + defp released?(_method, _code), do: false + + # Mirrors hackney's own pooling decision: :hackney.use_pool/1, plus the fact + # that a proxy tunnel always bypasses the pool. An unpooled connection is never + # checked in, and parks in `connected` with no idle timeout, so it leaks + # forever unless we stop it ourselves. + defp pooled?(opts) do + no_proxy?(Keyword.get(opts, :proxy)) and use_pool?(Keyword.get(opts, :pool)) + end + + # Hackney reads `false` as "no proxy", same as an absent option. + defp no_proxy?(nil), do: true + defp no_proxy?(false), do: true + defp no_proxy?(_proxy), do: false + + defp use_pool?(false), do: false + defp use_pool?(nil), do: Application.get_env(:hackney, :use_default_pool, true) + defp use_pool?(_pool_name), do: true + + defp receive_response(ref, recv_timeout) do + receive do + {:hackney_response, ^ref, {:error, error}} -> {:error, error} + {:hackney_response, ^ref, response} -> {:ok, response} + after + recv_timeout -> {:error, :timeout} + end + end + + defp flush_responses(ref) do + receive do + {:hackney_response, ^ref, _response} -> flush_responses(ref) + after + 0 -> :ok + end + end + defp redirect(url, resp_headers, env, limit) when limit == 0 do new_url = absolute_redirect_url(url, resp_headers) @@ -71,17 +185,18 @@ defmodule Pleroma.ReverseProxy.Client.Hackney do "#{__MODULE__}: Handling redirect #{url} -> #{new_url}; redirect limit was reached - returning response after final redirect" ) - :hackney.request(env.method, new_url, env.headers, env.body, env.req_opts) + do_request(env.method, new_url, env.headers, env.body, env.req_opts) end defp redirect(url, resp_headers, env, limit) do new_url = absolute_redirect_url(url, resp_headers) Logger.debug("#{__MODULE__}: handling redirect #{url} -> #{new_url}; limit = #{limit}") - res = :hackney.request(env.method, new_url, env.headers, env.body, env.req_opts) + res = do_request(env.method, new_url, env.headers, env.body, env.req_opts) case res do - {:ok, code, new_resp_headers, _client} when code in @redirect_statuses -> + {:ok, code, new_resp_headers, client} when code in @redirect_statuses -> + close(client) redirect(new_url, new_resp_headers, env, limit - 1) {:ok, code, new_resp_headers} when code in @redirect_statuses -> diff --git a/lib/pleroma/reverse_proxy/client/tesla.ex b/lib/pleroma/reverse_proxy/client/tesla.ex index 4596d7a7f..941d23ea2 100644 --- a/lib/pleroma/reverse_proxy/client/tesla.ex +++ b/lib/pleroma/reverse_proxy/client/tesla.ex @@ -6,6 +6,7 @@ defmodule Pleroma.ReverseProxy.Client.Tesla do @behaviour Pleroma.ReverseProxy.Client alias Pleroma.Gun.ConnectionPool + alias Pleroma.HTTP.AdapterHelper @type headers() :: [{String.t(), String.t()}] @type status() :: pos_integer() @@ -20,6 +21,13 @@ defmodule Pleroma.ReverseProxy.Client.Tesla do def request(method, url, headers, body, opts \\ []) do check_adapter() + # MediaProxyController inserts MediaProxy proxy configuration into the ReverseProxy call. + # This config can be unformatted and in String representation, while this works for Hackney, + # it does not work for Tesla and the proxy needs to be reformatted. + # Gun AdapterHelper doesn't do it, since it does not overwrite proxy config. + proxy = AdapterHelper.format_proxy(opts[:proxy]) + opts = if proxy, do: Keyword.put(opts, :proxy, proxy), else: opts + opts = Keyword.put(opts, :body_as, :chunks) with {:ok, response} <- @@ -45,8 +53,8 @@ defmodule Pleroma.ReverseProxy.Client.Tesla do @impl true @spec stream_body(map()) :: {:ok, binary(), map()} | {:error, atom() | String.t()} | :done | no_return() - def stream_body(%{pid: pid, fin: true}) do - ConnectionPool.release_conn(pid) + def stream_body(%{pid: pid, stream: stream, fin: true}) do + ConnectionPool.release_stream(pid, stream) :done end @@ -70,6 +78,10 @@ defmodule Pleroma.ReverseProxy.Client.Tesla do @impl true @spec close(map) :: :ok | no_return() + def close(%{pid: pid, stream: stream}) do + ConnectionPool.cancel_stream(pid, stream) + end + def close(%{pid: pid}) do ConnectionPool.release_conn(pid) end diff --git a/lib/pleroma/search.ex b/lib/pleroma/search.ex index 9cd2768c4..0d9c7c6e3 100644 --- a/lib/pleroma/search.ex +++ b/lib/pleroma/search.ex @@ -5,13 +5,12 @@ defmodule Pleroma.Search do alias Pleroma.Workers.SearchIndexingWorker @spec add_to_index(Activity.t()) :: {:ok, Oban.Job.t() | :noop} | {:error, Oban.Job.changeset()} - def add_to_index(%Activity{id: activity_id, object: %Object{} = object} = activity) do - with {_, true} <- {:indexable, indexable?(activity)}, - {_, "public"} <- {:visibility, Visibility.get_visibility(object)} do + def add_to_index(%Activity{id: activity_id, object: %Object{}} = activity) do + if indexable?(activity) do SearchIndexingWorker.new(%{"op" => "add_to_index", "activity" => activity_id}) |> Oban.insert() else - _ -> {:ok, :noop} + {:ok, :noop} end end @@ -41,40 +40,60 @@ defmodule Pleroma.Search do def object_to_search_data(%Object{} = object) do data = object.data - content_str = - case data["content"] do - [nil | rest] -> to_string(rest) - str -> str - end - content = - with {:ok, scrubbed} <- - FastSanitize.Sanitizer.scrub(content_str, Pleroma.HTML.Scrubber.SearchIndexing), - trimmed <- String.trim(scrubbed) do - trimmed - end - - # Make sure we have a non-empty string - if content != "" do - {:ok, published, _} = DateTime.from_iso8601(data["published"]) + data + |> search_texts() + |> Enum.map(&sanitize_search_text/1) + |> Enum.reject(&(&1 in ["", "."])) + |> Enum.join(" ") + with true <- content != "", + published when is_binary(published) <- data["published"], + {:ok, published, _} <- DateTime.from_iso8601(published) do %{ id: object.id, content: content, ap: data["id"], published: published |> DateTime.to_unix() } + else + _ -> nil end end - defp indexable?(%Activity{ - data: %{"type" => "Create"}, - object: %Object{ - data: %{"content" => content, "published" => published, "type" => "Note"} - } - }) - when not is_nil(content) and content not in ["", "."] and not is_nil(published), - do: true + def indexable?(%Activity{ + data: %{"type" => "Create"}, + object: %Object{data: %{"published" => published, "type" => "Note"}} = object + }) + when not is_nil(published) do + Visibility.get_visibility(object) in ["public", "unlisted"] and + not is_nil(object_to_search_data(object)) + end - defp indexable?(_), do: false + def indexable?(_), do: false + + defp search_texts(data) do + [data["content"], data["summary"] | attachment_names(data["attachment"])] + end + + defp attachment_names(attachments) when is_list(attachments) do + Enum.map(attachments, fn + %{"summary" => summary} when is_binary(summary) -> summary + %{"name" => name} -> name + _ -> nil + end) + end + + defp attachment_names(_), do: [] + + defp sanitize_search_text([nil | rest]), do: sanitize_search_text(to_string(rest)) + + defp sanitize_search_text(text) when is_binary(text) do + case FastSanitize.Sanitizer.scrub(text, Pleroma.HTML.Scrubber.SearchIndexing) do + {:ok, scrubbed} -> String.trim(scrubbed) + _ -> "" + end + end + + defp sanitize_search_text(_), do: "" end diff --git a/lib/pleroma/search/backfill.ex b/lib/pleroma/search/backfill.ex new file mode 100644 index 000000000..1b986bb59 --- /dev/null +++ b/lib/pleroma/search/backfill.ex @@ -0,0 +1,137 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2026 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Search.Backfill do + import Ecto.Query + + alias Pleroma.Activity + alias Pleroma.Workers.SearchIndexingWorker + + @default_chunk_size 100 + @default_step_size 1_000 + @default_limit 100_000 + + @doc """ + Enqueues existing Create activities for search indexing. + + The returned `next_cursor` can be passed as `before` to resume a bounded run. + Pass `limit: :infinity` to continue until the oldest activity is reached. + """ + def run(options \\ []) do + options = + Keyword.validate!(options, + chunk: @default_chunk_size, + step: @default_step_size, + limit: @default_limit, + before: nil, + on_page: nil + ) + + chunk_size = positive_option!(options, :chunk, @default_chunk_size) + step_size = positive_option!(options, :step, @default_step_size) + limit = limit_option!(options) + before = cursor_option!(options[:before]) + on_page = callback_option!(options[:on_page]) + + do_run(before, limit, step_size, chunk_size, on_page, 0) + end + + defp do_run(before, remaining, step_size, chunk_size, on_page, enqueued) do + page_size = page_size(remaining, step_size) + ids = fetch_ids(before, page_size) + + case ids do + [] -> + result(enqueued, before, true) + + ids -> + enqueue(ids, chunk_size) + + page_count = length(ids) + next_cursor = List.last(ids) + total = enqueued + page_count + remaining = decrement(remaining, page_count) + + on_page.(%{count: page_count, enqueued: total, next_cursor: next_cursor}) + + cond do + remaining == 0 -> result(total, next_cursor, false) + page_count < page_size -> result(total, next_cursor, true) + true -> do_run(next_cursor, remaining, step_size, chunk_size, on_page, total) + end + end + end + + defp fetch_ids(before, limit) do + Activity + |> where([a], fragment("?->>'type' = 'Create'", a.data)) + |> maybe_before(before) + |> order_by([a], desc: a.id) + |> limit(^limit) + |> select([a], a.id) + |> Pleroma.Repo.all() + end + + defp maybe_before(query, nil), do: query + defp maybe_before(query, before), do: where(query, [a], a.id < ^before) + + defp enqueue(ids, chunk_size) do + ids + |> Enum.chunk_every(chunk_size) + |> Enum.each(fn chunk -> + chunk + |> Enum.map(&SearchIndexingWorker.new(%{"op" => "add_to_index", "activity" => &1})) + |> Oban.insert_all() + end) + end + + defp page_size(:infinity, step_size), do: step_size + defp page_size(remaining, step_size), do: min(remaining, step_size) + + defp decrement(:infinity, _count), do: :infinity + defp decrement(remaining, count), do: remaining - count + + defp result(enqueued, next_cursor, exhausted) do + %{enqueued: enqueued, next_cursor: next_cursor, exhausted: exhausted} + end + + defp positive_option!(options, key, default) do + case Keyword.get(options, key, default) do + value when is_integer(value) and value > 0 -> value + value -> raise ArgumentError, "#{key} must be a positive integer, got: #{inspect(value)}" + end + end + + defp limit_option!(options) do + case Keyword.fetch!(options, :limit) do + :infinity -> :infinity + value when is_integer(value) and value > 0 -> value + value -> raise ArgumentError, "limit must be a positive integer, got: #{inspect(value)}" + end + end + + defp cursor_option!(nil), do: nil + + defp cursor_option!(cursor) do + valid? = + try do + match?({:ok, <<_::binary-size(16)>>}, FlakeId.Ecto.CompatType.dump(cursor)) + rescue + _ -> false + end + + if valid? do + cursor + else + raise ArgumentError, "before must be a valid activity id, got: #{inspect(cursor)}" + end + end + + defp callback_option!(nil), do: fn _ -> :ok end + defp callback_option!(callback) when is_function(callback, 1), do: callback + + defp callback_option!(callback) do + raise ArgumentError, "on_page must be a function of arity 1, got: #{inspect(callback)}" + end +end diff --git a/lib/pleroma/search/parade_db.ex b/lib/pleroma/search/parade_db.ex new file mode 100644 index 000000000..ee00e74af --- /dev/null +++ b/lib/pleroma/search/parade_db.ex @@ -0,0 +1,287 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Search.ParadeDB do + @behaviour Pleroma.Search.SearchBackend + + require Logger + require Pleroma.Constants + + import Ecto.Query + + import Pleroma.Search.DatabaseSearch, + only: [ + maybe_fetch: 3, + maybe_restrict_author: 2, + maybe_restrict_blocked: 2, + maybe_restrict_local: 2 + ] + + alias Pleroma.Activity + alias Pleroma.Config.Getting, as: Config + alias Pleroma.Search + + @max_limit 40 + @default_table "pleroma_search_documents" + + defp client_impl do + Config.get([Pleroma.Search.ParadeDB, :client_impl]) || + Pleroma.Search.ParadeDB.Client.Postgres + end + + defp table do + Config.get([Pleroma.Search.ParadeDB, :table], @default_table) + end + + defp fuzzy_distance do + case Config.get([Pleroma.Search.ParadeDB, :fuzzy_distance], 0) do + value when is_integer(value) and value in 0..2 -> + value + + value when is_binary(value) -> + case Integer.parse(value) do + {parsed, ""} when parsed in 0..2 -> parsed + _ -> 0 + end + + _ -> + 0 + end + end + + @impl true + def create_index do + with {:ok, _} <- client_impl().query("CREATE EXTENSION IF NOT EXISTS pg_search", []), + {:ok, _} <- client_impl().query(create_table_sql(table()), []), + {:ok, _} <- client_impl().query(create_object_id_index_sql(table()), []), + {:ok, _} <- client_impl().query(create_bm25_index_sql(table()), []) do + :ok + else + {:error, error} -> + Logger.error("ParadeDB create_index failed: #{inspect(error)}") + {:error, error} + end + end + + @impl true + def drop_index do + case client_impl().query("DROP TABLE IF EXISTS #{table()}", []) do + {:ok, _} -> + :ok + + {:error, error} -> + Logger.error("ParadeDB drop_index failed: #{inspect(error)}") + {:error, error} + end + end + + @impl true + def healthcheck_endpoints, do: nil + + @impl true + def add_to_index(activity) do + with true <- Search.indexable?(activity), + maybe_search_data when not is_nil(maybe_search_data) <- + Search.object_to_search_data(activity.object) do + dumped_activity_id = dump_activity_id(activity.id) + actor_ap_id = activity.data["actor"] + published_at = DateTime.from_unix!(maybe_search_data.published) + + if is_nil(dumped_activity_id) do + Logger.error("ParadeDB add_to_index failed: invalid activity id #{inspect(activity.id)}") + :ok + else + sql = + """ + INSERT INTO #{table()} (id, object_id, object_ap_id, actor_ap_id, content, published_at) + VALUES ($1, $2, $3, $4, $5, $6) + ON CONFLICT (id) DO UPDATE SET + object_id = EXCLUDED.object_id, + object_ap_id = EXCLUDED.object_ap_id, + actor_ap_id = EXCLUDED.actor_ap_id, + content = EXCLUDED.content, + published_at = EXCLUDED.published_at + """ + + params = [ + dumped_activity_id, + maybe_search_data.id, + maybe_search_data.ap, + actor_ap_id, + maybe_search_data.content, + published_at + ] + + case client_impl().query(sql, params) do + {:ok, _} -> + :ok + + {:error, error} -> + Logger.error("ParadeDB add_to_index failed: #{inspect(error)}") + {:error, error} + end + end + else + _ -> :ok + end + end + + @impl true + def remove_from_index(object) do + sql = "DELETE FROM #{table()} WHERE object_id = $1" + + case client_impl().query(sql, [object.id]) do + {:ok, _} -> + :ok + + {:error, error} -> + Logger.error("ParadeDB remove_from_index failed: #{inspect(error)}") + {:error, error} + end + end + + @impl true + def search(user, query, options \\ []) do + limit = options |> Keyword.get(:limit, @max_limit) |> min(@max_limit) + offset = Keyword.get(options, :offset, 0) + author = Keyword.get(options, :author) + + case search_ids(query, limit, offset, author) do + {:ok, []} -> + maybe_fetch([], user, query) + + {:ok, ids} -> + ids + |> fetch_activities(user, query, author) + + {:error, error} -> + Logger.error("ParadeDB search failed: #{inspect(error)}") + maybe_fetch([], user, query) + end + end + + defp fetch_activities(ids, user, query, author) do + dumped_ids = + ids + |> Enum.map(&dump_activity_id/1) + |> Enum.reject(&is_nil/1) + + from(a in Activity, where: a.id in ^dumped_ids) + |> Activity.with_preloaded_object() + |> Activity.restrict_deactivated_users() + |> maybe_restrict_public(user) + |> maybe_restrict_local(user) + |> maybe_restrict_author(author) + |> maybe_restrict_blocked(user) + |> order_by( + [a], + fragment( + "array_position(?, ?)", + ^dumped_ids, + a.id + ) + ) + |> Pleroma.Repo.all() + |> maybe_fetch(user, query) + end + + defp maybe_restrict_public(query, %Pleroma.User{}) do + intended_recipients = [ + Pleroma.Constants.as_public(), + Pleroma.Web.ActivityPub.Utils.as_local_public() + ] + + from(a in query, + where: fragment("?->>'type' = 'Create'", a.data), + where: fragment("? && ?", ^intended_recipients, a.recipients) + ) + end + + defp maybe_restrict_public(query, _user) do + from(a in query, + where: fragment("?->>'type' = 'Create'", a.data), + where: ^Pleroma.Constants.as_public() in a.recipients + ) + end + + defp search_ids(query, limit, offset, author) do + base_sql = + "SELECT id FROM #{table()} WHERE content &&& #{search_query_param_sql(fuzzy_distance())}" + + params = [query] + + {base_sql, params} = + case author do + %Pleroma.User{ap_id: ap_id} when is_binary(ap_id) -> + {base_sql <> " AND actor_ap_id = $2", params ++ [ap_id]} + + _ -> + {base_sql, params} + end + + limit_arg = length(params) + 1 + offset_arg = length(params) + 2 + + sql = + base_sql <> + " ORDER BY published_at DESC, id DESC LIMIT $#{limit_arg} OFFSET $#{offset_arg}" + + params = params ++ [limit, offset] + + with {:ok, %{rows: rows}} <- client_impl().query(sql, params) do + {:ok, Enum.map(rows, fn [id] -> id end)} + end + end + + defp search_query_param_sql(0), do: "$1" + defp search_query_param_sql(distance), do: "($1::text)::pdb.fuzzy(#{distance})" + + defp create_table_sql(table) do + """ + CREATE TABLE IF NOT EXISTS #{table} ( + id uuid PRIMARY KEY, + object_id bigint NOT NULL, + object_ap_id text NOT NULL, + actor_ap_id text, + content text NOT NULL, + published_at timestamptz + ) + """ + end + + defp create_object_id_index_sql(table) do + "CREATE INDEX IF NOT EXISTS #{table}_object_id_idx ON #{table} (object_id)" + end + + defp create_bm25_index_sql(table) do + """ + CREATE INDEX IF NOT EXISTS #{table}_bm25_idx ON #{table} + USING bm25 (id, content, (actor_ap_id::pdb.literal), published_at) + WITH (key_field = 'id') + """ + end + + defp dump_activity_id(<<_::binary-size(16)>> = uuid), do: uuid + + defp dump_activity_id(id) when is_binary(id) do + case Ecto.UUID.dump(id) do + {:ok, <<_::binary-size(16)>> = uuid} -> + uuid + + :error -> + dump_flake_activity_id(id) + end + end + + defp dump_activity_id(_), do: nil + + defp dump_flake_activity_id(id) do + case FlakeId.Ecto.CompatType.dump(id) do + {:ok, <<_::binary-size(16)>> = uuid} -> uuid + _ -> nil + end + rescue + _ -> nil + end +end diff --git a/lib/pleroma/search/parade_db/client.ex b/lib/pleroma/search/parade_db/client.ex new file mode 100644 index 000000000..d6dce0dd4 --- /dev/null +++ b/lib/pleroma/search/parade_db/client.ex @@ -0,0 +1,24 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Search.ParadeDB.Client do + @moduledoc false + + @type sql :: String.t() + @type params :: list() + @type result :: %{optional(:rows) => list(list())} + + @callback query(sql(), params()) :: {:ok, result()} | {:error, any()} +end + +defmodule Pleroma.Search.ParadeDB.Client.Postgres do + @moduledoc false + + @behaviour Pleroma.Search.ParadeDB.Client + + @impl true + def query(sql, params \\ []) do + Ecto.Adapters.SQL.query(Pleroma.Search.ParadeDB.Repo, sql, params) + end +end diff --git a/lib/pleroma/search/parade_db/repo.ex b/lib/pleroma/search/parade_db/repo.ex new file mode 100644 index 000000000..684271847 --- /dev/null +++ b/lib/pleroma/search/parade_db/repo.ex @@ -0,0 +1,24 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Search.ParadeDB.Repo do + use Ecto.Repo, + otp_app: :pleroma, + adapter: Ecto.Adapters.Postgres, + migration_timestamps: [type: :naive_datetime_usec] + + @doc """ + Loads the ParadeDB database URL from `PARADEDB_DATABASE_URL`. + + If the env var is not set, falls back to `config :pleroma, Pleroma.Search.ParadeDB, url: ...`. + """ + def init(_, opts) do + config = Application.get_env(:pleroma, Pleroma.Search.ParadeDB, []) + + url = System.get_env("PARADEDB_DATABASE_URL") || config[:url] + url = if is_nil(url), do: nil, else: to_string(url) + + {:ok, Keyword.put(opts, :url, url)} + end +end diff --git a/lib/pleroma/search/qdrant_search.ex b/lib/pleroma/search/qdrant_search.ex index 4d57cfa88..a8c5a3eb9 100644 --- a/lib/pleroma/search/qdrant_search.ex +++ b/lib/pleroma/search/qdrant_search.ex @@ -160,7 +160,7 @@ defmodule Pleroma.Search.QdrantSearch.OpenAIClient do end defp client do - Tesla.client(middleware()) + Tesla.client(middleware(), Pleroma.HTTP.AdapterHelper.tesla_adapter()) end defp middleware do @@ -191,7 +191,7 @@ defmodule Pleroma.Search.QdrantSearch.QdrantClient do end defp client do - Tesla.client(middleware()) + Tesla.client(middleware(), Pleroma.HTTP.AdapterHelper.tesla_adapter()) end defp middleware do diff --git a/lib/pleroma/telemetry/logger.ex b/lib/pleroma/telemetry/logger.ex index 31ce3cc20..e0522e869 100644 --- a/lib/pleroma/telemetry/logger.ex +++ b/lib/pleroma/telemetry/logger.ex @@ -55,8 +55,8 @@ defmodule Pleroma.Telemetry.Logger do def handle_event( [:pleroma, :connection_pool, :provision_failure], - %{opts: [key | _]}, _, + %{key: key}, _ ) do Logger.debug(fn -> diff --git a/lib/pleroma/tesla/middleware/connection_pool.ex b/lib/pleroma/tesla/middleware/connection_pool.ex index de74270f8..c2d8c5bb8 100644 --- a/lib/pleroma/tesla/middleware/connection_pool.ex +++ b/lib/pleroma/tesla/middleware/connection_pool.ex @@ -12,25 +12,38 @@ defmodule Pleroma.Tesla.Middleware.ConnectionPool do alias Pleroma.Gun.ConnectionPool @impl Tesla.Middleware - def call(%Tesla.Env{url: url, opts: opts} = env, next, _) do + def call(%Tesla.Env{opts: opts} = env, next, middleware_opts) do + if opts[:adapter][:body_as] == :stream do + {:error, :pooled_streaming_requires_chunks} + else + do_call(env, next, middleware_opts) + end + end + + defp do_call(%Tesla.Env{url: url, opts: opts} = env, next, _) do uri = URI.parse(url) # Avoid leaking connections when the middleware is called twice # with body_as: :chunks. We assume only the middleware can set # opts[:adapter][:conn] - if opts[:adapter][:conn] do - ConnectionPool.release_conn(opts[:adapter][:conn]) - end + cleanup_previous_request(env) case ConnectionPool.get_conn(uri, opts[:adapter]) do {:ok, conn_pid} -> - adapter_opts = Keyword.merge(opts[:adapter], conn: conn_pid, close_conn: false) + tunnel = ConnectionPool.tunnel_ref(conn_pid) + + adapter_opts = + opts[:adapter] + |> Keyword.drop([:conn, :stream, :tunnel]) + |> Keyword.merge(conn: conn_pid, close_conn: false, tunnel: tunnel) + opts = Keyword.put(opts, :adapter, adapter_opts) env = %{env | opts: opts} + next = use_pool_adapter(next) case Tesla.run(env, next) do {:ok, env} -> - unless opts[:adapter][:body_as] == :chunks do + unless opts[:adapter][:body_as] == :chunks and chunk_client?(env.body) do ConnectionPool.release_conn(conn_pid) {_, res} = pop_in(env.opts[:adapter][:conn]) {:ok, res} @@ -38,8 +51,11 @@ defmodule Pleroma.Tesla.Middleware.ConnectionPool do {:ok, env} end + {:error, {:gun_stream_error, reason}} -> + {:error, reason} + err -> - ConnectionPool.release_conn(conn_pid) + ConnectionPool.discard_conn(conn_pid) err end @@ -47,4 +63,34 @@ defmodule Pleroma.Tesla.Middleware.ConnectionPool do err end end + + defp cleanup_previous_request(%Tesla.Env{ + body: %{pid: pid, stream: stream}, + opts: opts + }) do + if opts[:adapter][:conn], do: ConnectionPool.cancel_stream(pid, stream) + end + + defp cleanup_previous_request(%Tesla.Env{opts: opts}) do + conn = opts[:adapter][:conn] + stream = opts[:adapter][:stream] + + cond do + conn && stream -> ConnectionPool.cancel_stream(conn, stream) + conn -> ConnectionPool.release_conn(conn) + true -> :ok + end + end + + defp chunk_client?(%{pid: pid, stream: stream}) when is_pid(pid) and not is_nil(stream), + do: true + + defp chunk_client?(_body), do: false + + defp use_pool_adapter(next) do + List.update_at(next, -1, fn + {Tesla.Adapter.Gun, function, args} -> {Pleroma.HTTP.Adapter.Gun, function, args} + adapter -> adapter + end) + end end diff --git a/lib/pleroma/tesla/middleware/max_body.ex b/lib/pleroma/tesla/middleware/max_body.ex new file mode 100644 index 000000000..7e74f0b56 --- /dev/null +++ b/lib/pleroma/tesla/middleware/max_body.ex @@ -0,0 +1,72 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Tesla.Middleware.MaxBody do + @moduledoc """ + Enforces the `:max_body` adapter option for every adapter. + + `Pleroma.HTTP.Adapter.Gun` applies the limit as it accumulates a `:plain` + body, so an oversized response is abandoned mid-transfer. Hackney cannot: + since 4.0 it ignores `with_body` and always hands back a fully read binary + (`hackney:request/5`), so `Tesla.Adapter.Hackney` never takes the branch that + would honour `:max_body`. Without this middleware the option is silently + unenforced there. + + The check therefore bounds what callers are handed, and refuses up front when + `content-length` already announces too much. It cannot bound peak memory on + the hackney path, because the adapter has read the body before any middleware + runs. + + Streaming bodies are left alone: neither adapter limits those, and the caller + that owns the stream is the one that has to drain and release it. + """ + + @behaviour Tesla.Middleware + + @impl Tesla.Middleware + def call(%Tesla.Env{opts: opts} = env, next, _opts) do + case opts[:adapter][:max_body] do + limit when is_integer(limit) and limit >= 0 -> run(env, next, limit) + _ -> Tesla.run(env, next) + end + end + + defp run(env, next, limit) do + with {:ok, env} <- Tesla.run(env, next), + :ok <- check(env, limit) do + {:ok, env} + end + end + + defp check(%Tesla.Env{body: body} = env, limit) when is_binary(body) do + with :ok <- check_content_length(env, limit) do + check_body(body, limit) + end + end + + # body_as: :chunks and response: :stream hand back a client the caller drains + # itself, applying its own limit (Pleroma.Web.RichMedia.Helpers). There is + # nothing to measure, and failing on the header alone would strand the pool + # lease that only the caller holds a handle to. + defp check(_env, _limit), do: :ok + + # A bodyless response may still announce the entity's size. + defp check_content_length(%Tesla.Env{method: :head}, _limit), do: :ok + + defp check_content_length(%Tesla.Env{status: status}, _limit) when status in [204, 304], + do: :ok + + defp check_content_length(env, limit) do + with value when is_binary(value) <- Tesla.get_header(env, "content-length"), + {length, ""} <- Integer.parse(value) do + if length > limit, do: {:error, :body_too_large}, else: :ok + else + _ -> :ok + end + end + + defp check_body(body, limit) do + if byte_size(body) > limit, do: {:error, :body_too_large}, else: :ok + end +end diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 75da41da9..3adc90c67 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -39,6 +39,7 @@ defmodule Pleroma.User do alias Pleroma.Web.Endpoint alias Pleroma.Web.OAuth alias Pleroma.Web.RelMe + alias Pleroma.Webhook.Notify alias Pleroma.Workers.BackgroundWorker alias Pleroma.Workers.DeleteWorker alias Pleroma.Workers.UserRefreshWorker @@ -272,13 +273,13 @@ defmodule Pleroma.User do end def cached_blocked_users_ap_ids(user) do - @cachex.fetch!(:user_cache, "blocked_users_ap_ids:#{user.ap_id}", fn _ -> + @cachex.fetch!(:user_cache, cache_key(:blocked_users_ap_ids, user), fn _ -> blocked_users_ap_ids(user) end) end def cached_muted_users_ap_ids(user) do - @cachex.fetch!(:user_cache, "muted_users_ap_ids:#{user.ap_id}", fn _ -> + @cachex.fetch!(:user_cache, cache_key(:muted_users_ap_ids, user), fn _ -> muted_users_ap_ids(user) end) end @@ -326,7 +327,7 @@ defmodule Pleroma.User do @spec visible_for(User.t(), User.t() | nil) :: :visible | :invisible - | :restricted_unauthenticated + | :restrict_unauthenticated | :deactivated | :confirmation_pending def visible_for(user, for_user \\ nil) @@ -987,6 +988,7 @@ defmodule Pleroma.User do @doc "Inserts provided changeset, performs post-registration actions (confirmation email sending etc.)" def register(%Ecto.Changeset{} = changeset) do with {:ok, user} <- Repo.insert(changeset) do + Notify.trigger_webhooks(user, :"account.created") post_register_action(user) end end @@ -1253,13 +1255,18 @@ defmodule Pleroma.User do get_cached_by_nickname(nickname) end + defp cache_key(:id, %User{id: id}), do: cache_key(:id, id) + defp cache_key(:nickname, %User{nickname: nickname}), do: cache_key(:nickname, nickname) + defp cache_key(namespace, %User{ap_id: ap_id}), do: cache_key(namespace, ap_id) + defp cache_key(namespace, value), do: "#{namespace}:#{value}" + def set_cache({:ok, user}), do: set_cache(user) def set_cache({:error, err}), do: {:error, err} def set_cache(%User{} = user) do - @cachex.put(:user_cache, "ap_id:#{user.ap_id}", user) - @cachex.put(:user_cache, "nickname:#{user.nickname}", user) - @cachex.put(:user_cache, "friends_ap_ids:#{user.nickname}", get_user_friends_ap_ids(user)) + @cachex.put(:user_cache, cache_key(:ap_id, user), user) + @cachex.put(:user_cache, cache_key(:nickname, user), user) + @cachex.put(:user_cache, cache_key(:friends_ap_ids, user), get_user_friends_ap_ids(user)) {:ok, user} end @@ -1287,22 +1294,23 @@ defmodule Pleroma.User do @spec get_cached_user_friends_ap_ids(User.t()) :: [String.t()] def get_cached_user_friends_ap_ids(user) do - @cachex.fetch!(:user_cache, "friends_ap_ids:#{user.ap_id}", fn _ -> + @cachex.fetch!(:user_cache, cache_key(:friends_ap_ids, user), fn _ -> get_user_friends_ap_ids(user) end) end def invalidate_cache(user) do - @cachex.del(:user_cache, "ap_id:#{user.ap_id}") - @cachex.del(:user_cache, "nickname:#{user.nickname}") - @cachex.del(:user_cache, "friends_ap_ids:#{user.ap_id}") - @cachex.del(:user_cache, "blocked_users_ap_ids:#{user.ap_id}") - @cachex.del(:user_cache, "muted_users_ap_ids:#{user.ap_id}") + @cachex.del(:user_cache, cache_key(:ap_id, user)) + @cachex.del(:user_cache, cache_key(:nickname, user)) + @cachex.del(:user_cache, cache_key(:id, user)) + @cachex.del(:user_cache, cache_key(:friends_ap_ids, user)) + @cachex.del(:user_cache, cache_key(:blocked_users_ap_ids, user)) + @cachex.del(:user_cache, cache_key(:muted_users_ap_ids, user)) end @spec get_cached_by_ap_id(String.t()) :: User.t() | nil def get_cached_by_ap_id(ap_id) do - key = "ap_id:#{ap_id}" + key = cache_key(:ap_id, ap_id) with {:ok, nil} <- @cachex.get(:user_cache, key), user when not is_nil(user) <- get_by_ap_id(ap_id), @@ -1315,14 +1323,14 @@ defmodule Pleroma.User do end def get_cached_by_id(id) do - key = "id:#{id}" + key = cache_key(:id, id) ap_id = @cachex.fetch!(:user_cache, key, fn _ -> user = get_by_id(id) if user do - @cachex.put(:user_cache, "ap_id:#{user.ap_id}", user) + @cachex.put(:user_cache, cache_key(:ap_id, user), user) {:commit, user.ap_id} else {:ignore, ""} @@ -1333,7 +1341,7 @@ defmodule Pleroma.User do end def get_cached_by_nickname(nickname) do - key = "nickname:#{nickname}" + key = cache_key(:nickname, nickname) @cachex.fetch!(:user_cache, key, fn _ -> case get_or_fetch_by_nickname(nickname) do @@ -1651,7 +1659,7 @@ defmodule Pleroma.User do |> Oban.insert() end - @cachex.del(:user_cache, "muted_users_ap_ids:#{muter.ap_id}") + @cachex.del(:user_cache, cache_key(:muted_users_ap_ids, muter)) {:ok, Enum.filter([user_mute, user_notification_mute], & &1)} end @@ -1661,7 +1669,7 @@ defmodule Pleroma.User do with {:ok, user_mute} <- UserRelationship.delete_mute(muter, mutee), {:ok, user_notification_mute} <- UserRelationship.delete_notification_mute(muter, mutee) do - @cachex.del(:user_cache, "muted_users_ap_ids:#{muter.ap_id}") + @cachex.del(:user_cache, cache_key(:muted_users_ap_ids, muter)) {:ok, [user_mute, user_notification_mute]} end end @@ -1964,6 +1972,12 @@ defmodule Pleroma.User do def approve(%User{} = user), do: {:ok, user} + def reject(%User{is_approved: false} = user) do + delete(user) + end + + def reject(%User{} = _user), do: {:error, "User is approved"} + def confirm(users) when is_list(users) do Repo.transaction(fn -> Enum.map(users, fn user -> @@ -2815,7 +2829,7 @@ defmodule Pleroma.User do {:ok, UserRelationship.t()} | {:error, Ecto.Changeset.t()} defp add_to_block(%User{} = user, %User{} = blocked, expires_at) do with {:ok, relationship} <- UserRelationship.create_block(user, blocked, expires_at) do - @cachex.del(:user_cache, "blocked_users_ap_ids:#{user.ap_id}") + @cachex.del(:user_cache, cache_key(:blocked_users_ap_ids, user)) {:ok, relationship} end end @@ -2824,7 +2838,7 @@ defmodule Pleroma.User do {:ok, UserRelationship.t()} | {:ok, nil} | {:error, Ecto.Changeset.t()} defp remove_from_block(%User{} = user, %User{} = blocked) do with {:ok, relationship} <- UserRelationship.delete_block(user, blocked) do - @cachex.del(:user_cache, "blocked_users_ap_ids:#{user.ap_id}") + @cachex.del(:user_cache, cache_key(:blocked_users_ap_ids, user)) {:ok, relationship} end end diff --git a/lib/pleroma/user/query.ex b/lib/pleroma/user/query.ex index cd9586452..33f19c454 100644 --- a/lib/pleroma/user/query.ex +++ b/lib/pleroma/user/query.ex @@ -63,7 +63,8 @@ defmodule Pleroma.User.Query do limit: pos_integer(), actor_types: [String.t()], birthday_day: pos_integer(), - birthday_month: pos_integer() + birthday_month: pos_integer(), + staff: boolean() } | map() @@ -179,6 +180,10 @@ defmodule Pleroma.User.Query do defp compose_query({:external, _}, query), do: location_query(query, false) + defp compose_query({:active, false}, query) do + where(query, [u], u.is_active == false or u.is_approved == false or u.is_confirmed == false) + end + defp compose_query({:active, _}, query) do where(query, [u], u.is_active == true) |> where([u], u.is_approved == true) @@ -202,6 +207,10 @@ defmodule Pleroma.User.Query do where(query, [u], u.is_confirmed != ^bool) end + defp compose_query({:need_approval, false}, query) do + where(query, [u], u.is_approved == true) + end + defp compose_query({:need_approval, _}, query) do where(query, [u], u.is_approved == false) end @@ -285,6 +294,19 @@ defmodule Pleroma.User.Query do |> where([u], fragment("date_part('month', ?)", u.birthday) == ^month) end + defp compose_query({:staff, false}, query) do + where(query, [u], u.is_admin == false and u.is_moderator == false) + end + + defp compose_query({:staff, _}, query) do + where(query, [u], u.is_admin == true or u.is_moderator == true) + end + + defp compose_query({:domain, domain}, query) do + query + |> where([u], like(u.nickname, ^"%@#{domain}")) + end + defp compose_query(_unsupported_param, query), do: query defp location_query(query, local) do diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 1e8bfe3dd..1279e0cb4 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -30,6 +30,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do import Ecto.Query import Pleroma.Web.ActivityPub.Utils import Pleroma.Web.ActivityPub.Visibility + import Pleroma.Webhook.Notify, only: [trigger_webhooks: 2] require Logger require Pleroma.Constants @@ -415,7 +416,9 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do with flag_data <- make_flag_data(params, additional), {:ok, activity} <- insert(flag_data, local), _ <- notify_and_stream(activity), - :ok <- maybe_federate(activity) do + _ <- trigger_webhooks(activity, :"report.created"), + :ok <- + maybe_federate(activity) do User.all_users_with_privilege(:reports_manage_reports) |> Enum.filter(fn user -> user.ap_id != actor end) |> Enum.filter(fn user -> not is_nil(user.email) end) @@ -537,9 +540,12 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do # and extra sorting on "activities.id DESC NULLS LAST" would worse the query plan opts = Map.put(opts, :skip_extra_order, true) - Pagination.fetch_paginated(query, opts, pagination) + Pagination.fetch_paginated(query, opts, pagination, pagination_binding(opts)) end + defp pagination_binding(%{favorited_by: _}), do: :favorited_activity + defp pagination_binding(_), do: nil + def fetch_activities(recipients, opts \\ %{}, pagination \\ :keyset) do list_memberships = Pleroma.List.memberships(opts[:user]) @@ -707,6 +713,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do |> Map.put(:type, ["Create", "Announce"]) |> Map.put(:user, reading_user) |> Map.put(:actor_id, user.ap_id) + |> Map.put(:actor_local, user.local) |> Map.put(:pinned_object_ids, Map.keys(user.pinned_objects)) params = @@ -786,6 +793,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do defp restrict_since(query, %{since_id: ""}), do: query + defp restrict_since(query, %{favorited_by: _, since_id: _}), do: query + defp restrict_since(query, %{since_id: since_id}) do from(activity in query, where: activity.id > ^since_id) end @@ -893,14 +902,22 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do |> Repo.all() # Note: NO extra ordering should be done on "activities.id desc nulls last" for optimal plan - from( - [_activity, object] in query, - join: hto in "hashtags_objects", - on: hto.object_id == object.id, - where: hto.hashtag_id in ^hashtag_ids, - distinct: [desc: object.id], - order_by: [desc: object.id] - ) + query = + from( + [_activity, object] in query, + join: hto in "hashtags_objects", + on: hto.object_id == object.id, + where: hto.hashtag_id in ^hashtag_ids + ) + + if is_nil(query.distinct) do + from([_activity, object] in query, + distinct: [desc: object.id], + order_by: [desc: object.id] + ) + else + query + end end defp restrict_hashtag_any(query, %{tag: tag}) when is_binary(tag) do @@ -1003,6 +1020,14 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do defp restrict_state(query, _), do: query + defp restrict_report_target(query, %{report_target_id: actor_id}) do + from(activity in query, + where: fragment("?->'object'->>0 = ?", activity.data, ^actor_id) + ) + end + + defp restrict_report_target(query, _), do: query + defp restrict_assigned_account(query, %{assigned_account: assigned_account}) do from(activity in query, where: fragment("?->>'assigned_account' = ?", activity.data, ^assigned_account) @@ -1012,9 +1037,31 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do defp restrict_assigned_account(query, _), do: query defp restrict_favorited_by(query, %{favorited_by: ap_id}) do + newer_favorite = + from(newer_favorite in Activity, + where: newer_favorite.actor == ^ap_id, + where: newer_favorite.id > parent_as(:favorited_activity).id, + where: fragment("?->>'type' = ?", newer_favorite.data, "Like"), + where: + fragment( + "associated_object_id(?) = associated_object_id(?)", + newer_favorite.data, + parent_as(:favorited_activity).data + ), + select: 1 + ) + from( - [_activity, object] in query, - where: fragment("(?)->'likes' \\? (?)", object.data, ^ap_id) + [activity, object: object] in query, + join: favorited_activity in Activity, + as: :favorited_activity, + on: + favorited_activity.actor == ^ap_id and + fragment("?->>'type' = ?", favorited_activity.data, "Like") and + fragment("associated_object_id(?) = (?)->>'id'", favorited_activity.data, object.data), + where: fragment("(?)->'likes' \\? (?)", object.data, ^ap_id), + where: not exists(subquery(newer_favorite)), + select: %Activity{activity | pagination_id: favorited_activity.id} ) end @@ -1314,6 +1361,50 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do defp restrict_unauthenticated(query, _), do: query + defp restrict_activity_and_object_origin(query, local?) do + local_object_prefix = Pleroma.Web.Endpoint.url() <> "/" + + # Keeping the correlated origin checks inside CASE prevents PostgreSQL from choosing a + # global bitmap-and-sort plan before applying the account statuses limit. + from([activity, object: object] in query, + where: + fragment( + "CASE WHEN ? = ? AND starts_with((?->>'id'), ?) = ? THEN true ELSE false END", + activity.local, + ^local?, + object.data, + ^local_object_prefix, + ^local? + ) + ) + end + + defp maybe_restrict_unauthenticated( + query, + %{restrict_unauthenticated: true, user: nil, actor_local: actor_local} + ) + when is_boolean(actor_local) do + local_restricted = Config.restrict_unauthenticated_access?(:activities, :local) + remote_restricted = Config.restrict_unauthenticated_access?(:activities, :remote) + actor_restricted = if actor_local, do: local_restricted, else: remote_restricted + + cond do + actor_restricted -> + from(activity in query, where: false) + + local_restricted -> + restrict_activity_and_object_origin(query, false) + + remote_restricted -> + restrict_activity_and_object_origin(query, true) + + true -> + query + end + end + + defp maybe_restrict_unauthenticated(query, _), do: query + defp restrict_quote_url(query, %{quote_url: quote_url}) do from([_activity, object] in query, where: fragment("(?)->'quoteUrl' = ?", object.data, ^quote_url) @@ -1471,6 +1562,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do |> maybe_preload_report_notes(opts) |> maybe_set_thread_muted_field(opts) |> maybe_order(opts) + |> maybe_restrict_unauthenticated(opts) |> restrict_recipients_or_hashtags(recipients, opts[:user], opts[:followed_hashtags]) |> restrict_replies(opts) |> restrict_since(opts) @@ -1479,6 +1571,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do |> restrict_actor(opts) |> restrict_type(opts) |> restrict_state(opts) + |> restrict_report_target(opts) |> restrict_assigned_account(opts) |> restrict_favorited_by(opts) |> restrict_blocked(restrict_blocked_opts) diff --git a/lib/pleroma/web/activity_pub/activity_pub_controller.ex b/lib/pleroma/web/activity_pub/activity_pub_controller.ex index 3d1875599..38b8b8bfc 100644 --- a/lib/pleroma/web/activity_pub/activity_pub_controller.ex +++ b/lib/pleroma/web/activity_pub/activity_pub_controller.ex @@ -309,7 +309,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do } = conn, %{"nickname" => nickname} = params ) do - with {:recipient_exists, %User{} = recipient} <- + with false <- unsupported_http_message_signature?(conn), + {:recipient_exists, %User{} = recipient} <- {:recipient_exists, User.get_cached_by_nickname(nickname)}, {:sender_exists, {:ok, %User{} = actor}} <- {:sender_exists, User.get_or_fetch_by_ap_id(params["actor"])}, @@ -320,6 +321,9 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do Federator.incoming_ap_doc(params) json(conn, "ok") else + true -> + reject_unsupported_http_message_signature(conn) + {:recipient_exists, _} -> conn |> put_status(:not_found) @@ -348,20 +352,28 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do end def inbox(%{assigns: %{valid_signature: true, valid_host_header: true}} = conn, params) do - Federator.incoming_ap_doc(params) - json(conn, "ok") + if unsupported_http_message_signature?(conn) do + reject_unsupported_http_message_signature(conn) + else + Federator.incoming_ap_doc(params) + json(conn, "ok") + end end def inbox(%{assigns: %{valid_signature: false}} = conn, params) do - Federator.incoming_failed_signature_ap_doc(%{ - method: conn.method, - req_headers: conn.req_headers, - request_path: conn.request_path, - params: params, - query_string: conn.query_string - }) + if unsupported_http_message_signature?(conn) do + reject_unsupported_http_message_signature(conn) + else + Federator.incoming_failed_signature_ap_doc(%{ + method: conn.method, + req_headers: conn.req_headers, + request_path: conn.request_path, + params: params, + query_string: conn.query_string + }) - json(conn, "ok") + json(conn, "ok") + end end # POST /relay/inbox -or- POST /internal/fetch/inbox @@ -381,6 +393,27 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do |> json("error, missing HTTP Signature") end + defp reject_unsupported_http_message_signature(conn) do + conn + |> put_status(:unauthorized) + |> json("error, unsupported HTTP Message Signature") + end + + defp unsupported_http_message_signature?(conn) do + signature_headers = Plug.Conn.get_req_header(conn, "signature") + + multiple_signature_headers?(signature_headers) or + (Plug.Conn.get_req_header(conn, "signature-input") != [] and + not cavage_signature?(signature_headers)) + end + + defp multiple_signature_headers?([_, _ | _]), do: true + defp multiple_signature_headers?(_), do: false + + defp cavage_signature?(signature_headers) do + Enum.any?(signature_headers, &Regex.match?(~r/(^|,)\s*keyId\s*=/, &1)) + end + defp post_inbox_relayed_create(conn, params) do Logger.debug( "Signature missing or not from author, relayed Create message, fetching object from source" diff --git a/lib/pleroma/web/activity_pub/mrf/activity_expiration_policy.ex b/lib/pleroma/web/activity_pub/mrf/activity_expiration_policy.ex index 88f6ca028..41abe17c0 100644 --- a/lib/pleroma/web/activity_pub/mrf/activity_expiration_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/activity_expiration_policy.ex @@ -6,6 +6,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.ActivityExpirationPolicy do @moduledoc "Adds expiration to all local Create activities" @behaviour Pleroma.Web.ActivityPub.MRF.Policy + use Pleroma.Web.ActivityPub.MRF.Policy + @impl true def filter(activity) do activity = diff --git a/lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex b/lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex index 8ea61aec2..990ff31c9 100644 --- a/lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex @@ -3,11 +3,11 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.ActivityPub.MRF.AntiFollowbotPolicy do + @moduledoc "Prevent followbots from following with a bit of heuristic" + @behaviour Pleroma.Web.ActivityPub.MRF.Policy alias Pleroma.User - @moduledoc "Prevent followbots from following with a bit of heuristic" - - @behaviour Pleroma.Web.ActivityPub.MRF.Policy + use Pleroma.Web.ActivityPub.MRF.Policy # XXX: this should become User.normalize_by_ap_id() or similar, really. defp normalize_by_ap_id(%{"id" => id}), do: User.get_cached_by_ap_id(id) diff --git a/lib/pleroma/web/activity_pub/mrf/anti_link_spam_policy.ex b/lib/pleroma/web/activity_pub/mrf/anti_link_spam_policy.ex index 2be6d8df4..5c3b66e37 100644 --- a/lib/pleroma/web/activity_pub/mrf/anti_link_spam_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/anti_link_spam_policy.ex @@ -3,9 +3,11 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.ActivityPub.MRF.AntiLinkSpamPolicy do + @behaviour Pleroma.Web.ActivityPub.MRF.Policy + alias Pleroma.User - @behaviour Pleroma.Web.ActivityPub.MRF.Policy + use Pleroma.Web.ActivityPub.MRF.Policy require Logger diff --git a/lib/pleroma/web/activity_pub/mrf/anti_mention_spam_policy.ex b/lib/pleroma/web/activity_pub/mrf/anti_mention_spam_policy.ex index 1d76a307b..c6f5781b3 100644 --- a/lib/pleroma/web/activity_pub/mrf/anti_mention_spam_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/anti_mention_spam_policy.ex @@ -3,11 +3,13 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.ActivityPub.MRF.AntiMentionSpamPolicy do + @behaviour Pleroma.Web.ActivityPub.MRF.Policy + alias Pleroma.Config alias Pleroma.User require Pleroma.Constants - @behaviour Pleroma.Web.ActivityPub.MRF.Policy + use Pleroma.Web.ActivityPub.MRF.Policy defp user_has_posted?(%User{} = u), do: u.note_count > 0 diff --git a/lib/pleroma/web/activity_pub/mrf/drop_policy.ex b/lib/pleroma/web/activity_pub/mrf/drop_policy.ex index cf07db7f3..4aa1968f7 100644 --- a/lib/pleroma/web/activity_pub/mrf/drop_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/drop_policy.ex @@ -3,10 +3,12 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.ActivityPub.MRF.DropPolicy do - require Logger @moduledoc "Drop and log everything received" @behaviour Pleroma.Web.ActivityPub.MRF.Policy + require Logger + use Pleroma.Web.ActivityPub.MRF.Policy + @impl true def filter(activity) do Logger.debug("REJECTING #{inspect(activity)}") diff --git a/lib/pleroma/web/activity_pub/mrf/emoji_policy.ex b/lib/pleroma/web/activity_pub/mrf/emoji_policy.ex index 1de5280d9..cc5fd49bf 100644 --- a/lib/pleroma/web/activity_pub/mrf/emoji_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/emoji_policy.ex @@ -3,6 +3,8 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.ActivityPub.MRF.EmojiPolicy do + @behaviour Pleroma.Web.ActivityPub.MRF.Policy + require Pleroma.Constants alias Pleroma.Object.Updater @@ -10,7 +12,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.EmojiPolicy do @moduledoc "Reject or force-unlisted emojis with certain URLs or names" - @behaviour Pleroma.Web.ActivityPub.MRF.Policy + use Pleroma.Web.ActivityPub.MRF.Policy defp config_remove_url do Pleroma.Config.get([:mrf_emoji, :remove_url], []) diff --git a/lib/pleroma/web/activity_pub/mrf/ensure_re_prepended.ex b/lib/pleroma/web/activity_pub/mrf/ensure_re_prepended.ex index f5983c8a7..6e43e746c 100644 --- a/lib/pleroma/web/activity_pub/mrf/ensure_re_prepended.ex +++ b/lib/pleroma/web/activity_pub/mrf/ensure_re_prepended.ex @@ -3,11 +3,13 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.ActivityPub.MRF.EnsureRePrepended do - alias Pleroma.Object - @moduledoc "Ensure a re: is prepended on replies to a post with a Subject" @behaviour Pleroma.Web.ActivityPub.MRF.Policy + alias Pleroma.Object + + use Pleroma.Web.ActivityPub.MRF.Policy + @reply_prefix Regex.compile!("^re:[[:space:]]*", [:caseless]) def history_awareness, do: :auto diff --git a/lib/pleroma/web/activity_pub/mrf/follow_bot_policy.ex b/lib/pleroma/web/activity_pub/mrf/follow_bot_policy.ex index 480a03ef6..4612b2cc5 100644 --- a/lib/pleroma/web/activity_pub/mrf/follow_bot_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/follow_bot_policy.ex @@ -4,12 +4,15 @@ defmodule Pleroma.Web.ActivityPub.MRF.FollowBotPolicy do @behaviour Pleroma.Web.ActivityPub.MRF.Policy + alias Pleroma.Config alias Pleroma.User alias Pleroma.Web.CommonAPI require Logger + use Pleroma.Web.ActivityPub.MRF.Policy + @impl true def filter(activity) do with follower_nickname <- Config.get([:mrf_follow_bot, :follower_nickname]), diff --git a/lib/pleroma/web/activity_pub/mrf/force_bot_unlisted_policy.ex b/lib/pleroma/web/activity_pub/mrf/force_bot_unlisted_policy.ex index 3b3251dc3..58d02f433 100644 --- a/lib/pleroma/web/activity_pub/mrf/force_bot_unlisted_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/force_bot_unlisted_policy.ex @@ -3,9 +3,12 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.ActivityPub.MRF.ForceBotUnlistedPolicy do - alias Pleroma.User - @behaviour Pleroma.Web.ActivityPub.MRF.Policy @moduledoc "Remove bot posts from federated timeline" + @behaviour Pleroma.Web.ActivityPub.MRF.Policy + + alias Pleroma.User + + use Pleroma.Web.ActivityPub.MRF.Policy require Pleroma.Constants diff --git a/lib/pleroma/web/activity_pub/mrf/force_mention.ex b/lib/pleroma/web/activity_pub/mrf/force_mention.ex index 4ea23540d..62aff36bc 100644 --- a/lib/pleroma/web/activity_pub/mrf/force_mention.ex +++ b/lib/pleroma/web/activity_pub/mrf/force_mention.ex @@ -3,13 +3,14 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.ActivityPub.MRF.ForceMention do + @behaviour Pleroma.Web.ActivityPub.MRF.Policy require Pleroma.Constants alias Pleroma.Config alias Pleroma.Object alias Pleroma.User - @behaviour Pleroma.Web.ActivityPub.MRF.Policy + use Pleroma.Web.ActivityPub.MRF.Policy defp get_author(url) do with %Object{data: %{"actor" => actor}} <- Object.normalize(url, fetch: false), diff --git a/lib/pleroma/web/activity_pub/mrf/force_mentions_in_content.ex b/lib/pleroma/web/activity_pub/mrf/force_mentions_in_content.ex index caae365e5..e7c3f2d88 100644 --- a/lib/pleroma/web/activity_pub/mrf/force_mentions_in_content.ex +++ b/lib/pleroma/web/activity_pub/mrf/force_mentions_in_content.ex @@ -3,13 +3,15 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.ActivityPub.MRF.ForceMentionsInContent do + @behaviour Pleroma.Web.ActivityPub.MRF.Policy + require Pleroma.Constants alias Pleroma.Formatter alias Pleroma.Object alias Pleroma.User - @behaviour Pleroma.Web.ActivityPub.MRF.Policy + use Pleroma.Web.ActivityPub.MRF.Policy @impl true def history_awareness, do: :auto diff --git a/lib/pleroma/web/activity_pub/mrf/hashtag_policy.ex b/lib/pleroma/web/activity_pub/mrf/hashtag_policy.ex index 72f2274ed..a3fae3bae 100644 --- a/lib/pleroma/web/activity_pub/mrf/hashtag_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/hashtag_policy.ex @@ -3,6 +3,8 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.ActivityPub.MRF.HashtagPolicy do + @behaviour Pleroma.Web.ActivityPub.MRF.Policy + require Pleroma.Constants alias Pleroma.Config @@ -14,7 +16,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.HashtagPolicy do Note: This MRF Policy is always enabled, if you want to disable it you have to set empty lists. """ - @behaviour Pleroma.Web.ActivityPub.MRF.Policy + use Pleroma.Web.ActivityPub.MRF.Policy @impl true def history_awareness, do: :manual diff --git a/lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex b/lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex index 3a80d0a69..a84a7b0c8 100644 --- a/lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex @@ -3,13 +3,14 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.ActivityPub.MRF.HellthreadPolicy do + @moduledoc "Block activities with too much mentions (configurable)" + @behaviour Pleroma.Web.ActivityPub.MRF.Policy + alias Pleroma.User require Pleroma.Constants - @moduledoc "Block activities with too much mentions (configurable)" - - @behaviour Pleroma.Web.ActivityPub.MRF.Policy + use Pleroma.Web.ActivityPub.MRF.Policy defp delist_activity(activity, threshold) when threshold > 0 do follower_collection = User.get_cached_by_ap_id(activity["actor"]).follower_address diff --git a/lib/pleroma/web/activity_pub/mrf/inline_quote_policy.ex b/lib/pleroma/web/activity_pub/mrf/inline_quote_policy.ex index 51dcd3918..17879b4bd 100644 --- a/lib/pleroma/web/activity_pub/mrf/inline_quote_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/inline_quote_policy.ex @@ -6,6 +6,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.InlineQuotePolicy do @moduledoc "Force a quote line into the message content." @behaviour Pleroma.Web.ActivityPub.MRF.Policy + use Pleroma.Web.ActivityPub.MRF.Policy + defp build_inline_quote(template, url) do quote_line = String.replace(template, "{url}", "#{url}") diff --git a/lib/pleroma/web/activity_pub/mrf/keyword_policy.ex b/lib/pleroma/web/activity_pub/mrf/keyword_policy.ex index 6ba6fd509..33f32e781 100644 --- a/lib/pleroma/web/activity_pub/mrf/keyword_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/keyword_policy.ex @@ -3,13 +3,14 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.ActivityPub.MRF.KeywordPolicy do + @moduledoc "Reject or Word-Replace activities with a keyword or regex" + @behaviour Pleroma.Web.ActivityPub.MRF.Policy + require Pleroma.Constants alias Pleroma.Web.ActivityPub.MRF.Utils - @moduledoc "Reject or Word-Replace activities with a keyword or regex" - - @behaviour Pleroma.Web.ActivityPub.MRF.Policy + use Pleroma.Web.ActivityPub.MRF.Policy defp string_matches?(string, pattern) when is_binary(pattern) do String.contains?(string, pattern) diff --git a/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex b/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex index 43c2c0449..07046a666 100644 --- a/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex @@ -6,6 +6,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do @moduledoc "Preloads any attachments in the MediaProxy cache by prefetching them" @behaviour Pleroma.Web.ActivityPub.MRF.Policy + use Pleroma.Web.ActivityPub.MRF.Policy + alias Pleroma.HTTP alias Pleroma.Web.MediaProxy diff --git a/lib/pleroma/web/activity_pub/mrf/mention_policy.ex b/lib/pleroma/web/activity_pub/mrf/mention_policy.ex index f7bff121f..8e108a0ae 100644 --- a/lib/pleroma/web/activity_pub/mrf/mention_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/mention_policy.ex @@ -4,9 +4,10 @@ defmodule Pleroma.Web.ActivityPub.MRF.MentionPolicy do @moduledoc "Block activities which mention a user" - @behaviour Pleroma.Web.ActivityPub.MRF.Policy + use Pleroma.Web.ActivityPub.MRF.Policy + @impl true def filter(%{"type" => "Create"} = activity) do reject_actors = Pleroma.Config.get([:mrf_mention, :actors], []) diff --git a/lib/pleroma/web/activity_pub/mrf/no_empty_policy.ex b/lib/pleroma/web/activity_pub/mrf/no_empty_policy.ex index 08dd39878..8283fd504 100644 --- a/lib/pleroma/web/activity_pub/mrf/no_empty_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/no_empty_policy.ex @@ -6,6 +6,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.NoEmptyPolicy do @moduledoc "Filter local activities which have no content" @behaviour Pleroma.Web.ActivityPub.MRF.Policy + use Pleroma.Web.ActivityPub.MRF.Policy + alias Pleroma.Web.Endpoint @impl true diff --git a/lib/pleroma/web/activity_pub/mrf/no_op_policy.ex b/lib/pleroma/web/activity_pub/mrf/no_op_policy.ex index 64a5872bc..3adbb598a 100644 --- a/lib/pleroma/web/activity_pub/mrf/no_op_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/no_op_policy.ex @@ -6,6 +6,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.NoOpPolicy do @moduledoc "Does nothing (lets the messages go through unmodified)" @behaviour Pleroma.Web.ActivityPub.MRF.Policy + use Pleroma.Web.ActivityPub.MRF.Policy + @impl true def filter(activity) do {:ok, activity} diff --git a/lib/pleroma/web/activity_pub/mrf/no_placeholder_text_policy.ex b/lib/pleroma/web/activity_pub/mrf/no_placeholder_text_policy.ex index c6f239a5e..2a10c7e97 100644 --- a/lib/pleroma/web/activity_pub/mrf/no_placeholder_text_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/no_placeholder_text_policy.ex @@ -6,6 +6,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.NoPlaceholderTextPolicy do @moduledoc "Ensure no content placeholder is present (such as the dot from mastodon)" @behaviour Pleroma.Web.ActivityPub.MRF.Policy + use Pleroma.Web.ActivityPub.MRF.Policy + @impl true def history_awareness, do: :auto diff --git a/lib/pleroma/web/activity_pub/mrf/normalize_markup.ex b/lib/pleroma/web/activity_pub/mrf/normalize_markup.ex index 91855ef84..de0f5b619 100644 --- a/lib/pleroma/web/activity_pub/mrf/normalize_markup.ex +++ b/lib/pleroma/web/activity_pub/mrf/normalize_markup.ex @@ -4,9 +4,11 @@ defmodule Pleroma.Web.ActivityPub.MRF.NormalizeMarkup do @moduledoc "Scrub configured hypertext markup" + @behaviour Pleroma.Web.ActivityPub.MRF.Policy + alias Pleroma.HTML - @behaviour Pleroma.Web.ActivityPub.MRF.Policy + use Pleroma.Web.ActivityPub.MRF.Policy @impl true def history_awareness, do: :auto diff --git a/lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex b/lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex index 52aaf05aa..7e3ec3de5 100644 --- a/lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex @@ -38,6 +38,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.NsfwApiPolicy do - `unlist`: Unlist all detected NSFW content? Default: `false` - `reject`: Reject all detected NSFW content (takes precedence)? Default: `false` """ + @behaviour Pleroma.Web.ActivityPub.MRF.Policy + alias Pleroma.Config alias Pleroma.Constants alias Pleroma.HTTP @@ -46,7 +48,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.NsfwApiPolicy do require Logger require Pleroma.Constants - @behaviour Pleroma.Web.ActivityPub.MRF.Policy + use Pleroma.Web.ActivityPub.MRF.Policy + @policy :mrf_nsfw_api def build_request_url(url) do diff --git a/lib/pleroma/web/activity_pub/mrf/object_age_policy.ex b/lib/pleroma/web/activity_pub/mrf/object_age_policy.ex index 34905fc21..00f4cf84b 100644 --- a/lib/pleroma/web/activity_pub/mrf/object_age_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/object_age_policy.ex @@ -3,13 +3,15 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.ActivityPub.MRF.ObjectAgePolicy do + @moduledoc "Filter activities depending on their age" + @behaviour Pleroma.Web.ActivityPub.MRF.Policy + alias Pleroma.Config alias Pleroma.User require Pleroma.Constants - @moduledoc "Filter activities depending on their age" - @behaviour Pleroma.Web.ActivityPub.MRF.Policy + use Pleroma.Web.ActivityPub.MRF.Policy defp check_date(%{"object" => %{"published" => published}} = activity) do with %DateTime{} = now <- DateTime.utc_now(), diff --git a/lib/pleroma/web/activity_pub/mrf/policy.ex b/lib/pleroma/web/activity_pub/mrf/policy.ex index 08bcac08a..b9a693c9e 100644 --- a/lib/pleroma/web/activity_pub/mrf/policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/policy.ex @@ -15,4 +15,12 @@ defmodule Pleroma.Web.ActivityPub.MRF.Policy do } @callback history_awareness() :: :auto | :manual @optional_callbacks config_description: 0, history_awareness: 0, id_filter: 1 + + defmacro __using__(_) do + quote do + # Otherwise Pleroma.Docs.Generator.list_behaviour_implementations fails + # since the Elixir compiler doesn't automatically load the module since v1.19. + @compile {:autoload, true} + end + end end diff --git a/lib/pleroma/web/activity_pub/mrf/quiet_reply.ex b/lib/pleroma/web/activity_pub/mrf/quiet_reply.ex index b3eb0b390..b47e9edb9 100644 --- a/lib/pleroma/web/activity_pub/mrf/quiet_reply.ex +++ b/lib/pleroma/web/activity_pub/mrf/quiet_reply.ex @@ -6,11 +6,13 @@ defmodule Pleroma.Web.ActivityPub.MRF.QuietReply do @moduledoc """ QuietReply alters the scope of activities from local users when replying by enforcing them to be "Unlisted" or "Quiet Public". This delivers the activity to all the expected recipients and instances, but it will not be published in the Federated / The Whole Known Network timelines. It will still be published to the Home timelines of the user's followers and visible to anyone who opens the thread. """ + @behaviour Pleroma.Web.ActivityPub.MRF.Policy + require Pleroma.Constants alias Pleroma.User - @behaviour Pleroma.Web.ActivityPub.MRF.Policy + use Pleroma.Web.ActivityPub.MRF.Policy @impl true def history_awareness, do: :auto diff --git a/lib/pleroma/web/activity_pub/mrf/quote_to_link_tag_policy.ex b/lib/pleroma/web/activity_pub/mrf/quote_to_link_tag_policy.ex index 2a17b6761..e1ad17585 100644 --- a/lib/pleroma/web/activity_pub/mrf/quote_to_link_tag_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/quote_to_link_tag_policy.ex @@ -6,6 +6,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.QuoteToLinkTagPolicy do @moduledoc "Force a Link tag for posts quoting another post. (may break outgoing federation of quote posts with older Pleroma versions)" @behaviour Pleroma.Web.ActivityPub.MRF.Policy + use Pleroma.Web.ActivityPub.MRF.Policy + alias Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes require Pleroma.Constants diff --git a/lib/pleroma/web/activity_pub/mrf/reject_non_public.ex b/lib/pleroma/web/activity_pub/mrf/reject_non_public.ex index 9d4a7a405..7d113dbf6 100644 --- a/lib/pleroma/web/activity_pub/mrf/reject_non_public.ex +++ b/lib/pleroma/web/activity_pub/mrf/reject_non_public.ex @@ -4,11 +4,12 @@ defmodule Pleroma.Web.ActivityPub.MRF.RejectNonPublic do @moduledoc "Rejects non-public (followers-only, direct) activities" + @behaviour Pleroma.Web.ActivityPub.MRF.Policy alias Pleroma.Config alias Pleroma.User - @behaviour Pleroma.Web.ActivityPub.MRF.Policy + use Pleroma.Web.ActivityPub.MRF.Policy require Pleroma.Constants diff --git a/lib/pleroma/web/activity_pub/mrf/remote_report_policy.ex b/lib/pleroma/web/activity_pub/mrf/remote_report_policy.ex index 8422832dd..f9f8d2298 100644 --- a/lib/pleroma/web/activity_pub/mrf/remote_report_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/remote_report_policy.ex @@ -2,6 +2,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.RemoteReportPolicy do @moduledoc "Drop remote reports if they don't contain enough information." @behaviour Pleroma.Web.ActivityPub.MRF.Policy + use Pleroma.Web.ActivityPub.MRF.Policy + alias Pleroma.Config @impl true diff --git a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex index a97e8db7b..7045fe90f 100644 --- a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex @@ -6,6 +6,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do @moduledoc "Filter activities depending on their origin instance" @behaviour Pleroma.Web.ActivityPub.MRF.Policy + use Pleroma.Web.ActivityPub.MRF.Policy + alias Pleroma.Config alias Pleroma.FollowingRelationship alias Pleroma.User diff --git a/lib/pleroma/web/activity_pub/mrf/steal_emoji_policy.ex b/lib/pleroma/web/activity_pub/mrf/steal_emoji_policy.ex index 54f0e6bc1..f1e99aead 100644 --- a/lib/pleroma/web/activity_pub/mrf/steal_emoji_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/steal_emoji_policy.ex @@ -3,12 +3,14 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.ActivityPub.MRF.StealEmojiPolicy do + @moduledoc "Detect new emojis by their shortcode and steals them" + @behaviour Pleroma.Web.ActivityPub.MRF.Policy + require Logger alias Pleroma.Config - @moduledoc "Detect new emojis by their shortcode and steals them" - @behaviour Pleroma.Web.ActivityPub.MRF.Policy + use Pleroma.Web.ActivityPub.MRF.Policy defp accept_host?(host), do: host in Config.get([:mrf_steal_emoji, :hosts], []) diff --git a/lib/pleroma/web/activity_pub/mrf/subchain_policy.ex b/lib/pleroma/web/activity_pub/mrf/subchain_policy.ex index 97acca7e8..1fe7b00e9 100644 --- a/lib/pleroma/web/activity_pub/mrf/subchain_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/subchain_policy.ex @@ -3,12 +3,14 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.ActivityPub.MRF.SubchainPolicy do + @behaviour Pleroma.Web.ActivityPub.MRF.Policy + alias Pleroma.Config alias Pleroma.Web.ActivityPub.MRF require Logger - @behaviour Pleroma.Web.ActivityPub.MRF.Policy + use Pleroma.Web.ActivityPub.MRF.Policy defp lookup_subchain(actor) do with matches <- Config.get([:mrf_subchain, :match_actor]), diff --git a/lib/pleroma/web/activity_pub/mrf/tag_policy.ex b/lib/pleroma/web/activity_pub/mrf/tag_policy.ex index c236a5a99..df5594808 100644 --- a/lib/pleroma/web/activity_pub/mrf/tag_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/tag_policy.ex @@ -3,8 +3,6 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.ActivityPub.MRF.TagPolicy do - alias Pleroma.User - @behaviour Pleroma.Web.ActivityPub.MRF.Policy @moduledoc """ Apply policies based on user tags @@ -18,6 +16,11 @@ defmodule Pleroma.Web.ActivityPub.MRF.TagPolicy do - `mrf_tag:disable-remote-subscription`: Reject non-local follow requests - `mrf_tag:disable-any-subscription`: Reject any follow requests """ + @behaviour Pleroma.Web.ActivityPub.MRF.Policy + + alias Pleroma.User + + use Pleroma.Web.ActivityPub.MRF.Policy require Pleroma.Constants diff --git a/lib/pleroma/web/activity_pub/mrf/user_allow_list_policy.ex b/lib/pleroma/web/activity_pub/mrf/user_allow_list_policy.ex index 10cc0e09d..a04ca90bd 100644 --- a/lib/pleroma/web/activity_pub/mrf/user_allow_list_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/user_allow_list_policy.ex @@ -3,11 +3,13 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.ActivityPub.MRF.UserAllowListPolicy do - alias Pleroma.Config - @moduledoc "Accept-list of users from specified instances" @behaviour Pleroma.Web.ActivityPub.MRF.Policy + alias Pleroma.Config + + use Pleroma.Web.ActivityPub.MRF.Policy + defp filter_by_list(activity, []), do: {:ok, activity} defp filter_by_list(%{"actor" => actor} = activity, allow_list) do diff --git a/lib/pleroma/web/activity_pub/mrf/vocabulary_policy.ex b/lib/pleroma/web/activity_pub/mrf/vocabulary_policy.ex index 5671e4cf3..ecba3a2c6 100644 --- a/lib/pleroma/web/activity_pub/mrf/vocabulary_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/vocabulary_policy.ex @@ -4,9 +4,10 @@ defmodule Pleroma.Web.ActivityPub.MRF.VocabularyPolicy do @moduledoc "Filter activities which belong to certain activity vocabularies" - @behaviour Pleroma.Web.ActivityPub.MRF.Policy + use Pleroma.Web.ActivityPub.MRF.Policy + @impl true def filter(%{"type" => "Undo", "object" => object} = activity) do with {:ok, _} <- filter(object) do diff --git a/lib/pleroma/web/activity_pub/object_validator.ex b/lib/pleroma/web/activity_pub/object_validator.ex index 17652a0de..0254d6dc1 100644 --- a/lib/pleroma/web/activity_pub/object_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validator.ex @@ -118,6 +118,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do when objtype in ~w[Question Answer Audio Video Image Event Article Note Page] do with {:ok, object_data} <- object + |> CommonFixes.maybe_set_attributed_to_from_activity(create_activity) |> CommonFixes.maybe_add_language_from_activity(create_activity) |> cast_and_apply_and_stringify_with_history(), meta = Keyword.put(meta, :object_data, object_data), @@ -172,6 +173,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do {_, {:ok, object_data, _}} <- {:object_validation, object + |> CommonFixes.maybe_set_attributed_to_from_activity(update_activity) |> CommonFixes.maybe_add_language_from_activity(update_activity) |> validate(meta)}, meta = Keyword.put(meta, :object_data, object_data), diff --git a/lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex b/lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex index 844ad2c8f..1d6e0abfb 100644 --- a/lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex @@ -50,7 +50,32 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNotePageValidator do end defp fix_url(%{"url" => url} = data) when is_bitstring(url), do: data - defp fix_url(%{"url" => url} = data) when is_map(url), do: Map.put(data, "url", url["href"]) + + defp fix_url(%{"url" => url} = data) when is_map(url) do + if is_binary(url["href"]) do + Map.put(data, "url", url["href"]) + else + Map.delete(data, "url") + end + end + + defp fix_url(%{"url" => url} = data) when is_list(url) do + first_element = Enum.at(url, 0) + + url_string = + cond do + is_bitstring(first_element) -> first_element + is_map(first_element) -> first_element["href"] + true -> nil + end + + if is_binary(url_string) do + Map.put(data, "url", url_string) + else + Map.delete(data, "url") + end + end + defp fix_url(data), do: data defp fix_tag(%{"tag" => tag} = data) when is_list(tag) do @@ -79,10 +104,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNotePageValidator do defp fix_replies(data), do: Map.delete(data, "replies") - def fix_attachments(%{"attachment" => attachment} = data) when is_map(attachment), - do: Map.put(data, "attachment", [attachment]) - - def fix_attachments(data), do: data + def fix_attachments(data), do: Transmogrifier.fix_attachments(data) defp remote_mention_resolver( %{"id" => ap_id, "tag" => tags}, diff --git a/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex b/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex index 2e0f7a8a0..c5defd6f4 100644 --- a/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex +++ b/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex @@ -41,22 +41,34 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes do def fix_object_defaults(data) do data = Maps.filter_empty_values(data) + in_reply_to_id = in_reply_to_id(data["inReplyTo"]) + context = Utils.maybe_create_context( - data["context"] || data["conversation"] || data["inReplyTo"] || data["id"] + data["context"] || data["conversation"] || in_reply_to_id || data["id"] ) - %User{follower_address: follower_collection} = User.get_cached_by_ap_id(data["attributedTo"]) + data = Map.put(data, "context", context) - data - |> Map.put("context", context) - |> cast_and_filter_recipients("to", follower_collection) - |> cast_and_filter_recipients("cc", follower_collection) - |> cast_and_filter_recipients("bto", follower_collection) - |> cast_and_filter_recipients("bcc", follower_collection) - |> Transmogrifier.fix_implicit_addressing(follower_collection) + with attributed_to when is_binary(attributed_to) <- data["attributedTo"], + %User{follower_address: follower_collection} <- User.get_cached_by_ap_id(attributed_to) do + data + |> cast_and_filter_recipients("to", follower_collection) + |> cast_and_filter_recipients("cc", follower_collection) + |> cast_and_filter_recipients("bto", follower_collection) + |> cast_and_filter_recipients("bcc", follower_collection) + |> Transmogrifier.fix_implicit_addressing(follower_collection) + else + _ -> data + end end + defp in_reply_to_id(in_reply_to) when is_binary(in_reply_to), do: in_reply_to + defp in_reply_to_id(%{"id" => in_reply_to}) when is_binary(in_reply_to), do: in_reply_to + defp in_reply_to_id(%{"href" => in_reply_to}) when is_binary(in_reply_to), do: in_reply_to + defp in_reply_to_id([in_reply_to | _]), do: in_reply_to_id(in_reply_to) + defp in_reply_to_id(_), do: nil + def fix_activity_addressing(activity) do %User{follower_address: follower_collection} = User.get_cached_by_ap_id(activity["actor"]) @@ -79,6 +91,17 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes do |> Map.put("attributedTo", actor) end + def maybe_set_attributed_to_from_activity(object, activity) when is_map(object) do + if is_nil(object["attributedTo"]) and is_nil(object["actor"]) do + case Containment.get_actor(activity) do + actor when is_binary(actor) -> Map.put(object, "attributedTo", actor) + _ -> object + end + else + object + end + end + def fix_activity_context(data, %Object{data: %{"context" => object_context}}) do data |> Map.put("context", object_context) @@ -96,23 +119,37 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes do Map.put(data, "to", to) end - def fix_quote_url(%{"quoteUrl" => _quote_url} = data), do: data + def fix_quote_url(%{"quoteUrl" => quote_url} = data) do + case normalize_object_id(quote_url) do + quote_url when is_binary(quote_url) -> Map.put(data, "quoteUrl", quote_url) + _ -> Map.delete(data, "quoteUrl") + end + end # Fedibird # https://github.com/fedibird/mastodon/commit/dbd7ae6cf58a92ec67c512296b4daaea0d01e6ac def fix_quote_url(%{"quoteUri" => quote_url} = data) do - Map.put(data, "quoteUrl", quote_url) + case normalize_object_id(quote_url) do + quote_url when is_binary(quote_url) -> Map.put(data, "quoteUrl", quote_url) + _ -> data + end end # Old Fedibird (bug) # https://github.com/fedibird/mastodon/issues/9 def fix_quote_url(%{"quoteURL" => quote_url} = data) do - Map.put(data, "quoteUrl", quote_url) + case normalize_object_id(quote_url) do + quote_url when is_binary(quote_url) -> Map.put(data, "quoteUrl", quote_url) + _ -> data + end end # Misskey fallback def fix_quote_url(%{"_misskey_quote" => quote_url} = data) do - Map.put(data, "quoteUrl", quote_url) + case normalize_object_id(quote_url) do + quote_url when is_binary(quote_url) -> Map.put(data, "quoteUrl", quote_url) + _ -> data + end end def fix_quote_url(%{"tag" => [_ | _] = tags} = data) do @@ -128,6 +165,11 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes do def fix_quote_url(data), do: data + defp normalize_object_id(object_id) when is_binary(object_id), do: object_id + defp normalize_object_id(%{"id" => object_id}) when is_binary(object_id), do: object_id + defp normalize_object_id([object_id | _]) when is_binary(object_id), do: object_id + defp normalize_object_id(_), do: nil + # On Mastodon, `"likes"` attribute includes an inlined `Collection` with `totalItems`, # not a list of users. # https://github.com/mastodon/mastodon/pull/32007 diff --git a/lib/pleroma/web/activity_pub/object_validators/tag_validator.ex b/lib/pleroma/web/activity_pub/object_validators/tag_validator.ex index dc2770189..0659c93d7 100644 --- a/lib/pleroma/web/activity_pub/object_validators/tag_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/tag_validator.ex @@ -12,6 +12,8 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.TagValidator do require Pleroma.Constants @primary_key false + @tag_types ~w[Mention Hashtag Emoji Link] + embedded_schema do # Common field(:type, :string) @@ -49,7 +51,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.TagValidator do |> validate_required([:type, :href]) end - def changeset(struct, %{"type" => "Hashtag", "name" => name} = data) do + def changeset(struct, %{"type" => "Hashtag", "name" => name} = data) when is_binary(name) do name = String.downcase(name) data = Map.put(data, "name", name) @@ -58,13 +60,42 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.TagValidator do |> validate_required([:type, :name]) end - def changeset(struct, %{"type" => "Emoji"} = data) do - data = Map.put(data, "name", String.trim(data["name"], ":")) - + def changeset(struct, %{"type" => "Hashtag"} = data) do struct - |> cast(data, [:type, :name, :updated, :id]) - |> cast_embed(:icon, with: &icon_changeset/2) - |> validate_required([:type, :name, :icon]) + |> cast(data, []) + |> Map.put(:action, :ignore) + end + + def changeset(struct, %{"type" => "Emoji", "name" => name} = data) when is_binary(name) do + data = + data + |> Map.put("name", String.trim(data["name"], ":")) + |> normalize_emoji_icon() + + case data["icon"] do + %{"url" => url} when is_binary(url) -> + if valid_http_url?(url) do + struct + |> cast(data, [:type, :name, :updated, :id]) + |> cast_embed(:icon, with: &icon_changeset/2) + |> validate_required([:type, :name, :icon]) + else + struct + |> cast(data, []) + |> Map.put(:action, :ignore) + end + + _ -> + struct + |> cast(data, []) + |> Map.put(:action, :ignore) + end + end + + def changeset(struct, %{"type" => "Emoji"} = data) do + struct + |> cast(data, []) + |> Map.put(:action, :ignore) end def changeset(struct, %{"type" => "Link"} = data) do @@ -74,12 +105,80 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.TagValidator do |> validate_required([:type, :href, :mediaType]) end + def changeset(struct, %{"type" => types} = data) when is_list(types) do + changeset(struct, infer_type(data)) + end + def changeset(struct, %{"type" => _} = data) do struct |> cast(data, []) |> Map.put(:action, :ignore) end + def changeset(struct, data) when is_map(data) do + data = normalize(data) + + if Map.has_key?(data, "type") do + changeset(struct, data) + else + struct + |> cast(data, []) + |> Map.put(:action, :ignore) + end + end + + def normalize(data) when is_map(data), do: infer_type(data) + + defp infer_type(%{"type" => types} = data) when is_list(types) do + type = Enum.find(types, &(&1 in @tag_types)) || Enum.find(types, &is_binary/1) + + if type, do: Map.put(data, "type", type), else: Map.delete(data, "type") + end + + defp infer_type(%{"type" => _} = data), do: data + defp infer_type(%{"name" => "#" <> _} = data), do: Map.put(data, "type", "Hashtag") + defp infer_type(%{"name" => "@" <> _} = data), do: Map.put(data, "type", "Mention") + + defp infer_type(%{"mediaType" => _media_type, "href" => _href} = data), + do: Map.put(data, "type", "Link") + + defp infer_type(data), do: data + + defp normalize_emoji_icon(%{"icon" => icon} = data) when is_binary(icon) do + Map.put(data, "icon", %{"type" => "Image", "url" => icon}) + end + + defp normalize_emoji_icon(%{"icon" => icon} = data) when is_map(icon) do + case extract_icon_url(icon) do + url when is_binary(url) -> Map.put(data, "icon", %{"type" => "Image", "url" => url}) + _ -> Map.delete(data, "icon") + end + end + + defp normalize_emoji_icon(data), do: data + + defp extract_icon_url(%{"url" => url}) when is_binary(url), do: url + + defp extract_icon_url(%{"url" => %{"href" => href}}) when is_binary(href), do: href + + defp extract_icon_url(%{"url" => [first | _]}) do + cond do + is_binary(first) -> first + is_map(first) -> first["href"] + true -> nil + end + end + + defp extract_icon_url(%{"href" => href}) when is_binary(href), do: href + + defp extract_icon_url(_), do: nil + + defp valid_http_url?(url) when is_binary(url) do + match?({:ok, _}, ObjectValidators.Uri.cast(url)) + end + + defp valid_http_url?(_), do: false + def icon_changeset(struct, data) do struct |> cast(data, [:type, :url]) diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index a013dd226..33ab62ca3 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -19,6 +19,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do alias Pleroma.Web.ActivityPub.Builder alias Pleroma.Web.ActivityPub.ObjectValidator alias Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes + alias Pleroma.Web.ActivityPub.ObjectValidators.TagValidator alias Pleroma.Web.ActivityPub.Pipeline alias Pleroma.Web.ActivityPub.Utils alias Pleroma.Web.ActivityPub.Visibility @@ -28,6 +29,9 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do require Pleroma.Constants + @supported_types Pleroma.Constants.activity_types() ++ + Pleroma.Constants.object_types() ++ Pleroma.Constants.actor_types() + @doc """ Modifies an incoming AP object (mastodon format) to our internal format. """ @@ -201,9 +205,29 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do def fix_in_reply_to(%{"inReplyTo" => in_reply_to} = object, options) when not is_nil(in_reply_to) do in_reply_to_id = prepare_in_reply_to(in_reply_to) + + object = + cond do + is_list(in_reply_to) -> + if not_empty_string(in_reply_to_id) do + Map.put(object, "inReplyTo", in_reply_to_id) + else + Map.delete(object, "inReplyTo") + end + + is_map(in_reply_to) and is_nil(in_reply_to["id"]) and is_binary(in_reply_to["href"]) -> + Map.put(object, "inReplyTo", in_reply_to["href"]) + + is_map(in_reply_to) and is_nil(in_reply_to["id"]) and is_nil(in_reply_to["href"]) -> + Map.delete(object, "inReplyTo") + + true -> + object + end + depth = (options[:depth] || 0) + 1 - if Federator.allowed_thread_distance?(depth) do + if not_empty_string(in_reply_to_id) and Federator.allowed_thread_distance?(depth) do with {:ok, replied_object} <- get_obj_helper(in_reply_to_id, options), %Activity{} <- Activity.get_create_by_object_ap_id(replied_object.data["id"]) do object @@ -249,9 +273,17 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do is_map(in_reply_to) && is_bitstring(in_reply_to["id"]) -> in_reply_to["id"] + is_map(in_reply_to) && is_bitstring(in_reply_to["href"]) -> + in_reply_to["href"] + is_list(in_reply_to) && is_bitstring(Enum.at(in_reply_to, 0)) -> Enum.at(in_reply_to, 0) + is_list(in_reply_to) && is_map(Enum.at(in_reply_to, 0)) -> + in_reply_to + |> Enum.at(0) + |> prepare_in_reply_to() + true -> "" end @@ -265,59 +297,74 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do |> Map.drop(["conversation"]) end + defp valid_http_url?(href) when is_binary(href) do + match?({:ok, _}, ObjectValidators.Uri.cast(href)) + end + + defp valid_http_url?(_), do: false + def fix_attachments(%{"attachment" => attachment} = object) when is_list(attachment) do attachments = - Enum.map(attachment, fn data -> - url = - cond do - is_list(data["url"]) -> List.first(data["url"]) - is_map(data["url"]) -> data["url"] - true -> nil - end + Enum.map(attachment, fn + data when is_map(data) -> + url = + cond do + is_list(data["url"]) -> List.first(data["url"]) + is_map(data["url"]) -> data["url"] + true -> nil + end - media_type = - cond do - is_map(url) && url =~ Pleroma.Constants.mime_regex() -> - url["mediaType"] + url_data = if is_map(url), do: url, else: %{} - is_bitstring(data["mediaType"]) && data["mediaType"] =~ Pleroma.Constants.mime_regex() -> - data["mediaType"] + media_type = + cond do + is_bitstring(url_data["mediaType"]) && + url_data["mediaType"] =~ Pleroma.Constants.mime_regex() -> + url_data["mediaType"] - is_bitstring(data["mimeType"]) && data["mimeType"] =~ Pleroma.Constants.mime_regex() -> - data["mimeType"] + is_bitstring(data["mediaType"]) && + data["mediaType"] =~ Pleroma.Constants.mime_regex() -> + data["mediaType"] - true -> - nil - end + is_bitstring(data["mimeType"]) && data["mimeType"] =~ Pleroma.Constants.mime_regex() -> + data["mimeType"] - href = - cond do - is_map(url) && is_binary(url["href"]) -> url["href"] - is_binary(data["url"]) -> data["url"] - is_binary(data["href"]) -> data["href"] - true -> nil - end + true -> + nil + end + + href = + cond do + is_binary(url) -> url + is_binary(url_data["href"]) -> url_data["href"] + is_binary(data["url"]) -> data["url"] + is_binary(data["href"]) -> data["href"] + true -> nil + end + + if href && valid_http_url?(href) do + attachment_url = + %{ + "href" => href, + "type" => Map.get(url_data, "type", "Link") + } + |> Maps.put_if_present("mediaType", media_type) + |> Maps.put_if_present("width", url_data["width"] || data["width"]) + |> Maps.put_if_present("height", url_data["height"] || data["height"]) - if href do - attachment_url = %{ - "href" => href, - "type" => Map.get(url || %{}, "type", "Link") + "url" => [attachment_url], + "type" => data["type"] || "Document" } |> Maps.put_if_present("mediaType", media_type) - |> Maps.put_if_present("width", (url || %{})["width"] || data["width"]) - |> Maps.put_if_present("height", (url || %{})["height"] || data["height"]) + |> Maps.put_if_present("name", data["name"]) + |> Maps.put_if_present("blurhash", data["blurhash"]) + else + nil + end - %{ - "url" => [attachment_url], - "type" => data["type"] || "Document" - } - |> Maps.put_if_present("mediaType", media_type) - |> Maps.put_if_present("name", data["name"]) - |> Maps.put_if_present("blurhash", data["blurhash"]) - else + _ -> nil - end end) |> Enum.filter(& &1) @@ -351,24 +398,58 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do def fix_url(object), do: object + defp emoji_icon_url(icon) when is_binary(icon), do: icon + + defp emoji_icon_url(%{"url" => url}) when is_binary(url), do: url + + defp emoji_icon_url(%{"url" => %{"href" => href}}) when is_binary(href), do: href + + defp emoji_icon_url(%{"url" => [first | _]}) do + cond do + is_binary(first) -> first + is_map(first) -> first["href"] || first["url"] || first["id"] + true -> nil + end + end + + defp emoji_icon_url(%{"href" => href}) when is_binary(href), do: href + + defp emoji_icon_url(_), do: nil + + defp valid_emoji_icon_url(icon) do + case emoji_icon_url(icon) do + url when is_binary(url) -> if valid_http_url?(url), do: url + _ -> nil + end + end + def fix_emoji(%{"tag" => tags} = object) when is_list(tags) do emoji = tags - |> Enum.filter(fn data -> is_map(data) and data["type"] == "Emoji" and data["icon"] end) + |> Enum.filter(fn data -> + is_map(data) and data["type"] == "Emoji" and is_binary(data["name"]) and data["icon"] + end) |> Enum.reduce(%{}, fn data, mapping -> name = String.trim(data["name"], ":") - Map.put(mapping, name, data["icon"]["url"]) + case valid_emoji_icon_url(data["icon"]) do + url when is_binary(url) -> Map.put(mapping, name, url) + _ -> mapping + end end) Map.put(object, "emoji", emoji) end - def fix_emoji(%{"tag" => %{"type" => "Emoji"} = tag} = object) do - name = String.trim(tag["name"], ":") - emoji = %{name => tag["icon"]["url"]} + def fix_emoji(%{"tag" => %{"type" => "Emoji", "name" => name} = tag} = object) + when is_binary(name) do + name = String.trim(name, ":") - Map.put(object, "emoji", emoji) + with url when is_binary(url) <- valid_emoji_icon_url(tag["icon"]) do + Map.put(object, "emoji", %{name => url}) + else + _ -> object + end end def fix_emoji(object), do: object @@ -376,7 +457,9 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do def fix_tag(%{"tag" => tag} = object) when is_list(tag) do tags = tag - |> Enum.filter(fn data -> data["type"] == "Hashtag" and data["name"] end) + |> Enum.filter(fn data -> + is_map(data) and data["type"] == "Hashtag" and is_binary(data["name"]) + end) |> Enum.map(fn %{"name" => "#" <> hashtag} -> String.downcase(hashtag) %{"name" => hashtag} -> String.downcase(hashtag) @@ -439,9 +522,37 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do def handle_incoming(data, options \\ []) do data |> fix_recursive(&strip_internal_fields/1) + |> fix_recursive(&fix_type_as_string/1) |> handle_incoming_normalized(options) end + defp fix_type_as_string(data) do + data + |> normalize_type() + |> normalize_tag_types() + end + + defp normalize_type(%{"type" => types} = data) when is_list(types) do + type = Enum.find(types, &(&1 in @supported_types)) || Enum.find(types, &is_binary/1) + + if type, do: Map.put(data, "type", type), else: data + end + + defp normalize_type(data), do: data + + defp normalize_tag_types(%{"tag" => tags} = data) when is_list(tags) do + Map.put(data, "tag", Enum.map(tags, &normalize_tag_type/1)) + end + + defp normalize_tag_types(%{"tag" => tag} = data) when is_map(tag) do + Map.put(data, "tag", normalize_tag_type(tag)) + end + + defp normalize_tag_types(data), do: data + + defp normalize_tag_type(tag) when is_map(tag), do: TagValidator.normalize(tag) + defp normalize_tag_type(tag), do: tag + # Flag objects are placed ahead of the ID check because Mastodon 2.8 and earlier send them # with nil ID. defp handle_incoming_normalized( diff --git a/lib/pleroma/web/activity_pub/views/user_view.ex b/lib/pleroma/web/activity_pub/views/user_view.ex index 8847692c8..c273043e1 100644 --- a/lib/pleroma/web/activity_pub/views/user_view.ex +++ b/lib/pleroma/web/activity_pub/views/user_view.ex @@ -68,7 +68,7 @@ defmodule Pleroma.Web.ActivityPub.UserView do capabilities = if is_boolean(user.accepts_chat_messages) do %{ - "acceptsChatMessages" => user.accepts_chat_messages + "acceptsChatMessages" => Pleroma.Chat.enabled?() and user.accepts_chat_messages } else %{} diff --git a/lib/pleroma/web/admin_api/controllers/config_controller.ex b/lib/pleroma/web/admin_api/controllers/config_controller.ex index 3eba03525..4c76d808d 100644 --- a/lib/pleroma/web/admin_api/controllers/config_controller.ex +++ b/lib/pleroma/web/admin_api/controllers/config_controller.ex @@ -71,7 +71,11 @@ defmodule Pleroma.Web.AdminAPI.ConfigController do end def descriptions(conn, _params) do - descriptions = Enum.filter(Pleroma.Docs.JSON.compiled_descriptions(), &whitelisted_config?/1) + descriptions = + Pleroma.Docs.JSON.compiled_descriptions() + |> Enum.filter(&allowed_config?(&1, :whitelist)) + |> Enum.reject(&allowed_config?(&1, :blacklist)) + |> Enum.reject(&ConfigDB.static?/1) json(conn, translate_descriptions(descriptions)) end @@ -89,7 +93,7 @@ defmodule Pleroma.Web.AdminAPI.ConfigController do def show(conn, _params) do with :ok <- configurable_from_database() do - configs = ConfigDB.get_all_as_keyword() + configs = ConfigDB.get_all_as_keyword(exclude_static: true) merged = Config.Holder.default_config() @@ -120,6 +124,7 @@ defmodule Pleroma.Web.AdminAPI.ConfigController do end) end) |> List.flatten() + |> Enum.reject(&ConfigDB.static?/1) render(conn, "index.json", %{ configs: merged, @@ -132,7 +137,9 @@ defmodule Pleroma.Web.AdminAPI.ConfigController do with :ok <- configurable_from_database() do results = configs - |> Enum.filter(&whitelisted_config?/1) + |> Enum.filter(&allowed_config?(&1, :whitelist)) + |> Enum.reject(&allowed_config?(&1, :blacklist)) + |> Enum.reject(&(ConfigDB.static?(&1) and not &1[:delete])) |> Enum.map(fn %{group: group, key: key, delete: true} = params -> ConfigDB.delete(%{group: group, key: key, subkeys: params[:subkeys]}) @@ -174,27 +181,34 @@ defmodule Pleroma.Web.AdminAPI.ConfigController do end end - defp whitelisted_config?(":pleroma", ":database_config_whitelist"), do: false + defp allowed_config?(":pleroma", ":database_config_whitelist", _), do: true + defp allowed_config?(":pleroma", ":database_config_blacklist", _), do: true - defp whitelisted_config?(group, key) do - if whitelisted_configs = Config.get(:database_config_whitelist) do - Enum.any?(whitelisted_configs, fn - {whitelisted_group} -> - group == inspect(whitelisted_group) + defp allowed_config?(group, key, mode) do + configs = + case mode do + :whitelist -> Config.get(:database_config_whitelist) + :blacklist -> Config.get(:database_config_blacklist) + end - {whitelisted_group, whitelisted_key} -> - group == inspect(whitelisted_group) && key == inspect(whitelisted_key) + if configs do + Enum.any?(configs, fn + {allowed_group} -> + group == inspect(allowed_group) + + {allowed_group, allowed_key} -> + group == inspect(allowed_group) && key == inspect(allowed_key) end) else true end end - defp whitelisted_config?(%{group: group, key: key}) do - whitelisted_config?(group, key) + defp allowed_config?(%{group: group, key: key}, mode) do + allowed_config?(group, key, mode) end - defp whitelisted_config?(%{group: group} = config) do - whitelisted_config?(group, config[:key]) + defp allowed_config?(%{group: group} = config, mode) do + allowed_config?(group, config[:key], mode) end end diff --git a/lib/pleroma/web/admin_api/controllers/fallback_controller.ex b/lib/pleroma/web/admin_api/controllers/fallback_controller.ex index e72f45c21..d8fb54cad 100644 --- a/lib/pleroma/web/admin_api/controllers/fallback_controller.ex +++ b/lib/pleroma/web/admin_api/controllers/fallback_controller.ex @@ -11,6 +11,12 @@ defmodule Pleroma.Web.AdminAPI.FallbackController do |> json(%{error: dgettext("errors", "Not found")}) end + def call(conn, {:error, :forbidden}) do + conn + |> put_status(:forbidden) + |> json(%{error: dgettext("errors", "Forbidden")}) + end + def call(conn, {:error, reason}) do conn |> put_status(:bad_request) diff --git a/lib/pleroma/web/admin_api/controllers/user_controller.ex b/lib/pleroma/web/admin_api/controllers/user_controller.ex index 9ac275396..ab8bf73ee 100644 --- a/lib/pleroma/web/admin_api/controllers/user_controller.ex +++ b/lib/pleroma/web/admin_api/controllers/user_controller.ex @@ -8,13 +8,17 @@ defmodule Pleroma.Web.AdminAPI.UserController do import Pleroma.Web.ControllerHelper, only: [fetch_integer_param: 3] + alias Ecto.Multi alias Pleroma.ModerationLog + alias Pleroma.PasswordResetToken alias Pleroma.User alias Pleroma.Web.ActivityPub.Builder alias Pleroma.Web.ActivityPub.Pipeline alias Pleroma.Web.AdminAPI alias Pleroma.Web.AdminAPI.Search + alias Pleroma.Web.Endpoint alias Pleroma.Web.Plugs.OAuthScopesPlug + alias Pleroma.Web.Router @users_page_size 50 @@ -148,9 +152,25 @@ defmodule Pleroma.Web.AdminAPI.UserController do } = conn, _ ) do - changesets = + conn = + if Enum.any?(users, fn user -> not Map.has_key?(user, :password) end) do + put_private(conn, :skip_idempotency_cache, true) + else + conn + end + + multi = users - |> Enum.map(fn %{nickname: nickname, email: email, password: password} -> + |> Enum.map(fn %{nickname: nickname, email: email} = attrs -> + {password, passwordless?} = + case Map.fetch(attrs, :password) do + {:ok, password} -> + {password, false} + + :error -> + {:crypto.strong_rand_bytes(16) |> Base.encode64(), true} + end + user_data = %{ nickname: nickname, name: nickname, @@ -160,44 +180,80 @@ defmodule Pleroma.Web.AdminAPI.UserController do bio: "." } - User.register_changeset(%User{}, user_data, need_confirmation: false) + {User.register_changeset(%User{}, user_data, confirmed: true), passwordless?} end) - |> Enum.reduce(Ecto.Multi.new(), fn changeset, multi -> - Ecto.Multi.insert(multi, Ecto.UUID.generate(), changeset) + |> Enum.reduce(Multi.new(), fn {changeset, passwordless?}, multi -> + user_operation = {:user, Ecto.UUID.generate()} + multi = Multi.insert(multi, user_operation, changeset) + + if passwordless? do + Multi.run( + multi, + {:password_reset_token, Ecto.UUID.generate()}, + fn _repo, changes -> + changes + |> Map.fetch!(user_operation) + |> PasswordResetToken.create_token() + end + ) + else + multi + end end) - case Pleroma.Repo.transaction(changesets) do - {:ok, users_map} -> - users = - users_map + case Pleroma.Repo.transaction(multi) do + {:ok, results} -> + {users, password_reset_tokens} = + results |> Map.values() + |> Enum.split_with(&match?(%User{}, &1)) + + users = + users |> Enum.map(fn user -> {:ok, user} = User.post_register_action(user) user end) + password_reset_links = + Map.new(password_reset_tokens, fn token -> + {token.user_id, Router.Helpers.reset_password_url(Endpoint, :reset, token.token)} + end) + ModerationLog.insert_log(%{ actor: admin, subjects: users, action: "create" }) - render(conn, "created_many.json", users: users) + render(conn, "created_many.json", + users: users, + password_reset_links: password_reset_links + ) - {:error, id, changeset, _} -> + {:error, {:user, _} = id, %Ecto.Changeset{} = changeset, _} -> changesets = - Enum.map(changesets.operations, fn + Enum.flat_map(multi.operations, fn {^id, {:changeset, _current_changeset, _}} -> - changeset + [changeset] - {_, {:changeset, current_changeset, _}} -> - current_changeset + {{:user, _}, {:changeset, current_changeset, _}} -> + [current_changeset] + + _ -> + [] end) conn |> put_status(:conflict) |> render("create_errors.json", changesets: changesets) + + {:error, {:password_reset_token, _}, _reason, _} -> + Pleroma.Web.AdminAPI.FallbackController.call( + conn, + {:password_reset_token_error, :creation_failed} + ) end end diff --git a/lib/pleroma/web/admin_api/controllers/webhook_controller.ex b/lib/pleroma/web/admin_api/controllers/webhook_controller.ex new file mode 100644 index 000000000..4d064e0b5 --- /dev/null +++ b/lib/pleroma/web/admin_api/controllers/webhook_controller.ex @@ -0,0 +1,101 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.AdminAPI.WebhookController do + use Pleroma.Web, :controller + + alias Pleroma.Repo + alias Pleroma.Web.Plugs.OAuthScopesPlug + alias Pleroma.Webhook + + plug(Pleroma.Web.ApiSpec.CastAndValidate) + + plug( + OAuthScopesPlug, + %{scopes: ["admin:write"]} + when action in [:update, :create, :delete, :enable, :disable, :rotate_secret] + ) + + plug(OAuthScopesPlug, %{scopes: ["admin:read"]} when action in [:index, :show]) + + action_fallback(Pleroma.Web.AdminAPI.FallbackController) + + defdelegate open_api_operation(action), to: Pleroma.Web.ApiSpec.Admin.WebhookOperation + + def index(conn, _) do + webhooks = + Webhook + |> Repo.all() + + render(conn, "index.json", webhooks: webhooks) + end + + def show(conn, %{id: id}) do + with %Webhook{} = webhook <- Webhook.get(id) do + render(conn, "show.json", webhook: webhook) + else + nil -> {:error, :not_found} + end + end + + def create(%{body_params: params} = conn, _) do + with {:ok, webhook} <- Webhook.create(webhook_params(params)) do + render(conn, "show.json", webhook: webhook) + end + end + + def update(%{body_params: params} = conn, %{id: id}) do + with %Webhook{internal: false} = webhook <- Webhook.get(id), + {:ok, webhook} <- Webhook.update(webhook, webhook_params(params)) do + render(conn, "show.json", webhook: webhook) + else + %Webhook{internal: true} -> {:error, :forbidden} + nil -> {:error, :not_found} + end + end + + def delete(conn, %{id: id}) do + with %Webhook{internal: false} = webhook <- Webhook.get(id), + {:ok, webhook} <- Webhook.delete(webhook) do + render(conn, "show.json", webhook: webhook) + else + %Webhook{internal: true} -> {:error, :forbidden} + nil -> {:error, :not_found} + end + end + + def enable(conn, %{id: id}) do + with %Webhook{internal: false} = webhook <- Webhook.get(id), + {:ok, webhook} <- Webhook.set_enabled(webhook, true) do + render(conn, "show.json", webhook: webhook) + else + %Webhook{internal: true} -> {:error, :forbidden} + nil -> {:error, :not_found} + end + end + + def disable(conn, %{id: id}) do + with %Webhook{internal: false} = webhook <- Webhook.get(id), + {:ok, webhook} <- Webhook.set_enabled(webhook, false) do + render(conn, "show.json", webhook: webhook) + else + %Webhook{internal: true} -> {:error, :forbidden} + nil -> {:error, :not_found} + end + end + + def rotate_secret(conn, %{id: id}) do + with %Webhook{internal: false} = webhook <- Webhook.get(id), + {:ok, webhook} <- Webhook.rotate_secret(webhook) do + render(conn, "show.json", webhook: webhook) + else + %Webhook{internal: true} -> {:error, :forbidden} + nil -> {:error, :not_found} + end + end + + defp webhook_params(params) do + Map.drop(params, [:internal, "internal"]) + end +end diff --git a/lib/pleroma/web/admin_api/views/account_view.ex b/lib/pleroma/web/admin_api/views/account_view.ex index 280152241..cc715c2a5 100644 --- a/lib/pleroma/web/admin_api/views/account_view.ex +++ b/lib/pleroma/web/admin_api/views/account_view.ex @@ -88,18 +88,29 @@ defmodule Pleroma.Web.AdminAPI.AccountView do } end - def render("created_many.json", %{users: users}) do - render_many(users, AccountView, "created.json", as: :user) + def render("created_many.json", %{users: users} = assigns) do + password_reset_links = Map.get(assigns, :password_reset_links, %{}) + + render_many(users, AccountView, "created.json", + as: :user, + password_reset_links: password_reset_links + ) end - def render("created.json", %{user: user}) do - %{ - type: "success", - code: 200, - data: %{ + def render("created.json", %{user: user} = assigns) do + password_reset_links = Map.get(assigns, :password_reset_links, %{}) + + data = + %{ nickname: user.nickname, email: user.email } + |> maybe_put_password_reset_link(Map.get(password_reset_links, user.id)) + + %{ + type: "success", + code: 200, + data: data } end @@ -119,6 +130,11 @@ defmodule Pleroma.Web.AdminAPI.AccountView do } end + defp maybe_put_password_reset_link(data, nil), do: data + + defp maybe_put_password_reset_link(data, link), + do: Map.put(data, :password_reset_link, link) + def merge_account_views(%User{} = user) do MastodonAPI.AccountView.render("show.json", %{user: user, skip_visibility_check: true}) |> Map.merge(AdminAPI.AccountView.render("show.json", %{user: user})) diff --git a/lib/pleroma/web/admin_api/views/webhook_view.ex b/lib/pleroma/web/admin_api/views/webhook_view.ex new file mode 100644 index 000000000..0b839090b --- /dev/null +++ b/lib/pleroma/web/admin_api/views/webhook_view.ex @@ -0,0 +1,34 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.AdminAPI.WebhookView do + use Pleroma.Web, :view + + alias Pleroma.Web.CommonAPI.Utils + + def render("index.json", %{webhooks: webhooks}) do + render_many(webhooks, __MODULE__, "show.json") + end + + def render("show.json", %{webhook: webhook}) do + %{ + id: webhook.id |> to_string(), + url: webhook.url, + events: webhook.events, + secret: webhook.secret, + enabled: webhook.enabled, + internal: webhook.internal, + created_at: Utils.to_masto_date(webhook.inserted_at), + updated_at: Utils.to_masto_date(webhook.updated_at) + } + end + + def render("event.json", %{type: type, object: object}) do + %{ + event: type, + created_at: Utils.to_masto_date(NaiveDateTime.utc_now()), + object: object + } + end +end diff --git a/lib/pleroma/web/api_spec.ex b/lib/pleroma/web/api_spec.ex index 5006167ea..e6741b74b 100644 --- a/lib/pleroma/web/api_spec.ex +++ b/lib/pleroma/web/api_spec.ex @@ -105,7 +105,15 @@ defmodule Pleroma.Web.ApiSpec do "Report management", "Status administration", "User administration", - "Announcement management" + "Announcement management", + "Webhooks" + ] + }, + %{ + "name" => "Administration (Mastodon API)", + "tags" => [ + "User administration (Mastodon API)", + "Report management (Mastodon API)" ] }, %{"name" => "Applications", "tags" => ["Applications", "Push subscriptions"]}, diff --git a/lib/pleroma/web/api_spec/operations/admin/user_operation.ex b/lib/pleroma/web/api_spec/operations/admin/user_operation.ex index a5179ac39..7c4725f56 100644 --- a/lib/pleroma/web/api_spec/operations/admin/user_operation.ex +++ b/lib/pleroma/web/api_spec/operations/admin/user_operation.ex @@ -82,8 +82,15 @@ defmodule Pleroma.Web.ApiSpec.Admin.UserOperation do properties: %{ nickname: %Schema{type: :string}, email: %Schema{type: :string}, - password: %Schema{type: :string} - } + password: %Schema{ + type: :string, + description: + "Optional. When omitted, a `password_reset_link` is returned, " <> + "allowing the user to set their own password. When provided, " <> + "it must not be empty." + } + }, + required: [:nickname, :email] } } } @@ -102,7 +109,8 @@ defmodule Pleroma.Web.ApiSpec.Admin.UserOperation do type: :object, properties: %{ email: %Schema{type: :string, format: :email}, - nickname: %Schema{type: :string} + nickname: %Schema{type: :string}, + password_reset_link: %Schema{type: :string} } } } diff --git a/lib/pleroma/web/api_spec/operations/admin/webhook_operation.ex b/lib/pleroma/web/api_spec/operations/admin/webhook_operation.ex new file mode 100644 index 000000000..f444857f0 --- /dev/null +++ b/lib/pleroma/web/api_spec/operations/admin/webhook_operation.ex @@ -0,0 +1,208 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ApiSpec.Admin.WebhookOperation do + alias OpenApiSpex.Operation + alias OpenApiSpex.Schema + alias Pleroma.Web.ApiSpec.Schemas.ApiError + + import Pleroma.Web.ApiSpec.Helpers + + def open_api_operation(action) do + operation = String.to_existing_atom("#{action}_operation") + apply(__MODULE__, operation, []) + end + + def index_operation do + %Operation{ + tags: ["Webhooks"], + summary: "Retrieve a list of webhooks", + operationId: "AdminAPI.WebhookController.index", + security: [%{"oAuth" => ["admin:read"]}], + responses: %{ + 200 => + Operation.response("Array of webhooks", "application/json", %Schema{ + type: :array, + items: webhook() + }) + } + } + end + + def show_operation do + %Operation{ + tags: ["Webhooks"], + summary: "Retrieve a webhook", + operationId: "AdminAPI.WebhookController.show", + security: [%{"oAuth" => ["admin:read"]}], + parameters: [id_param()], + responses: %{ + 200 => Operation.response("Webhook", "application/json", webhook()) + } + } + end + + def create_operation do + %Operation{ + tags: ["Webhooks"], + summary: "Create a webhook", + operationId: "AdminAPI.WebhookController.create", + security: [%{"oAuth" => ["admin:write"]}], + requestBody: + request_body( + "Parameters", + %Schema{ + description: "POST body for creating a webhook", + type: :object, + required: [:url, :events], + properties: %{ + url: %Schema{type: :string, format: :uri, maxLength: 255}, + events: event_type(min_items: 1), + enabled: %Schema{type: :boolean} + }, + additionalProperties: false + } + ), + responses: %{ + 200 => Operation.response("Webhook", "application/json", webhook()) + } + } + end + + def update_operation do + %Operation{ + tags: ["Webhooks"], + summary: "Update a webhook", + operationId: "AdminAPI.WebhookController.update", + security: [%{"oAuth" => ["admin:write"]}], + parameters: [id_param()], + requestBody: + request_body( + "Parameters", + %Schema{ + description: "POST body for updating a webhook", + type: :object, + properties: %{ + url: %Schema{type: :string, format: :uri, maxLength: 255}, + events: event_type(min_items: 1), + enabled: %Schema{type: :boolean} + }, + additionalProperties: false + } + ), + responses: %{ + 200 => Operation.response("Webhook", "application/json", webhook()), + 403 => Operation.response("Forbidden", "application/json", ApiError) + } + } + end + + def delete_operation do + %Operation{ + tags: ["Webhooks"], + summary: "Delete a webhook", + operationId: "AdminAPI.WebhookController.delete", + security: [%{"oAuth" => ["admin:write"]}], + parameters: [id_param()], + responses: %{ + 200 => Operation.response("Webhook", "application/json", webhook()), + 403 => Operation.response("Forbidden", "application/json", ApiError) + } + } + end + + def enable_operation do + %Operation{ + tags: ["Webhooks"], + summary: "Enable a webhook", + operationId: "AdminAPI.WebhookController.enable", + security: [%{"oAuth" => ["admin:write"]}], + parameters: [id_param()], + responses: %{ + 200 => Operation.response("Webhook", "application/json", webhook()), + 403 => Operation.response("Forbidden", "application/json", ApiError) + } + } + end + + def disable_operation do + %Operation{ + tags: ["Webhooks"], + summary: "Disable a webhook", + operationId: "AdminAPI.WebhookController.disable", + security: [%{"oAuth" => ["admin:write"]}], + parameters: [id_param()], + responses: %{ + 200 => Operation.response("Webhook", "application/json", webhook()), + 403 => Operation.response("Forbidden", "application/json", ApiError) + } + } + end + + def rotate_secret_operation do + %Operation{ + tags: ["Webhooks"], + summary: "Rotate webhook signing secret", + operationId: "AdminAPI.WebhookController.rotate_secret", + security: [%{"oAuth" => ["admin:write"]}], + parameters: [id_param()], + responses: %{ + 200 => Operation.response("Webhook", "application/json", webhook()), + 403 => Operation.response("Forbidden", "application/json", ApiError) + } + } + end + + defp webhook do + %Schema{ + title: "Webhook", + description: "Schema for a webhook", + type: :object, + properties: %{ + id: %Schema{type: :string}, + url: %Schema{type: :string, format: :uri, maxLength: 255}, + events: event_type(), + secret: %Schema{type: :string}, + enabled: %Schema{type: :boolean}, + internal: %Schema{type: :boolean}, + created_at: %Schema{type: :string, format: :"date-time"}, + updated_at: %Schema{type: :string, format: :"date-time"} + }, + example: %{ + "id" => "1", + "url" => "https://example.com/webhook", + "events" => ["report.created"], + "secret" => "d3d8cf4bc11fd9c41fd34dcc38d282e451c8bd34", + "enabled" => true, + "internal" => false, + "created_at" => "2022-06-24T16:19:38.523Z", + "updated_at" => "2022-06-24T16:19:38.523Z" + } + } + end + + defp event_type(opts \\ []) do + schema = %Schema{ + type: :array, + items: %Schema{ + title: "Event", + description: "Event type", + type: :string, + enum: ["account.created", "report.created"] + } + } + + case Keyword.fetch(opts, :min_items) do + {:ok, min_items} -> %{schema | minItems: min_items} + :error -> schema + end + end + + defp id_param do + Operation.parameter(:id, :path, :string, "Webhook ID", + example: "123", + required: true + ) + end +end diff --git a/lib/pleroma/web/api_spec/operations/chat_operation.ex b/lib/pleroma/web/api_spec/operations/chat_operation.ex index 76c2db57e..7bc39acc6 100644 --- a/lib/pleroma/web/api_spec/operations/chat_operation.ex +++ b/lib/pleroma/web/api_spec/operations/chat_operation.ex @@ -31,7 +31,8 @@ defmodule Pleroma.Web.ApiSpec.ChatOperation do "The updated chat", "application/json", Chat - ) + ), + 404 => Operation.response("Not Found", "application/json", ApiError) }, security: [ %{ @@ -56,7 +57,8 @@ defmodule Pleroma.Web.ApiSpec.ChatOperation do "The read ChatMessage", "application/json", ChatMessage - ) + ), + 404 => Operation.response("Not Found", "application/json", ApiError) }, security: [ %{ @@ -87,7 +89,8 @@ defmodule Pleroma.Web.ApiSpec.ChatOperation do "The existing chat", "application/json", Chat - ) + ), + 404 => Operation.response("Not Found", "application/json", ApiError) }, security: [ %{ @@ -118,7 +121,8 @@ defmodule Pleroma.Web.ApiSpec.ChatOperation do "The created or existing chat", "application/json", Chat - ) + ), + 404 => Operation.response("Not Found", "application/json", ApiError) }, security: [ %{ @@ -146,7 +150,8 @@ defmodule Pleroma.Web.ApiSpec.ChatOperation do Operation.parameter(:pinned, :query, BooleanLike.schema(), "Include only pinned chats") ], responses: %{ - 200 => Operation.response("The chats of the user", "application/json", chats_response()) + 200 => Operation.response("The chats of the user", "application/json", chats_response()), + 404 => Operation.response("Not Found", "application/json", ApiError) }, security: [ %{ @@ -172,7 +177,8 @@ defmodule Pleroma.Web.ApiSpec.ChatOperation do | pagination_params() ], responses: %{ - 200 => Operation.response("The chats of the user", "application/json", chats_response()) + 200 => Operation.response("The chats of the user", "application/json", chats_response()), + 404 => Operation.response("Not Found", "application/json", ApiError) }, security: [ %{ @@ -224,6 +230,7 @@ defmodule Pleroma.Web.ApiSpec.ChatOperation do ChatMessage ), 400 => Operation.response("Bad Request", "application/json", ApiError), + 404 => Operation.response("Not Found", "application/json", ApiError), 422 => Operation.response("MRF Rejection", "application/json", ApiError) }, security: [ @@ -249,7 +256,8 @@ defmodule Pleroma.Web.ApiSpec.ChatOperation do "The deleted ChatMessage", "application/json", ChatMessage - ) + ), + 404 => Operation.response("Not Found", "application/json", ApiError) }, security: [ %{ @@ -268,7 +276,8 @@ defmodule Pleroma.Web.ApiSpec.ChatOperation do Operation.parameter(:id, :path, :string, "The id of the chat", required: true) ], responses: %{ - 200 => Operation.response("The existing chat", "application/json", Chat) + 200 => Operation.response("The existing chat", "application/json", Chat), + 404 => Operation.response("Not Found", "application/json", ApiError) }, security: [ %{ @@ -287,7 +296,8 @@ defmodule Pleroma.Web.ApiSpec.ChatOperation do Operation.parameter(:id, :path, :string, "The id of the chat", required: true) ], responses: %{ - 200 => Operation.response("The existing chat", "application/json", Chat) + 200 => Operation.response("The existing chat", "application/json", Chat), + 404 => Operation.response("Not Found", "application/json", ApiError) }, security: [ %{ diff --git a/lib/pleroma/web/api_spec/operations/list_operation.ex b/lib/pleroma/web/api_spec/operations/list_operation.ex index 87189edc2..8aec1c352 100644 --- a/lib/pleroma/web/api_spec/operations/list_operation.ex +++ b/lib/pleroma/web/api_spec/operations/list_operation.ex @@ -41,7 +41,8 @@ defmodule Pleroma.Web.ApiSpec.ListOperation do responses: %{ 200 => Operation.response("List", "application/json", List), 400 => Operation.response("Error", "application/json", ApiError), - 404 => Operation.response("Error", "application/json", ApiError) + 404 => Operation.response("Error", "application/json", ApiError), + 422 => Operation.response("Error", "application/json", ApiError) } } end @@ -175,7 +176,8 @@ defmodule Pleroma.Web.ApiSpec.ListOperation do exclusive: %Schema{ type: :boolean, description: "Whether members of the list should be removed from the “Home” feed" - } + }, + emoji: %Schema{type: :string, description: "List emoji", nullable: true} }, required: [:title] }, @@ -194,7 +196,8 @@ defmodule Pleroma.Web.ApiSpec.ListOperation do exclusive: %Schema{ type: :boolean, description: "Whether members of the list should be removed from the “Home” feed" - } + }, + emoji: %Schema{type: :string, description: "List emoji", nullable: true} } }, required: true diff --git a/lib/pleroma/web/api_spec/operations/mastodon_admin/account_operation.ex b/lib/pleroma/web/api_spec/operations/mastodon_admin/account_operation.ex new file mode 100644 index 000000000..3b39cbe2a --- /dev/null +++ b/lib/pleroma/web/api_spec/operations/mastodon_admin/account_operation.ex @@ -0,0 +1,348 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ApiSpec.MastodonAdmin.AccountOperation do + alias OpenApiSpex.Operation + alias OpenApiSpex.Schema + alias Pleroma.Web.ApiSpec.Schemas.Account + alias Pleroma.Web.ApiSpec.Schemas.ApiError + alias Pleroma.Web.ApiSpec.Schemas.FlakeID + + import Pleroma.Web.ApiSpec.Helpers + + def open_api_operation(action) do + operation = String.to_existing_atom("#{action}_operation") + apply(__MODULE__, operation, []) + end + + def index_operation do + %Operation{ + tags: ["User administration (Mastodon API)"], + summary: "View accounts by criteria (v1)", + operationId: "MastodonAdmin.AccountController.index", + description: "View accounts matching certain criteria for filtering, up to 40 at a time.", + security: [%{"oAuth" => ["admin:read:accounts"]}], + parameters: + [ + Operation.parameter(:local, :query, :boolean, "Filter for local accounts?"), + Operation.parameter(:remote, :query, :boolean, "Filter for remote accounts?"), + Operation.parameter( + :active, + :query, + :boolean, + "Filter for currently active accounts?" + ), + Operation.parameter( + :pending, + :query, + :boolean, + "Filter for currently pending accounts?" + ), + Operation.parameter( + :disabled, + :query, + :boolean, + "Filter for currently disabled accounts?" + ), + Operation.parameter( + :silenced, + :query, + :boolean, + "Filter for currently silenced accounts? (not implemented yet)" + ), + Operation.parameter( + :suspended, + :query, + :boolean, + "Filter for currently suspended accounts? (not implemented yet)" + ), + Operation.parameter( + :sensitized, + :query, + :boolean, + "Filter for accounts force-marked as sensitive? (not implemented yet)" + ), + Operation.parameter(:username, :query, :string, "Search for the given username"), + Operation.parameter( + :display_name, + :query, + :string, + "Search for the given display name" + ), + Operation.parameter( + :by_domain, + :query, + :string, + "Filter by the given domain" + ), + Operation.parameter(:email, :query, :string, "Lookup a user with this email"), + Operation.parameter( + :ip, + :query, + :string, + "Lookup users with this IP address (not implemented yet)" + ), + Operation.parameter(:staff, :query, :boolean, "Filter for staff accounts?") + ] ++ + pagination_params(), + responses: %{ + 200 => + Operation.response("Account", "application/json", %Schema{ + title: "ArrayOfAccounts", + type: :array, + items: account() + }), + 401 => Operation.response("Error", "application/json", ApiError) + } + } + end + + def index2_operation do + %Operation{ + tags: ["User administration (Mastodon API)"], + summary: "View accounts by criteria (v2)", + operationId: "MastodonAdmin.AccountController.index2", + description: "View accounts matching certain criteria for filtering, up to 40 at a time.", + security: [%{"oAuth" => ["admin:read:accounts"]}], + parameters: + [ + Operation.parameter( + :origin, + :query, + %Schema{type: :string, enum: ["local", "remote"]}, + "Filter for local or remote accounts" + ), + Operation.parameter( + :status, + :query, + %Schema{ + type: :string, + enum: ["active", "inactive", "pending", "disabled", "silenced", "suspended"] + }, + "Filter for active, pending, disabled, silenced or suspended accounts" + ), + Operation.parameter( + :permissions, + :query, + :string, + "Filter for accounts with staff permissions (users that can manage reports)." + ), + Operation.parameter( + :role_ids, + :query, + %Schema{ + oneOf: [ + %Schema{type: :array, items: %Schema{type: :string}}, + %Schema{type: :string} + ] + }, + "Filter for users with these roles. (not implemented yet)" + ), + Operation.parameter( + :invited_by, + :query, + :string, + "Lookup users invited by the account with this ID. (not implemented yet)" + ), + Operation.parameter(:username, :query, :string, "Search for the given username"), + Operation.parameter( + :display_name, + :query, + :string, + "Search for the given display name" + ), + Operation.parameter( + :by_domain, + :query, + :string, + "Filter by the given domain" + ), + Operation.parameter(:email, :query, :string, "Lookup a user with this email"), + Operation.parameter( + :ip, + :query, + :string, + "Lookup users with this IP address (not implemented yet)" + ) + ] ++ + pagination_params(), + responses: %{ + 200 => + Operation.response("Account", "application/json", %Schema{ + title: "ArrayOfAccounts", + type: :array, + items: account() + }), + 401 => Operation.response("Error", "application/json", ApiError) + } + } + end + + def show_operation do + %Operation{ + tags: ["User administration (Mastodon API)"], + summary: "View a specific account", + operationId: "MastodonAdmin.AccountController.show", + description: "View admin-level information about the given account.", + security: [%{"oAuth" => ["admin:read:accounts"]}], + parameters: [ + Operation.parameter(:id, :path, :string, "ID of the account") + ], + responses: %{ + 200 => Operation.response("Account", "application/json", account()), + 401 => Operation.response("Error", "application/json", ApiError), + 404 => Operation.response("Error", "application/json", ApiError) + } + } + end + + def account_action_operation do + %Operation{ + tags: ["User administration (Mastodon API)"], + summary: "Perform an action against an account", + operationId: "MastodonAdmin.AccountController.account_action", + description: + "Perform an action against an account and log this action in the moderation history.", + security: [%{"oAuth" => ["admin:write:accounts"]}], + parameters: [ + Operation.parameter(:id, :path, :string, "ID of the account") + ], + requestBody: + request_body( + "Parameters", + %Schema{ + type: :object, + properties: %{ + type: %Schema{ + type: :string, + enum: ["none", "disable"] + }, + report_id: %Schema{ + type: :string, + nullable: true, + description: "ID of an associated report that caused this action to be taken" + } + }, + required: [:type] + }, + required: true + ), + responses: %{ + 204 => no_content_response(), + 400 => Operation.response("Error", "application/json", ApiError), + 401 => Operation.response("Error", "application/json", ApiError), + 403 => Operation.response("Error", "application/json", ApiError), + 404 => Operation.response("Error", "application/json", ApiError) + } + } + end + + def delete_operation do + %Operation{ + tags: ["User administration (Mastodon API)"], + summary: "Delete a specific account", + operationId: "MastodonAdmin.AccountController.delete", + description: "Delete the given account.", + security: [%{"oAuth" => ["admin:write:accounts"]}], + parameters: [ + Operation.parameter(:id, :path, :string, "ID of the account") + ], + responses: %{ + 200 => Operation.response("Account", "application/json", account()), + 401 => Operation.response("Error", "application/json", ApiError), + 404 => Operation.response("Error", "application/json", ApiError) + } + } + end + + def enable_operation do + %Operation{ + tags: ["User administration (Mastodon API)"], + summary: "Re-enable account", + operationId: "MastodonAdmin.AccountController.enable", + description: "Re-enable a local account whose login is currently disabled.", + security: [%{"oAuth" => ["admin:write:accounts"]}], + parameters: [ + Operation.parameter(:id, :path, :string, "ID of the account") + ], + responses: %{ + 200 => Operation.response("Account", "application/json", account()), + 401 => Operation.response("Error", "application/json", ApiError), + 404 => Operation.response("Error", "application/json", ApiError) + } + } + end + + def approve_operation do + %Operation{ + tags: ["User administration (Mastodon API)"], + summary: "Approve pending account", + operationId: "MastodonAdmin.AccountController.approve", + description: "Approve the given local account if it is currently pending approval.", + parameters: [ + Operation.parameter(:id, :path, :string, "ID of the account") + ], + responses: %{ + 200 => Operation.response("Account", "application/json", account()), + 401 => Operation.response("Error", "application/json", ApiError), + 404 => Operation.response("Error", "application/json", ApiError) + } + } + end + + def reject_operation do + %Operation{ + tags: ["User administration (Mastodon API)"], + summary: "Reject pending account", + operationId: "MastodonAdmin.AccountController.reject", + description: "Reject the given local account if it is currently pending approval.", + parameters: [ + Operation.parameter(:id, :path, :string, "ID of the account") + ], + responses: %{ + 200 => Operation.response("Account", "application/json", account()), + 400 => Operation.response("Error", "application/json", ApiError), + 401 => Operation.response("Error", "application/json", ApiError), + 404 => Operation.response("Error", "application/json", ApiError) + } + } + end + + def account do + %Schema{ + title: "Admin::Account", + description: "Admin-level information about a given account.", + type: :object, + properties: %{ + id: FlakeID, + username: %Schema{type: :string}, + domain: %Schema{type: :string, nullable: true}, + created_at: %Schema{type: :string, format: "date-time"}, + email: %Schema{type: :string, format: "email", nullable: true}, + ip: %Schema{type: :string, nullable: true}, + ips: %Schema{ + type: :array, + items: %Schema{ + type: :object, + properties: %{ + ip: %Schema{type: :string}, + used_at: %Schema{type: :string, format: "date-time"} + } + } + }, + locale: %Schema{type: :string, nullable: true}, + invite_request: %Schema{type: :string, nullable: true}, + role: %Schema{type: :string, nullable: true}, + confirmed: %Schema{type: :boolean}, + approved: %Schema{type: :boolean}, + disabled: %Schema{type: :boolean}, + silenced: %Schema{type: :boolean, nullable: true}, + suspended: %Schema{type: :boolean, nullable: true}, + account: Account, + created_by_application_id: %Schema{type: :string, nullable: true}, + invited_by_account_id: %Schema{type: :string, nullable: true} + } + } + end +end diff --git a/lib/pleroma/web/api_spec/operations/mastodon_admin/report_operation.ex b/lib/pleroma/web/api_spec/operations/mastodon_admin/report_operation.ex new file mode 100644 index 000000000..7e0bcc24b --- /dev/null +++ b/lib/pleroma/web/api_spec/operations/mastodon_admin/report_operation.ex @@ -0,0 +1,180 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ApiSpec.MastodonAdmin.ReportOperation do + alias OpenApiSpex.Operation + alias OpenApiSpex.Schema + alias Pleroma.Web.ApiSpec.MastodonAdmin.AccountOperation + alias Pleroma.Web.ApiSpec.Schemas.ApiError + alias Pleroma.Web.ApiSpec.Schemas.FlakeID + alias Pleroma.Web.ApiSpec.Schemas.Status + + import Pleroma.Web.ApiSpec.Helpers + + def open_api_operation(action) do + operation = String.to_existing_atom("#{action}_operation") + apply(__MODULE__, operation, []) + end + + def index_operation do + %Operation{ + tags: ["Report management (Mastodon API)"], + summary: "View all reports", + operationId: "MastodonAdmin.ReportController.index", + description: + "View all reports. Pagination may be done with HTTP Link header in the response.", + security: [%{"oAuth" => ["admin:read:reports"]}], + parameters: + [ + Operation.parameter(:resolved, :query, :boolean, "Filter for resolved reports"), + Operation.parameter(:unresolved, :query, :boolean, "Filter for open reports"), + Operation.parameter(:account_id, :query, :string, "Filter by author account id"), + Operation.parameter( + :target_account_id, + :query, + :string, + "Filter by report target account id" + ) + ] ++ + pagination_params(), + responses: %{ + 200 => + Operation.response("Reports", "application/json", %Schema{ + title: "ArrayOfReports", + type: :array, + items: report() + }), + 401 => Operation.response("Error", "application/json", ApiError) + } + } + end + + def show_operation do + %Operation{ + tags: ["Report management (Mastodon API)"], + summary: "View a single report", + operationId: "MastodonAdmin.ReportController.show", + description: "View information about the report with the given ID.", + security: [%{"oAuth" => ["admin:read:reports"]}], + parameters: [ + Operation.parameter(:id, :path, :string, "ID of the report") + ], + responses: %{ + 200 => Operation.response("Report", "application/json", report()), + 401 => Operation.response("Error", "application/json", ApiError), + 404 => Operation.response("Error", "application/json", ApiError) + } + } + end + + def resolve_operation do + %Operation{ + tags: ["Report management (Mastodon API)"], + summary: "Mark as resolved", + operationId: "MastodonAdmin.ReportController.resolve", + description: "Mark a report as resolved with no further action taken.", + security: [%{"oAuth" => ["admin:write:reports"]}], + parameters: [ + Operation.parameter(:id, :path, :string, "ID of the report") + ], + responses: %{ + 200 => Operation.response("Report", "application/json", report()), + 400 => Operation.response("Error", "application/json", ApiError), + 401 => Operation.response("Error", "application/json", ApiError), + 404 => Operation.response("Error", "application/json", ApiError) + } + } + end + + def reopen_operation do + %Operation{ + tags: ["Report management (Mastodon API)"], + summary: "Re-open report", + operationId: "MastodonAdmin.ReportController.reopen", + description: "Reopen a currently closed report.", + security: [%{"oAuth" => ["admin:write:reports"]}], + parameters: [ + Operation.parameter(:id, :path, :string, "ID of the report") + ], + responses: %{ + 200 => Operation.response("Report", "application/json", report()), + 400 => Operation.response("Error", "application/json", ApiError), + 401 => Operation.response("Error", "application/json", ApiError), + 404 => Operation.response("Error", "application/json", ApiError) + } + } + end + + def assign_to_self_operation do + %Operation{ + tags: ["Report management (Mastodon API)"], + summary: "Assign report to self", + operationId: "MastodonAdmin.ReportController.assign_to_self", + security: [%{"oAuth" => ["admin:write:reports"]}], + parameters: [Operation.parameter(:id, :path, :string, "ID of the report")], + responses: report_update_responses() + } + end + + def unassign_operation do + %Operation{ + tags: ["Report management (Mastodon API)"], + summary: "Unassign report", + operationId: "MastodonAdmin.ReportController.unassign", + security: [%{"oAuth" => ["admin:write:reports"]}], + parameters: [Operation.parameter(:id, :path, :string, "ID of the report")], + responses: report_update_responses() + } + end + + defp report_update_responses do + %{ + 200 => Operation.response("Report", "application/json", report()), + 400 => Operation.response("Error", "application/json", ApiError), + 401 => Operation.response("Error", "application/json", ApiError), + 404 => Operation.response("Error", "application/json", ApiError) + } + end + + defp report do + %Schema{ + title: "Admin::Report", + type: :object, + properties: %{ + id: FlakeID, + action_taken: %Schema{type: :boolean}, + action_taken_at: %Schema{type: :string, format: "date-time", nullable: true}, + category: %Schema{type: :string, enum: ["spam", "legal", "violation", "other"]}, + comment: %Schema{type: :string, nullable: true}, + forwarded: %Schema{type: :boolean, nullable: true}, + created_at: %Schema{type: :string, format: "date-time"}, + updated_at: %Schema{type: :string, format: "date-time"}, + account: AccountOperation.account(), + target_account: AccountOperation.account(), + assigned_account: %Schema{ + oneOf: [AccountOperation.account()], + nullable: true + }, + action_taken_by_account: %Schema{ + oneOf: [AccountOperation.account()], + nullable: true + }, + statuses: %Schema{ + type: :array, + items: Status + }, + rules: %Schema{ + type: :array, + items: %Schema{ + type: :object, + properties: %{ + id: %Schema{type: :string}, + text: %Schema{type: :string} + } + } + } + } + } + end +end diff --git a/lib/pleroma/web/api_spec/operations/notification_operation.ex b/lib/pleroma/web/api_spec/operations/notification_operation.ex index 94d1f6b82..740f7645d 100644 --- a/lib/pleroma/web/api_spec/operations/notification_operation.ex +++ b/lib/pleroma/web/api_spec/operations/notification_operation.ex @@ -3,12 +3,12 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.ApiSpec.NotificationOperation do - alias OpenApiSpex.Operation alias OpenApiSpex.Operation alias OpenApiSpex.Schema alias Pleroma.Web.ApiSpec.Schemas.Account alias Pleroma.Web.ApiSpec.Schemas.ApiError alias Pleroma.Web.ApiSpec.Schemas.BooleanLike + alias Pleroma.Web.ApiSpec.Schemas.ChatMessage alias Pleroma.Web.ApiSpec.Schemas.Status alias Pleroma.Web.ApiSpec.Schemas.VisibilityScope @@ -91,6 +91,79 @@ defmodule Pleroma.Web.ApiSpec.NotificationOperation do } end + def grouped_index_operation do + %Operation{ + tags: ["Notifications"], + summary: "Retrieve grouped notifications", + description: "Notifications concerning the user, grouped by supported notification types.", + operationId: "NotificationController.grouped_index", + security: [%{"oAuth" => ["read:notifications"]}], + parameters: grouped_notification_params() ++ pagination_params(), + responses: %{ + 200 => Operation.response("Grouped notifications", "application/json", grouped_result()), + 404 => Operation.response("Error", "application/json", ApiError) + } + } + end + + def show_group_operation do + %Operation{ + tags: ["Notifications"], + summary: "Retrieve a notification group", + operationId: "NotificationController.show_group", + security: [%{"oAuth" => ["read:notifications"]}], + parameters: [group_key_param()], + responses: %{ + 200 => Operation.response("Grouped notifications", "application/json", grouped_result()), + 404 => Operation.response("Error", "application/json", ApiError) + } + } + end + + def group_accounts_operation do + %Operation{ + tags: ["Notifications"], + summary: "Retrieve notification group accounts", + operationId: "NotificationController.group_accounts", + security: [%{"oAuth" => ["read:notifications"]}], + parameters: [group_key_param()], + responses: %{ + 200 => + Operation.response("Accounts", "application/json", %Schema{ + type: :array, + items: Account + }) + } + } + end + + def unread_count_operation do + %Operation{ + tags: ["Notifications"], + summary: "Retrieve unread grouped notification count", + operationId: "NotificationController.unread_count", + security: [%{"oAuth" => ["read:notifications"]}], + parameters: + grouped_notification_params() ++ + [ + Operation.parameter( + :limit, + :query, + %Schema{type: :integer}, + "Maximum number of notifications to count" + ) + ], + responses: %{ + 200 => + Operation.response("Unread notification group count", "application/json", %Schema{ + type: :object, + properties: %{count: %Schema{type: :integer}} + }), + 404 => Operation.response("Error", "application/json", ApiError) + } + } + end + def clear_operation do %Operation{ tags: ["Notifications"], @@ -114,6 +187,18 @@ defmodule Pleroma.Web.ApiSpec.NotificationOperation do } end + def dismiss_group_operation do + %Operation{ + tags: ["Notifications"], + summary: "Dismiss a notification group", + description: "Clear all notifications in a notification group from the server.", + operationId: "NotificationController.dismiss_group", + parameters: [group_key_param()], + security: [%{"oAuth" => ["write:notifications"]}], + responses: %{200 => empty_object_response()} + } + end + def dismiss_via_body_operation do %Operation{ tags: ["Notifications"], @@ -184,7 +269,7 @@ defmodule Pleroma.Web.ApiSpec.NotificationOperation do }, example: %{ "id" => "34975861", - "group-key" => "ungrouped-34975861", + "group_key" => "ungrouped-34975861", "type" => "mention", "created_at" => "2019-11-23T07:49:02.064Z", "account" => Account.schema().example, @@ -194,6 +279,91 @@ defmodule Pleroma.Web.ApiSpec.NotificationOperation do } end + defp grouped_result do + %Schema{ + title: "GroupedNotificationsResults", + type: :object, + properties: %{ + accounts: %Schema{type: :array, items: Account}, + statuses: %Schema{type: :array, items: Status}, + notification_groups: %Schema{type: :array, items: notification_group()} + }, + required: [:accounts, :statuses, :notification_groups] + } + end + + defp notification_group do + %Schema{ + title: "NotificationGroup", + type: :object, + properties: %{ + group_key: %Schema{type: :string}, + notifications_count: %Schema{type: :integer}, + type: notification_type(), + most_recent_notification_id: %Schema{type: :string}, + page_min_id: %Schema{type: :string, nullable: true}, + page_max_id: %Schema{type: :string, nullable: true}, + latest_page_notification_at: %Schema{type: :string, format: :"date-time", nullable: true}, + sample_account_ids: %Schema{type: :array, items: %Schema{type: :string}}, + status_id: %Schema{type: :string, nullable: true}, + target_id: %Schema{ + type: :string, + description: "ID of the target account for a move notification" + }, + emoji: %Schema{ + type: :string, + description: "Emoji used for an emoji reaction notification" + }, + emoji_url: %Schema{ + type: :string, + nullable: true, + description: "URL of the custom emoji used for an emoji reaction notification" + }, + chat_message: ChatMessage, + report: %Schema{ + type: :object, + description: "Report embedded in a report notification" + } + }, + required: [ + :group_key, + :notifications_count, + :type, + :most_recent_notification_id, + :sample_account_ids + ] + } + end + + defp grouped_notification_params do + [ + Operation.parameter( + :exclude_types, + :query, + %Schema{type: :array, items: notification_type()}, + "Array of types to exclude" + ), + Operation.parameter( + :account_id, + :query, + %Schema{type: :string}, + "Return only notifications received from this account" + ), + Operation.parameter( + :types, + :query, + %Schema{type: :array, items: notification_type()}, + "Include the notifications for activities with the given types" + ), + Operation.parameter( + :grouped_types, + :query, + %Schema{type: :array, items: notification_type()}, + "Notification types that may be grouped" + ) + ] + end + defp notification_type do %Schema{ type: :string, @@ -239,4 +409,11 @@ defmodule Pleroma.Web.ApiSpec.NotificationOperation do required: true ) end + + defp group_key_param do + Operation.parameter(:group_key, :path, :string, "Notification group key", + example: "favourite-123", + required: true + ) + end end diff --git a/lib/pleroma/web/api_spec/operations/status_operation.ex b/lib/pleroma/web/api_spec/operations/status_operation.ex index 9b1478d2a..9855cbb6e 100644 --- a/lib/pleroma/web/api_spec/operations/status_operation.ex +++ b/lib/pleroma/web/api_spec/operations/status_operation.ex @@ -356,34 +356,6 @@ defmodule Pleroma.Web.ApiSpec.StatusOperation do } end - def card_operation do - %Operation{ - tags: ["Retrieve status information"], - deprecated: true, - summary: "Preview card", - description: "Deprecated in favor of card property inlined on Status entity", - operationId: "StatusController.card", - parameters: [id_param()], - security: [%{"oAuth" => ["read:statuses"]}], - responses: %{ - 200 => - Operation.response("Card", "application/json", %Schema{ - type: :object, - nullable: true, - properties: %{ - type: %Schema{type: :string, enum: ["link", "photo", "video", "rich"]}, - provider_name: %Schema{type: :string, nullable: true}, - provider_url: %Schema{type: :string, format: :uri}, - url: %Schema{type: :string, format: :uri}, - image: %Schema{type: :string, nullable: true, format: :uri}, - title: %Schema{type: :string}, - description: %Schema{type: :string} - } - }) - } - } - end - def favourited_by_operation do %Operation{ tags: ["Retrieve status information"], diff --git a/lib/pleroma/web/api_spec/schemas/chat_message.ex b/lib/pleroma/web/api_spec/schemas/chat_message.ex index 57f7890e5..4c18063be 100644 --- a/lib/pleroma/web/api_spec/schemas/chat_message.ex +++ b/lib/pleroma/web/api_spec/schemas/chat_message.ex @@ -5,6 +5,7 @@ defmodule Pleroma.Web.ApiSpec.Schemas.ChatMessage do alias OpenApiSpex.Schema alias Pleroma.Web.ApiSpec.Schemas.Emoji + alias Pleroma.Web.ApiSpec.Schemas.PreviewCard require OpenApiSpex @@ -22,36 +23,8 @@ defmodule Pleroma.Web.ApiSpec.Schemas.ChatMessage do emojis: %Schema{type: :array, items: Emoji}, attachment: %Schema{type: :object, nullable: true}, card: %Schema{ - type: :object, - nullable: true, - description: "Preview card for links included within status content", - required: [:url, :title, :description, :type], - properties: %{ - type: %Schema{ - type: :string, - enum: ["link", "photo", "video", "rich"], - description: "The type of the preview card" - }, - provider_name: %Schema{ - type: :string, - nullable: true, - description: "The provider of the original resource" - }, - provider_url: %Schema{ - type: :string, - format: :uri, - description: "A link to the provider of the original resource" - }, - url: %Schema{type: :string, format: :uri, description: "Location of linked resource"}, - image: %Schema{ - type: :string, - nullable: true, - format: :uri, - description: "Preview thumbnail" - }, - title: %Schema{type: :string, description: "Title of linked resource"}, - description: %Schema{type: :string, description: "Description of preview"} - } + allOf: [PreviewCard], + nullable: true }, unread: %Schema{type: :boolean, description: "Whether a message has been marked as read."} }, diff --git a/lib/pleroma/web/api_spec/schemas/list.ex b/lib/pleroma/web/api_spec/schemas/list.ex index 5df674894..ff52b1def 100644 --- a/lib/pleroma/web/api_spec/schemas/list.ex +++ b/lib/pleroma/web/api_spec/schemas/list.ex @@ -17,11 +17,26 @@ defmodule Pleroma.Web.ApiSpec.Schemas.List do exclusive: %Schema{ type: :boolean, description: "Whether members of the list should be removed from the “Home” feed" + }, + pleroma: %Schema{ + type: :object, + properties: %{ + emoji: %Schema{type: :string, description: "List emoji", nullable: true}, + emoji_url: %Schema{ + type: :string, + description: "URL of the list emoji if it's a custom emoji, null otherwise", + nullable: true + } + } } }, example: %{ "id" => "12249", - "title" => "Friends" + "title" => "Friends", + "pleroma" => %{ + "emoji" => nil, + "emoji_url" => nil + } } }) end diff --git a/lib/pleroma/web/api_spec/schemas/preview_card.ex b/lib/pleroma/web/api_spec/schemas/preview_card.ex new file mode 100644 index 000000000..bb4432e18 --- /dev/null +++ b/lib/pleroma/web/api_spec/schemas/preview_card.ex @@ -0,0 +1,109 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ApiSpec.Schemas.PreviewCard do + alias OpenApiSpex.Schema + alias Pleroma.Web.ApiSpec.Schemas.Account + + require OpenApiSpex + + OpenApiSpex.schema(%{ + title: "PreviewCard", + description: "Preview card for a link included within status or chat message content", + type: :object, + required: [ + :url, + :title, + :description, + :language, + :type, + :authors, + :author_name, + :author_url, + :provider_name, + :provider_url, + :html, + :width, + :height, + :image, + :image_description, + :embed_url, + :blurhash, + :published_at + ], + properties: %{ + url: %Schema{type: :string, format: :uri, description: "Location of linked resource"}, + title: %Schema{type: :string, description: "Title of linked resource"}, + description: %Schema{type: :string, description: "Description of preview"}, + language: %Schema{ + type: :string, + nullable: true, + description: "Language of the linked resource" + }, + type: %Schema{ + type: :string, + enum: ["link", "photo", "video", "rich"], + description: "The type of the preview card" + }, + authors: %Schema{ + type: :array, + description: "Fediverse accounts of the authors of the original resource", + items: %Schema{ + type: :object, + required: [:name, :url, :account], + properties: %{ + name: %Schema{type: :string}, + url: %Schema{type: :string}, + account: %Schema{allOf: [Account], nullable: true} + } + } + }, + author_name: %Schema{type: :string, description: "Author of the original resource"}, + author_url: %Schema{ + type: :string, + description: "Link to the author of the original resource" + }, + provider_name: %Schema{ + type: :string, + description: "The provider of the original resource" + }, + provider_url: %Schema{ + type: :string, + description: "A link to the provider of the original resource" + }, + html: %Schema{ + type: :string, + format: :html, + description: "HTML to be used for generating the preview card" + }, + width: %Schema{type: :integer, description: "Width of preview, in pixels"}, + height: %Schema{type: :integer, description: "Height of preview, in pixels"}, + image: %Schema{ + type: :string, + nullable: true, + format: :uri, + description: "Preview thumbnail" + }, + image_description: %Schema{ + type: :string, + description: "Alternate text that describes what is in the thumbnail" + }, + embed_url: %Schema{ + type: :string, + description: "Used for photo embeds, instead of custom `html`" + }, + blurhash: %Schema{ + type: :string, + nullable: true, + description: + "A hash computed by the [BlurHash algorithm](https://github.com/woltapp/blurhash), for generating colorful preview thumbnails when media has not been downloaded yet." + }, + published_at: %Schema{ + type: :string, + nullable: true, + description: "Publication date of the linked resource" + } + } + }) +end diff --git a/lib/pleroma/web/api_spec/schemas/status.ex b/lib/pleroma/web/api_spec/schemas/status.ex index ac1735e35..63e9eebf0 100644 --- a/lib/pleroma/web/api_spec/schemas/status.ex +++ b/lib/pleroma/web/api_spec/schemas/status.ex @@ -9,6 +9,7 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Status do alias Pleroma.Web.ApiSpec.Schemas.Emoji alias Pleroma.Web.ApiSpec.Schemas.FlakeID alias Pleroma.Web.ApiSpec.Schemas.Poll + alias Pleroma.Web.ApiSpec.Schemas.PreviewCard alias Pleroma.Web.ApiSpec.Schemas.Tag alias Pleroma.Web.ApiSpec.Schemas.VisibilityScope @@ -31,40 +32,8 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Status do }, bookmarked: %Schema{type: :boolean, description: "Have you bookmarked this status?"}, card: %Schema{ - type: :object, - nullable: true, - description: "Preview card for links included within status content", - required: [:url, :title, :description, :type], - properties: %{ - type: %Schema{ - type: :string, - enum: ["link", "photo", "video", "rich"], - description: "The type of the preview card" - }, - provider_name: %Schema{ - type: :string, - nullable: true, - description: "The provider of the original resource" - }, - provider_url: %Schema{ - type: :string, - format: :uri, - description: "A link to the provider of the original resource" - }, - url: %Schema{type: :string, format: :uri, description: "Location of linked resource"}, - image: %Schema{ - type: :string, - nullable: true, - format: :uri, - description: "Preview thumbnail" - }, - image_description: %Schema{ - type: :string, - description: "Alternate text that describes what is in the thumbnail" - }, - title: %Schema{type: :string, description: "Title of linked resource"}, - description: %Schema{type: :string, description: "Description of preview"} - } + allOf: [PreviewCard], + nullable: true }, content: %Schema{type: :string, format: :html, description: "HTML-encoded status content"}, text: %Schema{ diff --git a/lib/pleroma/web/common_api.ex b/lib/pleroma/web/common_api.ex index cb9d521b3..220a8a228 100644 --- a/lib/pleroma/web/common_api.ex +++ b/lib/pleroma/web/common_api.ex @@ -700,7 +700,7 @@ defmodule Pleroma.Web.CommonAPI do end def update_report_state(activity_id, state) do - with %Activity{} = activity <- Activity.get_by_id(activity_id, filter: []) do + with %Activity{} = activity <- Activity.get_report(activity_id) do Utils.update_report_state(activity, state) else nil -> {:error, :not_found} @@ -715,7 +715,7 @@ defmodule Pleroma.Web.CommonAPI do end def assign_report_to_account(activity_id, user) do - with %Activity{} = activity <- Activity.get_by_id(activity_id) do + with %Activity{} = activity <- Activity.get_report(activity_id) do Utils.assign_report_to_account(activity, user) else nil -> {:error, :not_found} diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex index 26034d685..255803c14 100644 --- a/lib/pleroma/web/common_api/utils.ex +++ b/lib/pleroma/web/common_api/utils.ex @@ -324,7 +324,7 @@ defmodule Pleroma.Web.CommonAPI.Utils do def format_input(text, "text/x.misskeymarkdown", options) do text - |> Formatter.markdown_to_html(%{breaks: true}) + |> Formatter.markdown_to_html(%{breaks: true, strip_email_autolinks: true}) |> safe_mfm_to_html() |> Formatter.linkify(options) |> Formatter.html_escape("text/x.misskeymarkdown") diff --git a/lib/pleroma/web/controller_helper.ex b/lib/pleroma/web/controller_helper.ex index b15b0ea81..0c85e79f9 100644 --- a/lib/pleroma/web/controller_helper.ex +++ b/lib/pleroma/web/controller_helper.ex @@ -55,16 +55,28 @@ defmodule Pleroma.Web.ControllerHelper do # TODO: Only fetch the params from open_api_spex when everything is converted @id_keys Pagination.page_keys() -- ["limit", "order"] + @id_key_atoms Enum.map(@id_keys, &String.to_atom/1) + @drop_id_params_key :drop_id_params defp build_pagination_fields(conn, min_id, max_id, extra_params, order) do + drop_id_params? = Map.get(extra_params, @drop_id_params_key, false) + extra_params = Map.delete(extra_params, @drop_id_params_key) + + path_param_keys = + conn.path_params + |> Map.keys() + |> Enum.flat_map(&[&1, String.to_existing_atom(&1)]) + params = if Map.has_key?(conn.private, :open_api_spex) do get_in(conn, [Access.key(:private), Access.key(:open_api_spex), Access.key(:params)]) else conn.params end - |> Map.drop(Map.keys(conn.path_params) |> Enum.map(&String.to_existing_atom/1)) + |> Map.drop(path_param_keys) |> Map.merge(extra_params) - |> Map.drop(@id_keys) + # Some OpenApiSpex routes cast cursor params to atoms. Keep the historical default for + # existing endpoints, but allow grouped notifications to opt out of stale request cursors. + |> Map.drop(@id_keys ++ if(drop_id_params?, do: @id_key_atoms, else: [])) {{next_id, nid}, {prev_id, pid}} = if order == :desc, diff --git a/lib/pleroma/web/endpoint.ex b/lib/pleroma/web/endpoint.ex index 07b33f866..8552f5d0a 100644 --- a/lib/pleroma/web/endpoint.ex +++ b/lib/pleroma/web/endpoint.ex @@ -139,6 +139,8 @@ defmodule Pleroma.Web.Endpoint do from: {:pleroma, "priv/static/adminfe/"} ) + plug(Pleroma.Web.Plugs.StaticNotFoundPlug) + # Code reloading can be explicitly enabled under the # :code_reloader configuration of your endpoint. if code_reloading? do diff --git a/lib/pleroma/web/federator.ex b/lib/pleroma/web/federator.ex index 90cd2e54a..88b267331 100644 --- a/lib/pleroma/web/federator.ex +++ b/lib/pleroma/web/federator.ex @@ -89,7 +89,8 @@ defmodule Pleroma.Web.Federator do # Job Worker Callbacks - @spec perform(atom(), any()) :: {:ok, any()} | {:error, any()} + @impl true + @spec perform(atom(), any()) :: :ok | {:ok, any()} | {:error, any()} def perform(:publish_one, params) do Publisher.prepare_one(params) |> Publisher.publish_one() diff --git a/lib/pleroma/web/federator/publishing.ex b/lib/pleroma/web/federator/publishing.ex index 3a242b8dc..889b6b83e 100644 --- a/lib/pleroma/web/federator/publishing.ex +++ b/lib/pleroma/web/federator/publishing.ex @@ -3,5 +3,6 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.Federator.Publishing do + @callback perform(atom(), any()) :: :ok | {:ok, any()} | {:error, any()} @callback publish(map()) :: any() end diff --git a/lib/pleroma/web/feed/feed_view.ex b/lib/pleroma/web/feed/feed_view.ex index e1ee33d62..b409b1923 100644 --- a/lib/pleroma/web/feed/feed_view.ex +++ b/lib/pleroma/web/feed/feed_view.ex @@ -38,6 +38,20 @@ defmodule Pleroma.Web.Feed.FeedView do end end + def most_recent_update([], :rss), do: nil + + def most_recent_update(activities, :rss) do + activities + |> Enum.map(fn activity -> + case activity.object do + %Object{updated_at: updated_at} -> updated_at + _ -> activity.updated_at + end + end) + |> Enum.max_by(&NaiveDateTime.to_erl/1) + |> to_rfc2822() + end + def most_recent_update(activities, user, :atom) do (List.first(activities) || user).updated_at |> to_rfc3339() @@ -97,6 +111,67 @@ defmodule Pleroma.Web.Feed.FeedView do end end + def rss_content_encoded(data) do + data + |> feed_content_encoded() + |> String.replace("]]>", "]]>") + end + + def atom_content_encoded(data), do: feed_content_encoded(data) + + defp feed_content_encoded(data) do + base_content = + case activity_content(data) do + "" -> + (data["summary"] || data["type"] || "") + |> escape() + + content -> + content + end + + attachments_html = + if sensitive?(data) do + "" + else + (data["attachment"] || []) + |> Enum.map(&rss_attachment_preview/1) + |> Enum.reject(&(&1 == "")) + |> Enum.join("

") + end + + [base_content, attachments_html] + |> Enum.reject(&(&1 == "")) + |> Enum.join("

") + end + + defp rss_attachment_preview(attachment) do + href = attachment_href(attachment) + + if is_binary(href) and href != "" do + media_type = attachment_type(attachment) || "application/octet-stream" + escaped_href = escape(href) + name = attachment["name"] || "Attachment" + escaped_name = escape(name) + + cond do + String.starts_with?(media_type, "image/") -> + ~s(

#{escaped_name}

) + + String.starts_with?(media_type, "video/") -> + ~s(

#{escaped_name}

) + + String.starts_with?(media_type, "audio/") -> + ~s(

#{escaped_name}

) + + true -> + ~s(

#{escaped_name}

) + end + else + "" + end + end + def activity_content(%{"content" => content}) do content |> String.replace(~r/[\n\r]/, "") @@ -106,16 +181,104 @@ defmodule Pleroma.Web.Feed.FeedView do def activity_context(activity), do: escape(activity.data["context"]) + def sensitive?(data), do: data["sensitive"] in [true, "true"] + + def feed_self_url(conn) do + conn + |> Phoenix.Controller.current_url() + |> escape() + end + + def activity_link(activity, data) do + cond do + activity.local -> data["id"] || data["url"] + true -> data["external_url"] || data["id"] || data["url"] + end + end + def attachment_href(attachment) do - attachment["url"] - |> hd() + attachment + |> attachment_url_data() |> Map.get("href") end def attachment_type(attachment) do - attachment["url"] - |> hd() - |> Map.get("mediaType") + attachment + |> attachment_url_data() + |> Map.get("mediaType") || attachment["mediaType"] + end + + def attachment_size(attachment) do + attachment + |> attachment_url_data() + |> Map.get("size", 0) + end + + def attachment_size_positive(attachment) do + case attachment_size(attachment) do + size when is_integer(size) and size > 0 -> + size + + size when is_binary(size) -> + case Integer.parse(size) do + {parsed, ""} when parsed > 0 -> parsed + _ -> nil + end + + _ -> + nil + end + end + + def attachment_medium(attachment) do + case attachment_type(attachment) do + "image/" <> _rest -> "image" + "video/" <> _rest -> "video" + "audio/" <> _rest -> "audio" + _ -> "document" + end + end + + def rss_enclosure_attachment(attachments) do + Enum.find(attachments || [], &(not is_nil(attachment_size_positive(&1)))) + end + + def attachment_description(attachment) do + attachment["name"] || "" + end + + def media_content_xml(_attachment, sensitive) when sensitive in [true, "true"], do: "" + + def media_content_xml(attachment, _sensitive) do + href = escape(attachment_href(attachment) || "") + type = escape(attachment_type(attachment) || "application/octet-stream") + medium = escape(attachment_medium(attachment)) + + file_size_attr = + case attachment_size_positive(attachment) do + size when is_integer(size) -> ~s( fileSize="#{size}") + _ -> "" + end + + case attachment_description(attachment) do + "" -> + ~s(\n) + + description -> + """ + + #{escape(description)} + + """ + end + end + + defp attachment_url_data(attachment) do + case attachment["url"] do + [first | _] when is_map(first) -> first + %{} = map -> map + _ -> %{} + end end def get_href(id) do diff --git a/lib/pleroma/web/mastodon_api/admin/controllers/account_controller.ex b/lib/pleroma/web/mastodon_api/admin/controllers/account_controller.ex new file mode 100644 index 000000000..b4e038f0e --- /dev/null +++ b/lib/pleroma/web/mastodon_api/admin/controllers/account_controller.ex @@ -0,0 +1,305 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.MastodonAPI.Admin.AccountController do + use Pleroma.Web, :controller + + import Pleroma.Web.ControllerHelper, + only: [ + add_link_headers: 2, + assign_account_by_id: 2, + json_response: 3 + ] + + alias Pleroma.Activity + alias Pleroma.ModerationLog + alias Pleroma.Pagination + alias Pleroma.User + alias Pleroma.Web.ActivityPub.Builder + alias Pleroma.Web.ActivityPub.Pipeline + alias Pleroma.Web.CommonAPI + alias Pleroma.Web.Plugs.OAuthScopesPlug + + @filter_params ~W( + local external active needing_approval deactivated nickname name domain email staff origin status + ) + + plug(Pleroma.Web.ApiSpec.CastAndValidate) + + plug( + OAuthScopesPlug, + %{scopes: ["admin:read:accounts"]} + when action in [:index, :index2, :show] + ) + + action_fallback(Pleroma.Web.MastodonAPI.FallbackController) + + plug( + OAuthScopesPlug, + %{scopes: ["admin:write:accounts"]} + when action in [ + :delete, + :enable, + :account_action, + :approve, + :reject + ] + ) + + plug( + :assign_account_by_id + when action in [ + :show, + :delete, + :enable, + :account_action, + :approve, + :reject + ] + ) + + defdelegate open_api_operation(action), to: Pleroma.Web.ApiSpec.MastodonAdmin.AccountOperation + + def index(conn, params) do + users = + params + |> build_criteria() + |> User.Query.build() + |> Pagination.fetch_paginated(params) + + conn + |> add_link_headers(users) + |> render("index.json", users: users) + end + + def index2(conn, params) do + users = + params + |> build_criteria_v2() + |> User.Query.build() + |> Pagination.fetch_paginated(params) + + conn + |> add_link_headers(users) + |> render("index.json", users: users) + end + + def show(%{assigns: %{user: _admin, account: user}} = conn, _params) do + render(conn, "show.json", user: user) + end + + def account_action( + %{assigns: %{user: admin, account: user}, body_params: %{type: type} = body_params} = + conn, + _params + ) do + with :ok <- validate_account_action(user, type), + {:ok, report} <- associated_report(user, body_params), + :ok <- resolve_report(admin, report), + {:ok, _user} <- handle_account_action(user, admin, type) do + json_response(conn, :no_content, "") + else + {:error, :not_found} -> + {:error, :not_found} + + {:error, error} -> + json_response(conn, :bad_request, %{error: error}) + end + end + + def delete(%{assigns: %{user: admin, account: user}} = conn, _params) do + {:ok, delete_data, _} = Builder.delete(admin, user.ap_id) + Pipeline.common_pipeline(delete_data, local: true) + + ModerationLog.insert_log(%{ + actor: admin, + subject: [user], + action: "delete" + }) + + render(conn, "show.json", user: user) + end + + def enable(%{assigns: %{user: admin, account: user}} = conn, _params) do + {:ok, user} = User.set_activation(user, true) + + ModerationLog.insert_log(%{ + actor: admin, + subject: [user], + action: "activate" + }) + + render(conn, "show.json", user: user) + end + + def approve(%{assigns: %{user: admin, account: user}} = conn, _params) do + {:ok, user} = User.approve(user) + + ModerationLog.insert_log(%{ + actor: admin, + subject: [user], + action: "approve" + }) + + render(conn, "show.json", user: user) + end + + def reject(%{assigns: %{user: admin, account: user}} = conn, _params) do + with {:ok, _} <- User.reject(user) do + ModerationLog.insert_log(%{ + actor: admin, + subject: [user], + action: "reject" + }) + + render(conn, "show.json", user: user) + else + {:error, error} -> + json_response(conn, :bad_request, %{error: error}) + end + end + + defp handle_account_action(%User{local: true} = user, admin, "disable") do + ModerationLog.insert_log(%{ + actor: admin, + subject: [user], + action: "deactivate" + }) + + User.set_activation(user, false) + end + + defp handle_account_action(%User{}, _admin, "disable") do + {:error, "Only local accounts can be disabled"} + end + + defp handle_account_action(user, _admin, "none") do + {:ok, user} + end + + defp validate_account_action(%User{local: true}, "disable"), do: :ok + + defp validate_account_action(%User{}, "disable"), + do: {:error, "Only local accounts can be disabled"} + + defp validate_account_action(%User{}, "none"), do: :ok + + defp build_criteria(params) do + %{} + |> maybe_filter_local(params) + |> maybe_filter_external(params) + |> maybe_filter_active(params) + |> maybe_filter_needing_approval(params) + |> maybe_filter_deactivated(params) + |> maybe_filter_nickname(params) + |> maybe_filter_name(params) + |> maybe_filter_domain(params) + |> maybe_filter_email(params) + |> maybe_filter_staff(params) + end + + defp build_criteria_v2(params) do + %{} + |> maybe_filter_origin(params) + |> maybe_filter_status(params) + |> maybe_filter_staff(params) + |> maybe_filter_nickname(params) + |> maybe_filter_name(params) + |> maybe_filter_domain(params) + |> maybe_filter_email(params) + end + + defp maybe_filter_local(criteria, %{local: true} = _params), + do: Map.put(criteria, :local, true) + + defp maybe_filter_local(criteria, %{local: false} = _params), + do: Map.put(criteria, :external, true) + + defp maybe_filter_external(criteria, %{remote: true} = _params), + do: Map.put(criteria, :external, true) + + defp maybe_filter_external(criteria, %{remote: false} = _params), + do: Map.put(criteria, :local, true) + + defp maybe_filter_origin(criteria, %{origin: "local"} = _params), + do: Map.put(criteria, :local, true) + + defp maybe_filter_origin(criteria, %{origin: "remote"} = _params), + do: Map.put(criteria, :external, true) + + defp maybe_filter_active(criteria, %{active: active} = _params), + do: Map.put(criteria, :active, active) + + defp maybe_filter_needing_approval(criteria, %{pending: need_approval} = _params), + do: Map.put(criteria, :need_approval, need_approval) + + defp maybe_filter_deactivated(criteria, %{disabled: deactivated} = _params), + do: Map.put(criteria, :deactivated, deactivated) + + defp maybe_filter_status(criteria, %{status: "active"} = _params), + do: Map.put(criteria, :active, true) + + defp maybe_filter_status(criteria, %{status: "inactive"} = _params), + do: Map.put(criteria, :active, false) + + defp maybe_filter_status(criteria, %{status: "pending"} = _params), + do: Map.put(criteria, :need_approval, true) + + defp maybe_filter_status(criteria, %{status: "disabled"} = _params), + do: Map.put(criteria, :deactivated, true) + + defp maybe_filter_nickname(criteria, %{username: nickname} = _params), + do: Map.put(criteria, :nickname, nickname) + + defp maybe_filter_name(criteria, %{display_name: name} = _params), + do: Map.put(criteria, :name, name) + + defp maybe_filter_domain(criteria, %{by_domain: domain} = _params), + do: Map.put(criteria, :domain, domain) + + defp maybe_filter_email(criteria, %{email: email} = _params), + do: Map.put(criteria, :email, email) + + defp maybe_filter_staff(criteria, %{permissions: "staff"} = _params), + do: Map.put(criteria, :staff, true) + + defp maybe_filter_staff(criteria, %{staff: staff} = _params), + do: Map.put(criteria, :staff, staff) + + for filter_param <- @filter_params do + defp unquote(:"maybe_filter_#{filter_param}")(criteria, _params), do: criteria + end + + defp associated_report(user, %{report_id: id}) do + case Activity.get_report(id) do + %Activity{data: %{"object" => [target_ap_id | _]}} = report + when target_ap_id == user.ap_id -> + {:ok, report} + + %Activity{} -> + {:error, "Report does not target this account"} + + nil -> + {:error, :not_found} + end + end + + defp associated_report(_user, _params), do: {:ok, nil} + + defp resolve_report(admin, %Activity{id: id}) do + with {:ok, activity} <- CommonAPI.update_report_state(id, "resolved"), + report <- Activity.get_by_id_with_user_actor(activity.id) do + ModerationLog.insert_log(%{ + action: "report_update", + actor: admin, + subject: activity, + subject_actor: report.user_actor + }) + + :ok + end + end + + defp resolve_report(_admin, nil), do: :ok +end diff --git a/lib/pleroma/web/mastodon_api/admin/controllers/report_controller.ex b/lib/pleroma/web/mastodon_api/admin/controllers/report_controller.ex new file mode 100644 index 000000000..062b90fc6 --- /dev/null +++ b/lib/pleroma/web/mastodon_api/admin/controllers/report_controller.ex @@ -0,0 +1,171 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.MastodonAPI.Admin.ReportController do + use Pleroma.Web, :controller + + import Pleroma.Web.ControllerHelper, + only: [ + add_link_headers: 2, + json_response: 3 + ] + + alias Pleroma.Activity + alias Pleroma.ModerationLog + alias Pleroma.Pagination + alias Pleroma.User + alias Pleroma.Web.ActivityPub.ActivityPub + alias Pleroma.Web.AdminAPI.Report + alias Pleroma.Web.CommonAPI + alias Pleroma.Web.Plugs.OAuthScopesPlug + + plug(Pleroma.Web.ApiSpec.CastAndValidate) + + plug(OAuthScopesPlug, %{scopes: ["admin:read:reports"]} when action in [:index, :show]) + + plug(OAuthScopesPlug, %{scopes: ["admin:write:reports"]} when action in [:resolve, :reopen]) + + plug( + OAuthScopesPlug, + %{scopes: ["admin:write:reports"]} when action in [:assign_to_self, :unassign] + ) + + action_fallback(Pleroma.Web.MastodonAPI.FallbackController) + + defdelegate open_api_operation(action), to: Pleroma.Web.ApiSpec.MastodonAdmin.ReportOperation + + def index(conn, params) do + opts = + %{} + |> Map.put(:type, "Flag") + |> Map.put(:skip_preload, true) + |> Map.put(:preload_report_notes, true) + |> Map.put(:total, true) + |> restrict_state(params) + |> restrict_actor(params) + |> restrict_target(params) + + reports = + ActivityPub.fetch_activities_query([], opts) + |> Pagination.fetch_paginated(params) + + conn + |> add_link_headers(reports) + |> render("index.json", reports: reports) + end + + def show(conn, %{id: id}) do + with %Activity{} = report <- Activity.get_report(id) do + render(conn, "show.json", Report.extract_report_info(report)) + else + _ -> {:error, :not_found} + end + end + + def resolve(%{assigns: %{user: admin}} = conn, %{id: id}) do + with {:ok, activity} <- CommonAPI.update_report_state(id, "resolved"), + report <- Activity.get_by_id_with_user_actor(activity.id) do + ModerationLog.insert_log(%{ + action: "report_update", + actor: admin, + subject: activity, + subject_actor: report.user_actor + }) + + render(conn, "show.json", Report.extract_report_info(report)) + else + {:error, :not_found} -> + {:error, :not_found} + + {:error, error} -> + json_response(conn, :bad_request, %{error: error}) + end + end + + def reopen(%{assigns: %{user: admin}} = conn, %{id: id}) do + with {:ok, activity} <- CommonAPI.update_report_state(id, "open"), + report <- Activity.get_by_id_with_user_actor(activity.id) do + ModerationLog.insert_log(%{ + action: "report_update", + actor: admin, + subject: activity, + subject_actor: report.user_actor + }) + + render(conn, "show.json", Report.extract_report_info(report)) + else + {:error, :not_found} -> + {:error, :not_found} + + {:error, error} -> + json_response(conn, :bad_request, %{error: error}) + end + end + + def assign_to_self(%{assigns: %{user: admin}} = conn, %{id: id}) do + with {:ok, activity} <- CommonAPI.assign_report_to_account(id, admin.id), + report <- Activity.get_by_id_with_user_actor(activity.id) do + ModerationLog.insert_log(%{ + action: "report_assigned", + actor: admin, + subject: activity, + subject_actor: report.user_actor, + assigned_account: admin.nickname + }) + + render(conn, "show.json", Report.extract_report_info(report)) + else + {:error, :not_found} -> + {:error, :not_found} + + {:error, error} -> + json_response(conn, :bad_request, %{error: error}) + end + end + + def unassign(%{assigns: %{user: admin}} = conn, %{id: id}) do + with {:ok, activity} <- CommonAPI.assign_report_to_account(id, nil), + report <- Activity.get_by_id_with_user_actor(activity.id) do + ModerationLog.insert_log(%{ + action: "report_unassigned", + actor: admin, + subject: activity, + subject_actor: report.user_actor + }) + + render(conn, "show.json", Report.extract_report_info(report)) + else + {:error, :not_found} -> + {:error, :not_found} + + {:error, error} -> + json_response(conn, :bad_request, %{error: error}) + end + end + + defp restrict_state(opts, %{resolved: true, unresolved: true}), do: opts + + defp restrict_state(opts, %{resolved: true}), do: Map.put(opts, :state, "resolved") + + defp restrict_state(opts, _params), do: Map.put(opts, :state, "open") + + defp restrict_actor(opts, %{account_id: actor}) do + with %User{ap_id: ap_id} <- User.get_by_id(actor) do + Map.put(opts, :actor_id, ap_id) + else + _ -> Map.put(opts, :actor_id, actor) + end + end + + defp restrict_actor(opts, _params), do: opts + + defp restrict_target(opts, %{target_account_id: actor}) do + case User.get_by_id(actor) do + %User{ap_id: ap_id} -> Map.put(opts, :report_target_id, ap_id) + _ -> Map.put(opts, :report_target_id, actor) + end + end + + defp restrict_target(opts, _params), do: opts +end diff --git a/lib/pleroma/web/mastodon_api/admin/views/account_view.ex b/lib/pleroma/web/mastodon_api/admin/views/account_view.ex new file mode 100644 index 000000000..20ccfd51c --- /dev/null +++ b/lib/pleroma/web/mastodon_api/admin/views/account_view.ex @@ -0,0 +1,64 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.MastodonAPI.Admin.AccountView do + use Pleroma.Web, :view + + alias Pleroma.User + alias Pleroma.Web.CommonAPI.Utils + alias Pleroma.Web.MastodonAPI + + def render("index.json", %{users: users}) do + render_many(users, __MODULE__, "show.json", as: :user) + end + + def render("show.json", %{user: user}) do + account = + MastodonAPI.AccountView.render("show.json", %{user: user, skip_visibility_check: true}) + + %{ + id: user.id, + username: username_from_nickname(user.nickname), + domain: domain_from_nickname(user.nickname), + created_at: Utils.to_masto_date(user.inserted_at), + email: user.email, + ip: nil, + ips: [], + locale: nil, + invite_request: user.registration_reason, + role: role(user), + confirmed: user.is_confirmed, + approved: user.is_approved, + disabled: !user.is_active, + silenced: nil, + suspended: nil, + account: account + } + end + + defp username_from_nickname(string) when is_binary(string) do + hd(String.split(string, "@")) + end + + defp username_from_nickname(_), do: nil + + defp domain_from_nickname(string) when is_binary(string) do + String.split(string, "@") + |> Enum.at(1, nil) + end + + defp domain_from_nickname(_), do: nil + + defp role(%User{is_admin: true}) do + "admin" + end + + defp role(%User{is_moderator: true}) do + "moderator" + end + + defp role(_user) do + nil + end +end diff --git a/lib/pleroma/web/mastodon_api/admin/views/report_view.ex b/lib/pleroma/web/mastodon_api/admin/views/report_view.ex new file mode 100644 index 000000000..088824d03 --- /dev/null +++ b/lib/pleroma/web/mastodon_api/admin/views/report_view.ex @@ -0,0 +1,71 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.MastodonAPI.Admin.ReportView do + use Pleroma.Web, :view + + alias Pleroma.HTML + alias Pleroma.Rule + alias Pleroma.Web.AdminAPI.Report + alias Pleroma.Web.CommonAPI.Utils + alias Pleroma.Web.MastodonAPI.Admin.AccountView + alias Pleroma.Web.MastodonAPI.StatusView + + def render("index.json", %{reports: reports}) do + reports + |> Enum.map(&Report.extract_report_info/1) + |> Enum.map(&render(__MODULE__, "show.json", &1)) + end + + def render("show.json", %{ + report: report, + user: account, + account: target_account, + statuses: statuses, + assigned_account: assigned_account + }) do + created_at = Utils.to_masto_date(report.data["published"]) + updated_at = Utils.to_masto_date(report.updated_at) + action_taken = report.data["state"] != "open" + + content = + unless is_nil(report.data["content"]) do + HTML.filter_tags(report.data["content"]) + else + nil + end + + %{ + id: report.id, + action_taken: action_taken, + action_taken_at: if(action_taken, do: updated_at), + category: "other", + comment: content, + forwarded: false, + created_at: created_at, + updated_at: updated_at, + account: AccountView.render("show.json", %{user: account}), + target_account: AccountView.render("show.json", %{user: target_account}), + assigned_account: render_account(assigned_account), + action_taken_by_account: nil, + statuses: + StatusView.render("index.json", %{ + activities: statuses, + as: :activity + }), + rules: rules(report.data["rules"]) + } + end + + defp render_account(nil), do: nil + defp render_account(account), do: AccountView.render("show.json", %{user: account}) + + defp rules(nil), do: [] + + defp rules(rule_ids) do + rule_ids + |> Rule.get() + |> Enum.map(&%{id: to_string(&1.id), text: &1.text}) + end +end diff --git a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex index 6d5851029..631102843 100644 --- a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex @@ -348,6 +348,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountController do params |> Map.delete(:tagged) |> Map.put(:tag, params[:tagged]) + |> Map.put(:restrict_unauthenticated, true) activities = ActivityPub.fetch_user_activities(user, reading_user, params) diff --git a/lib/pleroma/web/mastodon_api/controllers/notification_controller.ex b/lib/pleroma/web/mastodon_api/controllers/notification_controller.ex index afd83b785..8a5f8e7b6 100644 --- a/lib/pleroma/web/mastodon_api/controllers/notification_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/notification_controller.ex @@ -5,13 +5,17 @@ defmodule Pleroma.Web.MastodonAPI.NotificationController do use Pleroma.Web, :controller - import Pleroma.Web.ControllerHelper, only: [add_link_headers: 2] + import Pleroma.Web.ControllerHelper, only: [add_link_headers: 2, add_link_headers: 3] alias Pleroma.Notification + alias Pleroma.User + alias Pleroma.Web.MastodonAPI.AccountView alias Pleroma.Web.MastodonAPI.MastodonAPI alias Pleroma.Web.Plugs.OAuthScopesPlug - @oauth_read_actions [:show, :index] + # Mastodon's docs currently list write:notifications for group accounts, but the endpoint is + # read-only and Mastodon's implementation accepts read:notifications. Prefer least privilege. + @oauth_read_actions [:show, :index, :grouped_index, :show_group, :group_accounts, :unread_count] plug(Pleroma.Web.ApiSpec.CastAndValidate, replace_params: false) @@ -39,7 +43,7 @@ defmodule Pleroma.Web.MastodonAPI.NotificationController do # GET /api/v1/notifications def index(%{private: %{open_api_spex: %{params: %{account_id: account_id} = params}}} = conn, _) do - case Pleroma.User.get_cached_by_id(account_id) do + case User.get_cached_by_id(account_id) do %{ap_id: account_ap_id} -> params = params @@ -59,10 +63,108 @@ defmodule Pleroma.Web.MastodonAPI.NotificationController do do_get_notifications(conn, params) end + # GET /api/v2/notifications + def grouped_index( + %{private: %{open_api_spex: %{params: %{account_id: account_id} = params}}} = conn, + _ + ) do + case User.get_cached_by_id(account_id) do + %{ap_id: account_ap_id} -> + params = + params + |> Map.delete(:account_id) + |> Map.put(:account_ap_id, account_ap_id) + + do_get_grouped_notifications(conn, params) + + _ -> + conn + |> put_status(:not_found) + |> json(%{"error" => "Account is not found"}) + end + end + + def grouped_index(%{private: %{open_api_spex: %{params: params}}} = conn, _) do + do_get_grouped_notifications(conn, params) + end + + # GET /api/v2/notifications/:group_key + def show_group( + %{assigns: %{user: user}, private: %{open_api_spex: %{params: %{group_key: group_key}}}} = + conn, + _ + ) do + {notifications, notification_group_counts, notification_group_bounds} = + MastodonAPI.get_notification_group_result(user, group_key, %{}) + + if Enum.empty?(notifications) do + conn + |> put_status(:not_found) + |> json(%{"error" => "Notification group is not found"}) + else + grouped_types = if String.starts_with?(group_key, "ungrouped-"), do: [], else: nil + + render(conn, "grouped_index.json", + notification_groups: [notifications], + notification_group_counts: notification_group_counts, + notification_group_bounds: notification_group_bounds, + for: user, + grouped_types: grouped_types, + include_page_metadata: false + ) + end + end + + # GET /api/v2/notifications/:group_key/accounts + def group_accounts( + %{assigns: %{user: user}, private: %{open_api_spex: %{params: %{group_key: group_key}}}} = + conn, + _ + ) do + # Mastodon paginates this endpoint in code, but the public docs say it returns accounts of all + # notifications in the group and do not document cursor params here. Follow the documented API. + users = MastodonAPI.get_notification_group_accounts(user, group_key) + + json(conn, AccountView.render("index.json", %{users: users, for: user})) + end + + # GET /api/v2/notifications/unread_count + def unread_count( + %{private: %{open_api_spex: %{params: %{account_id: account_id} = params}}} = conn, + _ + ) do + case User.get_cached_by_id(account_id) do + %{ap_id: account_ap_id} -> + params = + params + |> Map.delete(:account_id) + |> Map.put(:account_ap_id, account_ap_id) + + do_get_unread_group_count(conn, params) + + _ -> + conn + |> put_status(:not_found) + |> json(%{"error" => "Account is not found"}) + end + end + + def unread_count(%{private: %{open_api_spex: %{params: params}}} = conn, _) do + do_get_unread_group_count(conn, params) + end + + # POST /api/v2/notifications/:group_key/dismiss + def dismiss_group( + %{assigns: %{user: user}, private: %{open_api_spex: %{params: %{group_key: group_key}}}} = + conn, + _ + ) do + MastodonAPI.dismiss_notification_group(user, group_key) + json(conn, %{}) + end + defp do_get_notifications(%{assigns: %{user: user}} = conn, params) do - params = - Map.new(params, fn {k, v} -> {to_string(k), v} end) - |> Map.put_new("types", Map.get(params, :include_types, @default_notification_types)) + params = normalize_notification_params(params) notifications = MastodonAPI.get_notifications(user, params) @@ -74,6 +176,35 @@ defmodule Pleroma.Web.MastodonAPI.NotificationController do ) end + defp do_get_grouped_notifications(%{assigns: %{user: user}} = conn, params) do + params = normalize_notification_params(params) + + {notification_groups, page_notifications, notification_group_counts, + notification_group_bounds} = + MastodonAPI.get_grouped_notification_page(user, params) + + conn + |> add_link_headers(page_notifications, %{drop_id_params: true}) + |> render("grouped_index.json", + notification_groups: notification_groups, + notification_group_counts: notification_group_counts, + notification_group_bounds: notification_group_bounds, + for: user, + grouped_types: params["grouped_types"] + ) + end + + defp do_get_unread_group_count(%{assigns: %{user: user}} = conn, params) do + params = normalize_notification_params(params) + json(conn, %{count: MastodonAPI.unread_notification_group_count(user, params)}) + end + + defp normalize_notification_params(params) do + params + |> Map.new(fn {k, v} -> {to_string(k), v} end) + |> Map.put_new("types", Map.get(params, :include_types, @default_notification_types)) + end + # GET /api/v1/notifications/:id def show(%{assigns: %{user: user}, private: %{open_api_spex: %{params: %{id: id}}}} = conn, _) do with {:ok, notification} <- Notification.get(user, id) do diff --git a/lib/pleroma/web/mastodon_api/controllers/status_controller.ex b/lib/pleroma/web/mastodon_api/controllers/status_controller.ex index 26e38aac8..6cd7469f9 100644 --- a/lib/pleroma/web/mastodon_api/controllers/status_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/status_controller.ex @@ -716,7 +716,10 @@ defmodule Pleroma.Web.MastodonAPI.StatusController do end end - defp put_application(params, %{assigns: %{token: %Token{user: %User{} = user} = token}} = _conn) do + defp put_application( + params, + %{assigns: %{token: %Token{} = token, user: %User{} = user}} = _conn + ) do if user.disclose_client do %{client_name: client_name, website: website} = Repo.preload(token, :app).app Map.put(params, :generator, %{type: "Application", name: client_name, url: website}) diff --git a/lib/pleroma/web/mastodon_api/mastodon_api.ex b/lib/pleroma/web/mastodon_api/mastodon_api.ex index c9e045d23..a65326226 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api.ex @@ -6,12 +6,16 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPI do import Ecto.Query import Ecto.Changeset + alias Pleroma.Marker alias Pleroma.Notification alias Pleroma.Pagination + alias Pleroma.Repo alias Pleroma.ScheduledActivity alias Pleroma.User alias Pleroma.Web.CommonAPI + @notification_group_sample_limit 8 + @spec follow(User.t(), User.t(), map) :: {:ok, User.t()} | {:error, String.t()} def follow(follower, followed, params \\ %{}) do result = @@ -61,29 +65,365 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPI do end def get_notifications(user, params \\ %{}) do - options = - cast_params(params) |> Map.update(:include_types, [], fn include_types -> include_types end) + user + |> notifications_query(params) + |> Pagination.fetch_paginated(params) + end - options = - if ("pleroma:report" not in options.include_types and - User.privileged?(user, :reports_manage_reports)) or - User.privileged?(user, :reports_manage_reports) do - options - else - options - |> Map.update(:exclude_types, ["pleroma:report"], fn current_exclude_types -> - current_exclude_types ++ ["pleroma:report"] - end) - end + def get_grouped_notification_page(user, params \\ %{}) do + grouped_types = + params + |> Map.get("grouped_types") + |> Notification.normalize_grouped_types() + + query = notifications_query(user, params) + {order, cursor_filters} = group_pagination(params) + + group_rows = + query + |> notification_group_rows(grouped_types, grouped_limit(params), order, cursor_filters) + + group_rows = if order == :asc, do: Enum.reverse(group_rows), else: group_rows + + page_notifications = representative_notifications(query, group_rows) + + notification_groups = + Enum.map(group_rows, ¬ification_group_sample(user, params, &1.group_key)) + + { + notification_groups, + page_notifications, + notification_group_counts(group_rows), + notification_group_bounds(group_rows) + } + end + + def get_notification_group_result(user, group_key, params \\ %{}) do + notifications = notification_group_sample(user, params, group_key) + metadata = notification_group_metadata(user, group_key, params) + + if Enum.empty?(notifications) or is_nil(metadata) do + {[], %{}, %{}} + else + { + notifications, + %{group_key => metadata.notifications_count}, + %{group_key => Map.drop(metadata, [:notifications_count])} + } + end + end + + def get_notification_group(user, group_key, params \\ %{}) + + def get_notification_group(user, "ungrouped-" <> notification_id, _params) do + case Notification.get(user, notification_id) do + {:ok, notification} -> [notification] + _ -> [] + end + end + + def get_notification_group(user, group_key, params) do + notification_group_sample(user, params, group_key) + end + + def get_notification_group_accounts(user, "ungrouped-" <> notification_id) do + with {:ok, notification} <- Notification.get(user, notification_id), + %User{} = actor <- User.get_cached_by_ap_id(notification.activity.data["actor"]) do + [actor] + else + _ -> [] + end + end + + def get_notification_group_accounts(user, group_key) do + user + |> notification_group_query(group_key, %{}) + |> exclude(:preload) + |> distinct(true) + |> select([user_actor: user_actor], user_actor) + |> Repo.all() + end + + def dismiss_notification_group(user, "ungrouped-" <> notification_id) do + Notification.destroy_multiple(user, [notification_id]) + end + + def dismiss_notification_group(%User{id: user_id}, group_key) do + Notification + |> where([n], n.user_id == ^user_id and n.group_key == ^group_key) + |> Repo.delete_all() + end + + defp group_pagination(params) do + cond do + min_id = Map.get(params, "min_id") -> + cursor_filters = [{:gt, min_id}] + + cursor_filters = + case Map.get(params, "max_id") do + nil -> cursor_filters + max_id -> [{:lt, max_id} | cursor_filters] + end + + {:asc, cursor_filters} + + since_id = Map.get(params, "since_id") -> + {:desc, [{:gt, since_id}]} + + max_id = Map.get(params, "max_id") -> + {:desc, [{:lt, max_id}]} + + true -> + {:desc, []} + end + end + + defp grouped_limit(params) do + params + |> Map.get("limit", 40) + |> parse_limit(40) + |> min(80) + end + + def unread_notification_group_count(user, params \\ %{}) do + grouped_types = + params + |> Map.get("grouped_types") + |> Notification.normalize_grouped_types() + + limit = unread_count_limit(params) + + user + |> notifications_query(params) + # The grouped API docs define unread by the notifications marker, not by Pleroma's per-row + # seen flag used by the v1 unread count. Keep this marker-based for Mastodon clients. + |> restrict_after_marker(notification_marker_last_read_id(user)) + |> notification_group_rows(grouped_types, limit, :desc, []) + |> length() + end + + defp notification_group_rows(query, grouped_types, group_limit, :desc, cursor_filters) do + query + |> notification_group_keyed_query(grouped_types) + |> group_by([n], n.group_key) + |> apply_group_cursor_filters(cursor_filters) + |> select([n], %{ + group_key: n.group_key, + representative_id: max(n.id), + notifications_count: count(n.id), + page_min_id: min(n.id), + page_max_id: max(n.id), + latest_page_notification_at: max(n.inserted_at) + }) + |> order_by([n], desc: max(n.id)) + |> limit(^group_limit) + |> Repo.all() + end + + defp notification_group_rows(query, grouped_types, group_limit, :asc, cursor_filters) do + query + |> notification_group_keyed_query(grouped_types) + |> group_by([n], n.group_key) + |> apply_group_cursor_filters(cursor_filters) + |> select([n], %{ + group_key: n.group_key, + representative_id: max(n.id), + notifications_count: count(n.id), + page_min_id: min(n.id), + page_max_id: max(n.id), + latest_page_notification_at: max(n.inserted_at) + }) + |> order_by([n], asc: max(n.id)) + |> limit(^group_limit) + |> Repo.all() + end + + defp apply_group_cursor_filters(query, []), do: query + + defp apply_group_cursor_filters(query, [{:gt, id} | rest]) do + query + |> having([n], max(n.id) > ^id) + |> apply_group_cursor_filters(rest) + end + + defp apply_group_cursor_filters(query, [{:lt, id} | rest]) do + query + |> having([n], max(n.id) < ^id) + |> apply_group_cursor_filters(rest) + end + + defp notification_group_keyed_query(query, grouped_types) do + query + |> exclude(:preload) + |> select([n], %{ + id: n.id, + inserted_at: n.inserted_at, + group_key: + fragment( + "CASE WHEN ? IS NOT NULL AND ?::text = ANY(?) THEN ? ELSE 'ungrouped-' || ?::text END", + n.group_key, + n.type, + type(^grouped_types, {:array, :string}), + n.group_key, + n.id + ) + }) + |> subquery() + end + + defp representative_notifications(_query, []), do: [] + + defp representative_notifications(query, group_rows) do + representative_ids = Enum.map(group_rows, & &1.representative_id) + + notifications_by_id = + query + |> where([n], n.id in ^representative_ids) + |> Repo.all() + |> Map.new(&{to_string(&1.id), &1}) + + group_rows + |> Enum.map(&Map.get(notifications_by_id, to_string(&1.representative_id))) + |> Enum.filter(& &1) + end + + defp notification_group_counts(group_rows) do + Map.new(group_rows, &{&1.group_key, &1.notifications_count}) + end + + defp notification_group_bounds(group_rows) do + Map.new(group_rows, fn row -> + {row.group_key, + %{ + page_min_id: row.page_min_id, + page_max_id: row.page_max_id, + latest_page_notification_at: row.latest_page_notification_at + }} + end) + end + + defp notification_group_sample(user, _params, "ungrouped-" <> notification_id) do + case Notification.get(user, notification_id) do + {:ok, notification} -> [notification] + _ -> [] + end + end + + defp notification_group_sample(user, params, group_key) do + user + |> notification_group_query(group_key, params) + |> order_by([n], desc: n.id) + |> limit(^@notification_group_sample_limit) + |> Repo.all() + end + + defp notification_group_metadata(user, "ungrouped-" <> notification_id, _params) do + case Notification.get(user, notification_id) do + {:ok, notification} -> + %{ + notifications_count: 1, + page_min_id: notification.id, + page_max_id: notification.id, + latest_page_notification_at: notification.inserted_at + } + + _ -> + nil + end + end + + defp notification_group_metadata(user, group_key, params) do + user + |> notification_group_query(group_key, params) + |> exclude(:preload) + |> select([n], %{ + notifications_count: count(n.id), + page_min_id: min(n.id), + page_max_id: max(n.id), + latest_page_notification_at: max(n.inserted_at) + }) + |> Repo.one() + |> case do + %{notifications_count: 0} -> nil + metadata -> metadata + end + end + + defp notification_group_query(user, group_key, params) do + user + |> notifications_query(params) + |> where([n], n.group_key == ^group_key) + end + + defp notification_marker_last_read_id(user) do + Marker + |> where([m], m.user_id == ^user.id and m.timeline == "notifications") + |> select([m], m.last_read_id) + |> Repo.one() + end + + defp restrict_after_marker(query, last_read_id) + when is_binary(last_read_id) and last_read_id != "" do + where(query, [n], n.id > ^last_read_id) + end + + defp restrict_after_marker(query, _last_read_id), do: query + + defp notifications_query(user, params) do + options = notification_options(user, params) user |> Notification.for_user_query(options) |> restrict(:types, options) |> restrict(:exclude_types, options) |> restrict(:account_ap_id, options) - |> Pagination.fetch_paginated(params) end + defp notification_options(user, params) do + options = + params + |> cast_params() + |> Map.update(:include_types, [], fn include_types -> include_types end) + + if "pleroma:chat_mention" in options.include_types and not Pleroma.Chat.enabled?() do + options + |> Map.update(:include_types, [], fn include_types -> + List.delete(include_types, "pleroma:chat_mention") + end) + else + options + end + + if ("pleroma:report" not in options.include_types and + User.privileged?(user, :reports_manage_reports)) or + User.privileged?(user, :reports_manage_reports) do + options + else + options + |> Map.update(:exclude_types, ["pleroma:report"], fn current_exclude_types -> + current_exclude_types ++ ["pleroma:report"] + end) + end + end + + defp unread_count_limit(params) do + params + |> Map.get("limit", 100) + |> parse_limit(100) + |> min(1000) + end + + defp parse_limit(limit, _default) when is_integer(limit) and limit > 0, do: limit + + defp parse_limit(limit, default) when is_binary(limit) do + case Integer.parse(limit) do + {limit, _} when limit > 0 -> limit + _ -> default + end + end + + defp parse_limit(_, default), do: default + def get_scheduled_activities(user, params \\ %{}) do user |> ScheduledActivity.for_user_query() @@ -95,6 +435,8 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPI do exclude_types: {:array, :string}, types: {:array, :string}, exclude_visibilities: {:array, :string}, + grouped_types: {:array, :string}, + limit: :integer, reblogs: :boolean, with_muted: :boolean, account_ap_id: :string, diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex index 5386c5a6c..1e3882fc6 100644 --- a/lib/pleroma/web/mastodon_api/views/account_view.ex +++ b/lib/pleroma/web/mastodon_api/views/account_view.ex @@ -287,6 +287,8 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do user.last_status_at && user.last_status_at |> NaiveDateTime.to_date() |> Date.to_iso8601() + accepts_chat_messages = Pleroma.Chat.enabled?() and user.accepts_chat_messages + %{ id: to_string(user.id), username: username_from_nickname(user.nickname), @@ -336,7 +338,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do relationship: relationship, skip_thread_containment: user.skip_thread_containment, background_image: image_url(user.background) |> MediaProxy.url(), - accepts_chat_messages: user.accepts_chat_messages, + accepts_chat_messages: accepts_chat_messages, favicon: favicon, avatar_description: avatar_description, header_description: header_description @@ -391,6 +393,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do |> Kernel.put_in([:source, :pleroma, :show_role], user.show_role) |> Kernel.put_in([:source, :pleroma, :no_rich_text], user.no_rich_text) |> Kernel.put_in([:source, :pleroma, :show_birthday], user.show_birthday) + |> Kernel.put_in([:source, :pleroma, :accepts_chat_messages], user.accepts_chat_messages) end defp maybe_put_settings(data, _, _, _), do: data diff --git a/lib/pleroma/web/mastodon_api/views/instance_view.ex b/lib/pleroma/web/mastodon_api/views/instance_view.ex index d64ce4fb5..437e42580 100644 --- a/lib/pleroma/web/mastodon_api/views/instance_view.ex +++ b/lib/pleroma/web/mastodon_api/views/instance_view.ex @@ -17,6 +17,8 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do reject: "suspend" } + def mastodon_api_level, do: @mastodon_api_level + def render("show.json", _) do instance = Config.get(:instance) @@ -166,6 +168,7 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do [ "pleroma_api", "mastodon_api", + "mastodon_api_grouped_notifications", "mastodon_api_streaming", "polls", "v2_suggestions", @@ -199,8 +202,12 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do end, "pleroma_emoji_reactions", "pleroma_custom_emoji_reactions", - "pleroma_chat_messages", - "pleroma:pin_chats", + if Pleroma.Chat.enabled?() do + "pleroma_chat_messages" + end, + if Pleroma.Chat.enabled?() do + "pleroma:pin_chats" + end, if Config.get([:instance, :show_reactions]) do "exposable_reactions" end, diff --git a/lib/pleroma/web/mastodon_api/views/list_view.ex b/lib/pleroma/web/mastodon_api/views/list_view.ex index 740f458d4..508169fc1 100644 --- a/lib/pleroma/web/mastodon_api/views/list_view.ex +++ b/lib/pleroma/web/mastodon_api/views/list_view.ex @@ -4,6 +4,8 @@ defmodule Pleroma.Web.MastodonAPI.ListView do use Pleroma.Web, :view + + alias Pleroma.Emoji alias Pleroma.Web.MastodonAPI.ListView def render("index.json", %{lists: lists} = opts) do @@ -11,10 +13,29 @@ defmodule Pleroma.Web.MastodonAPI.ListView do end def render("show.json", %{list: list}) do + {emoji, emoji_url} = get_emoji(list.emoji) + %{ id: to_string(list.id), title: list.title, - exclusive: list.exclusive + exclusive: list.exclusive, + pleroma: %{ + emoji: emoji, + emoji_url: emoji_url + } } end + + defp get_emoji(nil), do: {nil, nil} + + defp get_emoji(emoji) do + if Emoji.unicode?(emoji) do + {emoji, nil} + else + case Emoji.get(emoji) do + nil -> {nil, nil} + emoji_data -> {emoji, Emoji.local_url(emoji_data.file)} + end + end + end end diff --git a/lib/pleroma/web/mastodon_api/views/notification_view.ex b/lib/pleroma/web/mastodon_api/views/notification_view.ex index 27532c42a..eadfac4a0 100644 --- a/lib/pleroma/web/mastodon_api/views/notification_view.ex +++ b/lib/pleroma/web/mastodon_api/views/notification_view.ex @@ -72,6 +72,58 @@ defmodule Pleroma.Web.MastodonAPI.NotificationView do safe_render_many(notifications, NotificationView, "show.json", opts) end + def render("grouped_index.json", %{notifications: notifications} = opts) do + grouped_types = Notification.normalize_grouped_types(opts[:grouped_types]) + + opts + |> Map.delete(:notifications) + |> Map.put( + :notification_groups, + Notification.group_notifications(notifications, grouped_types) + ) + |> then(&render("grouped_index.json", &1)) + end + + def render( + "grouped_index.json", + %{notification_groups: notification_groups, for: reading_user} = opts + ) do + grouped_types = Notification.normalize_grouped_types(opts[:grouped_types]) + notification_group_counts = Map.get(opts, :notification_group_counts, %{}) + notification_group_bounds = Map.get(opts, :notification_group_bounds, %{}) + include_page_metadata = Map.get(opts, :include_page_metadata, true) + + statuses = + notification_groups + |> Enum.map(&List.first/1) + |> Enum.map(&render("show.json", %{notification: &1, for: reading_user})) + |> Enum.map(& &1[:status]) + |> Enum.filter(& &1) + |> Enum.uniq_by(& &1[:id]) + + accounts = + notification_groups + |> List.flatten() + |> notification_accounts() + + %{ + accounts: AccountView.render("index.json", %{users: accounts, for: reading_user}), + statuses: statuses, + notification_groups: + Enum.map( + notification_groups, + &render_group( + &1, + reading_user, + grouped_types, + notification_group_counts, + notification_group_bounds, + include_page_metadata + ) + ) + } + end + def render( "show.json", %{ @@ -95,7 +147,7 @@ defmodule Pleroma.Web.MastodonAPI.NotificationView do response = %{ id: to_string(notification.id), - group_key: "ungrouped-" <> to_string(notification.id), + group_key: Notification.group_key(notification), type: notification.type, created_at: CommonAPI.Utils.to_masto_date(notification.inserted_at), account: account, @@ -137,6 +189,153 @@ defmodule Pleroma.Web.MastodonAPI.NotificationView do Map.put(response, :report, report_render) end + defp render_group( + [%Notification{} = notification | _] = notifications, + reading_user, + grouped_types, + notification_group_counts, + notification_group_bounds, + include_page_metadata + ) do + latest_notification = List.first(notifications) + oldest_notification = List.last(notifications) + status_activity = status_activity_for_group(notifications, grouped_types) + group_key = Notification.group_key(notification, grouped_types) + bounds = Map.get(notification_group_bounds, group_key, %{}) + + response = %{ + group_key: group_key, + notifications_count: Map.get(notification_group_counts, group_key, length(notifications)), + type: notification.type, + most_recent_notification_id: + bounds + |> Map.get(:page_max_id, latest_notification.id) + |> to_string(), + sample_account_ids: + notifications + |> notification_actors() + |> Enum.map(&to_string(&1.id)) + } + + response = + if include_page_metadata do + Map.merge(response, %{ + page_min_id: + bounds + |> Map.get(:page_min_id, oldest_notification.id) + |> to_string(), + page_max_id: + bounds + |> Map.get(:page_max_id, latest_notification.id) + |> to_string(), + latest_page_notification_at: + bounds + |> Map.get(:latest_page_notification_at, latest_notification.inserted_at) + |> CommonAPI.Utils.to_masto_date() + }) + else + response + end + + response = + if status_activity do + Map.put(response, :status_id, to_string(status_activity.id)) + else + response + end + + put_group_details(response, notification, reading_user) + end + + defp put_group_details(response, %Notification{type: "move", activity: activity}, _reading_user) do + case User.get_cached_by_ap_id(activity.data["target"]) do + %User{id: id} -> Map.put(response, :target_id, to_string(id)) + _ -> response + end + end + + defp put_group_details( + response, + %Notification{type: "pleroma:emoji_reaction", activity: activity}, + _reading_user + ) do + put_emoji(response, activity) + end + + defp put_group_details( + response, + %Notification{type: "pleroma:chat_mention", activity: activity}, + reading_user + ) do + put_chat_message(response, activity, reading_user, %{}) + end + + defp put_group_details( + response, + %Notification{type: "pleroma:report", activity: activity}, + _reading_user + ) do + put_report(response, activity) + end + + defp put_group_details(response, _notification, _reading_user), do: response + + defp status_activity_for_group([%Notification{} = notification | _], grouped_types) do + status_activity_for(notification, grouped_types) + end + + defp status_activity_for(%Notification{type: type, activity: activity}, _grouped_types) + when type in ["mention", "status", "poll"] do + activity + end + + defp status_activity_for(%Notification{type: type} = notification, grouped_types) + when type in ["favourite", "reblog"] do + group_key = Notification.group_key(notification, grouped_types) + + case String.split(group_key, "-", parts: 3) do + [^type, activity_id, _bucket] -> + case Activity.create_by_id_with_object(activity_id) do + %Activity{} = activity -> activity + _ -> parent_status_activity(notification.activity) + end + + _ -> + parent_status_activity(notification.activity) + end + end + + defp status_activity_for(%Notification{type: type, activity: activity}, _grouped_types) + when type in ["update", "pleroma:emoji_reaction"] do + parent_status_activity(activity) + end + + defp status_activity_for(_, _grouped_types), do: nil + + defp parent_status_activity(activity) do + Activity.get_create_by_object_ap_id(object_id_for(activity)) + end + + defp notification_actors(notifications) do + notifications + |> Enum.map(&User.get_cached_by_ap_id(&1.activity.data["actor"])) + |> Enum.filter(& &1) + |> Enum.uniq_by(& &1.id) + end + + defp notification_accounts(notifications) do + move_targets = + notifications + |> Enum.filter(&(&1.type == "move")) + |> Enum.map(&User.get_cached_by_ap_id(&1.activity.data["target"])) + |> Enum.filter(& &1) + + notifications + |> notification_actors() + |> Kernel.++(move_targets) + |> Enum.uniq_by(& &1.id) + end + defp put_emoji(response, activity) do response |> Map.put(:emoji, activity.data["content"]) diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index 6949c5f2d..4625be47c 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -587,14 +587,25 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do video_url = proxied_url(rich_media["video"], page_url_data) %{ - type: "link", - provider_name: page_url_data.host, - provider_url: page_url_data.scheme <> "://" <> page_url_data.host, + type: card_type(rich_media["type"]), + provider_name: rich_media["provider_name"] || page_url_data.host, + provider_url: + rich_media["provider_url"] || page_url_data.scheme <> "://" <> page_url_data.host, url: page_url, image: image_url, image_description: rich_media["image:alt"] || "", title: rich_media["title"] || "", description: rich_media["description"] || "", + language: nil, + authors: [], + author_name: rich_media["author_name"] || "", + author_url: rich_media["author_url"] || "", + html: rich_media["html"] || "", + width: card_dimension(rich_media["width"]), + height: card_dimension(rich_media["height"]), + embed_url: rich_media["embed_url"] || "", + blurhash: nil, + published_at: nil, pleroma: %{ opengraph: rich_media @@ -698,6 +709,20 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do %{content: content, detected_source_language: detected_source_language, provider: provider} end + defp card_type(type) when type in ["link", "photo", "video", "rich"], do: type + defp card_type(_), do: "link" + + defp card_dimension(value) when is_integer(value) and value >= 0, do: value + + defp card_dimension(value) when is_binary(value) do + case Integer.parse(value) do + {dimension, ""} when dimension >= 0 -> dimension + _ -> 0 + end + end + + defp card_dimension(_), do: 0 + def get_reply_to(activity, %{replied_to_activities: replied_to_activities}) do object = Object.normalize(activity, fetch: false) diff --git a/lib/pleroma/web/o_auth/app.ex b/lib/pleroma/web/o_auth/app.ex index a2841b2bb..9c16c23b6 100644 --- a/lib/pleroma/web/o_auth/app.ex +++ b/lib/pleroma/web/o_auth/app.ex @@ -8,6 +8,7 @@ defmodule Pleroma.Web.OAuth.App do import Ecto.Query alias Pleroma.Repo alias Pleroma.User + alias Pleroma.Web.OAuth.Authorization alias Pleroma.Web.OAuth.Token @type t :: %__MODULE__{} @@ -191,15 +192,26 @@ defmodule Pleroma.Web.OAuth.App do @spec remove_orphans(pos_integer()) :: :ok def remove_orphans(limit \\ 100) do - fifteen_mins_ago = DateTime.add(DateTime.utc_now(), -900, :second) + fifteen_mins_ago = NaiveDateTime.add(NaiveDateTime.utc_now(), -900) Repo.transaction(fn -> + app_ids = + from(a in __MODULE__, + where: is_nil(a.user_id) and a.inserted_at < ^fifteen_mins_ago, + limit: ^limit, + lock: "FOR UPDATE SKIP LOCKED", + select: a.id + ) + |> Repo.all() + + # Bulk deletes bypass the schema's association cleanup. + from(a in Authorization, where: a.app_id in ^app_ids) |> Repo.delete_all() + from(t in Token, where: t.app_id in ^app_ids) |> Repo.delete_all() + from(a in __MODULE__, - where: is_nil(a.user_id) and a.inserted_at < ^fifteen_mins_ago, - limit: ^limit + where: a.id in ^app_ids and is_nil(a.user_id) and a.inserted_at < ^fifteen_mins_ago ) - |> Repo.all() - |> Enum.each(&Repo.delete(&1)) + |> Repo.delete_all() end) :ok diff --git a/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex b/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex index ac9bb2779..9c9f17e86 100644 --- a/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex +++ b/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex @@ -41,6 +41,7 @@ defmodule Pleroma.Web.PleromaAPI.ChatController do ) plug(Pleroma.Web.ApiSpec.CastAndValidate, replace_params: false) + plug(:ensure_chats_enabled) defdelegate open_api_operation(action), to: Pleroma.Web.ApiSpec.ChatOperation @@ -244,4 +245,16 @@ defmodule Pleroma.Web.PleromaAPI.ChatController do _ -> nil end end + + defp ensure_chats_enabled(conn, _opts) do + if Chat.enabled?() do + conn + else + conn + |> put_status(:not_found) + |> put_view(Pleroma.Web.ErrorView) + |> render("404.json") + |> halt() + end + end end diff --git a/lib/pleroma/web/plugs/ensure_host_matches_plug.ex b/lib/pleroma/web/plugs/ensure_host_matches_plug.ex index ab2129f9e..960e86d4f 100644 --- a/lib/pleroma/web/plugs/ensure_host_matches_plug.ex +++ b/lib/pleroma/web/plugs/ensure_host_matches_plug.ex @@ -4,6 +4,7 @@ defmodule Pleroma.Web.Plugs.EnsureHostMatchesPlug do @moduledoc "Ensures Host header matches instance" + @behaviour Plug alias Pleroma.Web.Endpoint diff --git a/lib/pleroma/web/plugs/idempotency_plug.ex b/lib/pleroma/web/plugs/idempotency_plug.ex index a3b7af869..1cb328622 100644 --- a/lib/pleroma/web/plugs/idempotency_plug.ex +++ b/lib/pleroma/web/plugs/idempotency_plug.ex @@ -6,6 +6,8 @@ defmodule Pleroma.Web.Plugs.IdempotencyPlug do import Phoenix.Controller, only: [json: 2] import Plug.Conn + alias Pleroma.Config + @behaviour Plug @cachex Pleroma.Config.get([:cachex, :provider], Cachex) @@ -27,9 +29,11 @@ defmodule Pleroma.Web.Plugs.IdempotencyPlug do def call(conn, _), do: conn def process_request(conn, key) do - case @cachex.get(:idempotency_cache, key) do + cache_key = {conn.method, conn.request_path, actor_key(conn), key} + + case @cachex.get(:idempotency_cache, cache_key) do {:ok, nil} -> - cache_resposnse(conn, key) + cache_response(conn, key, cache_key) {:ok, record} -> send_cached(conn, key, record) @@ -39,18 +43,48 @@ defmodule Pleroma.Web.Plugs.IdempotencyPlug do end end - defp cache_resposnse(conn, key) do - register_before_send(conn, fn conn -> - [request_id] = get_resp_header(conn, "x-request-id") - content_type = get_content_type(conn) + defp cache_response(conn, key, cache_key) do + register_before_send(conn, &maybe_cache_response(&1, key, cache_key)) + end - record = {request_id, content_type, conn.status, conn.resp_body} - {:ok, _} = @cachex.put(:idempotency_cache, key, record) + defp maybe_cache_response( + %{private: %{skip_idempotency_cache: true}} = conn, + _key, + _cache_key + ), + do: conn - conn - |> put_resp_header("idempotency-key", key) - |> put_resp_header("x-original-request-id", request_id) - end) + defp maybe_cache_response(conn, key, cache_key) do + [request_id] = get_resp_header(conn, "x-request-id") + content_type = get_content_type(conn) + + record = {request_id, content_type, conn.status, conn.resp_body} + {:ok, _} = @cachex.put(:idempotency_cache, cache_key, record) + + conn + |> put_resp_header("idempotency-key", key) + |> put_resp_header("x-original-request-id", request_id) + end + + defp actor_key(%{assigns: assigns}) do + {principal_key(assigns), token_key(assigns[:token]), staff_privileges()} + end + + defp principal_key(%{user: %{id: id} = user}) do + {:user, id, Map.get(user, :is_admin), Map.get(user, :is_moderator)} + end + + defp principal_key(%{app: %{id: id}}), do: {:app, id} + defp principal_key(_assigns), do: nil + + defp token_key(%{id: id} = token), do: {id, token |> Map.get(:scopes, []) |> Enum.sort()} + defp token_key(_token), do: nil + + defp staff_privileges do + { + Config.get([:instance, :admin_privileges], []) |> Enum.sort(), + Config.get([:instance, :moderator_privileges], []) |> Enum.sort() + } end defp send_cached(conn, key, record) do diff --git a/lib/pleroma/web/plugs/instance_static.ex b/lib/pleroma/web/plugs/instance_static.ex index 948188287..d2a674d39 100644 --- a/lib/pleroma/web/plugs/instance_static.ex +++ b/lib/pleroma/web/plugs/instance_static.ex @@ -4,7 +4,7 @@ defmodule Pleroma.Web.Plugs.InstanceStatic do require Pleroma.Constants - import Plug.Conn, only: [put_resp_header: 3, put_status: 2, send_resp: 3, halt: 1] + import Plug.Conn, only: [put_resp_header: 3] @moduledoc """ This is a shim to call `Plug.Static` but with runtime `from` configuration. @@ -51,37 +51,10 @@ defmodule Pleroma.Web.Plugs.InstanceStatic do |> Map.put(:from, from) |> Map.put(:content_types, false) - conn = - conn - |> set_content_type(conn.request_path) - |> Plug.Static.call(opts) + conn = set_content_type(conn, conn.request_path) - if conn.halted do - conn - else - path = String.trim_leading(conn.request_path, "/") - - if not File.exists?(file_path(path)) do - conn - |> put_status(:not_found) - |> send_404() - |> halt() - else - conn - end - end - end - - defp send_404(conn) do - if String.ends_with?(String.downcase(conn.request_path), ".json") do - conn - |> put_resp_header("content-type", "application/json") - |> send_resp(404, Jason.encode!(%{error: "not found"})) - else - conn - |> put_resp_header("content-type", "text/plain") - |> send_resp(404, "Not found") - end + # Call Plug.Static with our sanitized content-type + Plug.Static.call(conn, opts) end defp set_content_type(conn, "/emoji/" <> filepath) do diff --git a/lib/pleroma/web/plugs/o_auth_plug.ex b/lib/pleroma/web/plugs/o_auth_plug.ex index 488968691..b59ac9d3e 100644 --- a/lib/pleroma/web/plugs/o_auth_plug.ex +++ b/lib/pleroma/web/plugs/o_auth_plug.ex @@ -52,7 +52,7 @@ defmodule Pleroma.Web.Plugs.OAuthPlug do where: t.token == ^token ) - with %Token{user_id: user_id} = token_record <- Repo.one(token_query) |> Repo.preload(:user), + with %Token{user_id: user_id} = token_record <- Repo.one(token_query), false <- is_nil(user_id), %User{} = user <- User.get_cached_by_id(user_id) do {:ok, user, token_record} diff --git a/lib/pleroma/web/plugs/static_not_found_plug.ex b/lib/pleroma/web/plugs/static_not_found_plug.ex new file mode 100644 index 000000000..e2e7a622c --- /dev/null +++ b/lib/pleroma/web/plugs/static_not_found_plug.ex @@ -0,0 +1,38 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2026 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.Plugs.StaticNotFoundPlug do + @moduledoc """ + A small plug that enforces returning 404 on static files when they don't exist. + """ + @behaviour Plug + + require Pleroma.Constants + import Plug.Conn, only: [put_resp_header: 3, send_resp: 3, halt: 1] + + def init(opts), do: opts + + for only <- Pleroma.Constants.static_only_files() do + def call(%{request_path: "/" <> unquote(only) <> _} = conn, _opts) do + send_404(conn) + end + end + + def call(conn, _) do + conn + end + + defp send_404(conn) do + if String.ends_with?(String.downcase(conn.request_path), ".json") do + conn + |> put_resp_header("content-type", "application/json") + |> send_resp(404, Jason.encode!(%{error: "not found"})) + else + conn + |> put_resp_header("content-type", "text/plain") + |> send_resp(404, "Not found") + end + |> halt() + end +end diff --git a/lib/pleroma/web/rich_media/helpers.ex b/lib/pleroma/web/rich_media/helpers.ex index 963076510..7b4ca677f 100644 --- a/lib/pleroma/web/rich_media/helpers.ex +++ b/lib/pleroma/web/rich_media/helpers.ex @@ -4,6 +4,8 @@ defmodule Pleroma.Web.RichMedia.Helpers do alias Pleroma.Config + alias Pleroma.Gun.ConnectionPool + alias Pleroma.ReverseProxy.Client.Tesla, as: TeslaClient require Logger @@ -19,12 +21,30 @@ defmodule Pleroma.Web.RichMedia.Helpers do end defp stream(url) do - with {_, {:ok, %Tesla.Env{status: 200, body: stream_body, headers: headers}}} <- - {:get, Pleroma.HTTP.get(url, req_headers(), http_options())}, - {_, :ok} <- {:content_type, check_content_type(headers)}, - {_, :ok} <- {:content_length, check_content_length(headers)}, - {:read_stream, {:ok, body}} <- {:read_stream, read_stream(stream_body)} do - {:ok, body} + case Pleroma.HTTP.get(url, req_headers(), http_options()) do + {:ok, %Tesla.Env{status: 200, body: stream_body, headers: headers}} -> + result = + try do + with {_, :ok} <- {:content_type, check_content_type(headers)}, + {_, :ok} <- {:content_length, check_content_length(headers)}, + {:read_stream, {:ok, body}} <- {:read_stream, read_stream(stream_body)} do + {:ok, body} + end + rescue + exception -> + cleanup_stream(stream_body, :error) + reraise exception, __STACKTRACE__ + end + + cleanup_stream(stream_body, result) + result + + {:ok, %Tesla.Env{body: stream_body}} = response -> + cleanup_stream(stream_body, :rejected) + {:get, response} + + response -> + {:get, response} end end @@ -95,6 +115,10 @@ defmodule Pleroma.Web.RichMedia.Helpers do end end + defp read_stream(%{pid: pid, stream: stream, opts: opts}) do + read_chunks(pid, stream, opts, "", 0, Keyword.get(http_options(), :max_body)) + end + defp read_stream(stream) do max_body = Keyword.get(http_options(), :max_body) @@ -117,6 +141,33 @@ defmodule Pleroma.Web.RichMedia.Helpers do end end + defp read_chunks(pid, stream, opts, acc, total_bytes, max_body) do + case Tesla.Adapter.Gun.read_chunk(pid, stream, opts) do + {fin, chunk} when fin in [:fin, :nofin] -> + total_bytes = total_bytes + byte_size(chunk) + + cond do + total_bytes > max_body -> + :error + + fin == :fin -> + {:ok, acc <> chunk} + + true -> + read_chunks(pid, stream, opts, acc <> chunk, total_bytes, max_body) + end + + {:error, _reason} -> + :error + end + end + + defp cleanup_stream(%{pid: pid, stream: stream}, {:ok, _body}), + do: ConnectionPool.release_stream(pid, stream) + + defp cleanup_stream(%{pid: _pid} = client, _result), do: TeslaClient.close(client) + defp cleanup_stream(_stream, _result), do: :ok + defp http_options do [ pool: :rich_media, diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 813c5c482..8cae63ae6 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -137,6 +137,12 @@ defmodule Pleroma.Web.Router do plug(Pleroma.Web.Plugs.EnsurePrivilegedPlug, :users_manage_activation_state) end + pipeline :require_privileged_role_account_actions do + plug(:admin_api) + plug(Pleroma.Web.Plugs.EnsurePrivilegedPlug, :users_manage_activation_state) + plug(Pleroma.Web.Plugs.EnsurePrivilegedPlug, :reports_manage_reports) + end + pipeline :require_privileged_role_users_manage_invites do plug(:admin_api) plug(Pleroma.Web.Plugs.EnsurePrivilegedPlug, :users_manage_invites) @@ -316,6 +322,15 @@ defmodule Pleroma.Web.Router do post("/rules", RuleController, :create) patch("/rules/:id", RuleController, :update) delete("/rules/:id", RuleController, :delete) + + get("/webhooks", WebhookController, :index) + get("/webhooks/:id", WebhookController, :show) + post("/webhooks", WebhookController, :create) + patch("/webhooks/:id", WebhookController, :update) + delete("/webhooks/:id", WebhookController, :delete) + post("/webhooks/:id/enable", WebhookController, :enable) + post("/webhooks/:id/disable", WebhookController, :disable) + post("/webhooks/:id/rotate_secret", WebhookController, :rotate_secret) end # AdminAPI: admins and mods (staff) can perform these actions (if privileged by role) @@ -424,6 +439,60 @@ defmodule Pleroma.Web.Router do delete("/chats/:id/messages/:message_id", ChatController, :delete_message) end + # Mastodon AdminAPI: admins and mods (staff) can perform these actions (if privileged by role) + scope "/api/v1/admin", Pleroma.Web.MastodonAPI.Admin do + pipe_through([:require_privileged_role_users_read]) + + get("/accounts", AccountController, :index) + get("/accounts/:id", AccountController, :show) + end + + # Mastodon AdminAPI: admins and mods (staff) can perform these actions (if privileged by role) + scope "/api/v1/admin", Pleroma.Web.MastodonAPI.Admin do + pipe_through(:require_privileged_role_users_delete) + + delete("/accounts/:id", AccountController, :delete) + end + + # Mastodon AdminAPI: admins and mods (staff) can perform these actions (if privileged by role) + scope "/api/v1/admin", Pleroma.Web.MastodonAPI.Admin do + pipe_through([:require_privileged_role_users_manage_activation_state]) + + post("/accounts/:id/enable", AccountController, :enable) + end + + scope "/api/v1/admin", Pleroma.Web.MastodonAPI.Admin do + pipe_through([:require_privileged_role_account_actions]) + + post("/accounts/:id/action", AccountController, :account_action) + end + + # Mastodon AdminAPI: admins and mods (staff) can perform these actions (if privileged by role) + scope "/api/v1/admin", Pleroma.Web.MastodonAPI.Admin do + pipe_through([:require_privileged_role_users_manage_invites]) + post("/accounts/:id/approve", AccountController, :approve) + post("/accounts/:id/reject", AccountController, :reject) + end + + # Mastodon AdminAPI: admins and mods (staff) can perform these actions (if privileged by role) + scope "/api/v1/admin", Pleroma.Web.MastodonAPI.Admin do + pipe_through([:require_privileged_role_reports_manage_reports]) + + get("/reports", ReportController, :index) + get("/reports/:id", ReportController, :show) + post("/reports/:id/resolve", ReportController, :resolve) + post("/reports/:id/reopen", ReportController, :reopen) + post("/reports/:id/assign_to_self", ReportController, :assign_to_self) + post("/reports/:id/unassign", ReportController, :unassign) + end + + # Mastodon AdminAPI: admins and mods (staff) can perform these actions (if privileged by role) + scope "/api/v2/admin", Pleroma.Web.MastodonAPI.Admin do + pipe_through([:require_privileged_role_users_read]) + + get("/accounts", AccountController, :index2) + end + # AdminAPI: admins and mods (staff) can perform these actions (if privileged by role) scope "/api/v1/pleroma/admin", Pleroma.Web.AdminAPI do pipe_through(:require_privileged_role_emoji_manage_emoji) @@ -844,6 +913,16 @@ defmodule Pleroma.Web.Router do get("/directory", DirectoryController, :index) end + scope "/api/v2", Pleroma.Web.MastodonAPI do + pipe_through(:authenticated_api) + + get("/notifications", NotificationController, :grouped_index) + get("/notifications/unread_count", NotificationController, :unread_count) + get("/notifications/:group_key/accounts", NotificationController, :group_accounts) + get("/notifications/:group_key", NotificationController, :show_group) + post("/notifications/:group_key/dismiss", NotificationController, :dismiss_group) + end + scope "/api/v2", Pleroma.Web.MastodonAPI do pipe_through(:api) diff --git a/lib/pleroma/web/streamer.ex b/lib/pleroma/web/streamer.ex index 120e94751..3c154288b 100644 --- a/lib/pleroma/web/streamer.ex +++ b/lib/pleroma/web/streamer.ex @@ -195,6 +195,9 @@ defmodule Pleroma.Web.Streamer do recipients = MapSet.new(item.recipients) domain_blocks = Pleroma.Web.ActivityPub.MRF.subdomains_regex(user.domain_blocks) + following_ap_ids = + if user.domain_blocks == [], do: [], else: User.get_cached_user_friends_ap_ids(user) + with parent <- Object.normalize(item, fetch: false) || item, true <- Enum.all?([blocked_ap_ids, muted_ap_ids], &(item.actor not in &1)), true <- item.data["type"] != "Announce" || item.actor not in reblog_muted_ap_ids, @@ -205,8 +208,12 @@ defmodule Pleroma.Web.Streamer do true <- MapSet.disjoint?(recipients, recipient_blocks), %{host: item_host} <- URI.parse(item.actor), %{host: parent_host} <- URI.parse(parent.data["actor"]), - false <- Pleroma.Web.ActivityPub.MRF.subdomain_match?(domain_blocks, item_host), - false <- Pleroma.Web.ActivityPub.MRF.subdomain_match?(domain_blocks, parent_host), + false <- + item.actor not in following_ap_ids and + Pleroma.Web.ActivityPub.MRF.subdomain_match?(domain_blocks, item_host), + false <- + parent.data["actor"] not in following_ap_ids and + Pleroma.Web.ActivityPub.MRF.subdomain_match?(domain_blocks, parent_host), true <- thread_containment(item, user), false <- CommonAPI.thread_muted?(parent, user) do false @@ -289,15 +296,17 @@ defmodule Pleroma.Web.Streamer do defp do_stream(topic, {user, %MessageReference{} = cm_ref}) when topic in ["user", "user:pleroma_chat"] do - topic = "#{topic}:#{user.id}" + if Pleroma.Chat.enabled?() do + topic = "#{topic}:#{user.id}" - text = StreamerView.render("chat_update.json", %{chat_message_reference: cm_ref}, topic) + text = StreamerView.render("chat_update.json", %{chat_message_reference: cm_ref}, topic) - Registry.dispatch(@registry, topic, fn list -> - Enum.each(list, fn {pid, _auth} -> - send(pid, {:text, text}) + Registry.dispatch(@registry, topic, fn list -> + Enum.each(list, fn {pid, _auth} -> + send(pid, {:text, text}) + end) end) - end) + end end defp do_stream("user", %Activity{} = item) do diff --git a/lib/pleroma/web/templates/feed/feed/_activity.atom.eex b/lib/pleroma/web/templates/feed/feed/_activity.atom.eex index b774f7984..f94ba0793 100644 --- a/lib/pleroma/web/templates/feed/feed/_activity.atom.eex +++ b/lib/pleroma/web/templates/feed/feed/_activity.atom.eex @@ -1,9 +1,9 @@ http://activitystrea.ms/schema/1.0/note http://activitystrea.ms/schema/1.0/post - <%= @data["id"] %> + <%= escape(@data["id"] || "") %> <%= activity_title(@data, Keyword.get(@feed_config, :post_title, %{})) %> - <%= activity_description(@data) %> + <%= escape(atom_content_encoded(@data)) %> <%= to_rfc3339(@data["published"]) %> <%= to_rfc3339(@data["published"]) %> @@ -15,35 +15,26 @@ <%= escape(@data["summary"]) %> <% end %> - <%= if @activity.local do %> - - - <% else %> - - <% end %> + <%= for tag <- Pleroma.Object.hashtags(@object) do %> <% end %> - <%= for attachment <- @data["attachment"] || [] do %> - + <%= unless sensitive?(@data) do %> + <%= for attachment <- @data["attachment"] || [] do %> + <%= if size = attachment_size_positive(attachment) do %> + " type="<%= escape(attachment_type(attachment) || "application/octet-stream") %>" length="<%= size %>"/> + <% else %> + " type="<%= escape(attachment_type(attachment) || "application/octet-stream") %>"/> + <% end %> + <% end %> <% end %> <%= if @data["inReplyTo"] do %> <% end %> - <%= for id <- @activity.recipients do %> - <%= if id == Pleroma.Constants.as_public() do %> - - <% else %> - <%= unless Regex.match?(~r/^#{Pleroma.Web.Endpoint.url()}.+followers$/, id) do %> - - <% end %> - <% end %> - <% end %> - <%= for {emoji, file} <- @data["emoji"] || %{} do %> <% end %> diff --git a/lib/pleroma/web/templates/feed/feed/_activity.rss.eex b/lib/pleroma/web/templates/feed/feed/_activity.rss.eex index 7de98f736..4d71a45c7 100644 --- a/lib/pleroma/web/templates/feed/feed/_activity.rss.eex +++ b/lib/pleroma/web/templates/feed/feed/_activity.rss.eex @@ -1,45 +1,34 @@ http://activitystrea.ms/schema/1.0/note http://activitystrea.ms/schema/1.0/post - <%= @data["id"] %> + <%= escape(@data["id"] || "") %> <%= activity_title(@data, Keyword.get(@feed_config, :post_title, %{})) %> <%= activity_description(@data) %> + ]]> <%= to_rfc2822(@data["published"]) %> <%= activity_context(@activity) %> - <%= if @activity.local do %> - <%= @data["id"] %> - <% else %> - <%= @data["external_url"] %> - <% end %> - - <%= activity_context(@activity) %> + <%= escape(activity_link(@activity, @data) || "") %> <%= for tag <- Pleroma.Object.hashtags(@object) do %> - + <%= if tag && tag != "" do %> + <%= tag %> + <% end %> + <% end %> + + <%= unless sensitive?(@data) do %> + <%= with attachment when not is_nil(attachment) <- rss_enclosure_attachment(@data["attachment"]) do %> + " type="<%= escape(attachment_type(attachment) || "application/octet-stream") %>" length="<%= attachment_size_positive(attachment) %>" /> + <% end %> <% end %> <%= for attachment <- @data["attachment"] || [] do %> - + <%= media_content_xml(attachment, sensitive?(@data)) %> <% end %> <%= if @data["inReplyTo"] do %> <% end %> - - <%= for id <- @activity.recipients do %> - <%= if id == Pleroma.Constants.as_public() do %> - http://activityschema.org/collection/public - <% else %> - <%= unless Regex.match?(~r/^#{Pleroma.Web.Endpoint.url()}.+followers$/, id) do %> - <%= id %> - <% end %> - <% end %> - <% end %> - - <%= for {emoji, file} <- @data["emoji"] || %{} do %> - <%= file %> - <% end %> diff --git a/lib/pleroma/web/templates/feed/feed/_tag_activity.atom.eex b/lib/pleroma/web/templates/feed/feed/_tag_activity.atom.eex index 03c222975..6c74c2ab7 100644 --- a/lib/pleroma/web/templates/feed/feed/_tag_activity.atom.eex +++ b/lib/pleroma/web/templates/feed/feed/_tag_activity.atom.eex @@ -4,9 +4,9 @@ <%= render Phoenix.Controller.view_module(@conn), "_tag_author.atom", assigns %> - <%= @data["id"] %> + <%= escape(@data["id"] || "") %> <%= activity_title(@data, Keyword.get(@feed_config, :post_title, %{})) %> - <%= activity_description(@data) %> + <%= escape(atom_content_encoded(@data)) %> <%= to_rfc3339(@data["published"]) %> <%= to_rfc3339(@data["published"]) %> @@ -15,34 +15,25 @@ <%= if @data["summary"] != "" do %> - <%= @data["summary"] %> + <%= escape(@data["summary"]) %> <% end %> - <%= if @activity.local do %> - - - <% else %> - - <% end %> - - <%= for id <- @activity.recipients do %> - <%= if id == Pleroma.Constants.as_public() do %> - - <% else %> - <%= unless Regex.match?(~r/^#{Pleroma.Web.Endpoint.url()}.+followers$/, id) do %> - - <% end %> - <% end %> - <% end %> + <%= for tag <- Pleroma.Object.hashtags(@object) do %> <% end %> + <%= unless sensitive?(@data) do %> + <%= for attachment <- @data["attachment"] || [] do %> + <%= if size = attachment_size_positive(attachment) do %> + " type="<%= escape(attachment_type(attachment) || "application/octet-stream") %>" length="<%= size %>"/> + <% else %> + " type="<%= escape(attachment_type(attachment) || "application/octet-stream") %>"/> + <% end %> + <% end %> + <% end %> + <%= for {emoji, file} <- @data["emoji"] || %{} do %> <% end %> diff --git a/lib/pleroma/web/templates/feed/feed/_tag_activity.xml.eex b/lib/pleroma/web/templates/feed/feed/_tag_activity.xml.eex index 1b8c34b87..64e4c1d81 100644 --- a/lib/pleroma/web/templates/feed/feed/_tag_activity.xml.eex +++ b/lib/pleroma/web/templates/feed/feed/_tag_activity.xml.eex @@ -1,14 +1,24 @@ - <%= activity_title(@data, Keyword.get(@feed_config, :post_title, %{})) %> - - - <%= activity_context(@activity) %> - <%= activity_context(@activity) %> + <%= escape(activity_link(@activity, @data) || "") %> + <%= escape(activity_link(@activity, @data) || "") %> <%= to_rfc2822(@data["published"]) %> - + <%= activity_title(@data, Keyword.get(@feed_config, :post_title, %{})) %> <%= activity_description(@data) %> - <%= for attachment <- @data["attachment"] || [] do %> - + ]]> + + <%= unless sensitive?(@data) do %> + <%= with attachment when not is_nil(attachment) <- rss_enclosure_attachment(@data["attachment"]) do %> + " type="<%= escape(attachment_type(attachment) || "application/octet-stream") %>" length="<%= attachment_size_positive(attachment) %>"/> + <% end %> <% end %> + <%= for attachment <- @data["attachment"] || [] do %> + <%= media_content_xml(attachment, sensitive?(@data)) %> + <% end %> + + <%= for tag <- Pleroma.Object.hashtags(@object) do %> + <%= if tag && tag != "" do %> + <%= tag %> + <% end %> + <% end %> diff --git a/lib/pleroma/web/templates/feed/feed/tag.atom.eex b/lib/pleroma/web/templates/feed/feed/tag.atom.eex index 3449c97ff..fb04544d3 100644 --- a/lib/pleroma/web/templates/feed/feed/tag.atom.eex +++ b/lib/pleroma/web/templates/feed/feed/tag.atom.eex @@ -9,10 +9,12 @@ <%= Routes.tag_feed_url(@conn, :feed, @tag) <> ".atom" %> #<%= @tag %> - <%= Gettext.dpgettext("static_pages", "tag feed description", "These are public toots tagged with #%{tag}. You can interact with them if you have an account anywhere in the fediverse.", tag: @tag) %> + <%= with subtitle when subtitle not in [nil, ""] <- Gettext.dpgettext("static_pages", "tag feed description", "These are public toots tagged with #%{tag}. You can interact with them if you have an account anywhere in the fediverse.", tag: @tag) do %> + <%= subtitle %> + <% end %> <%= feed_logo() %> - <%= most_recent_update(@activities) %> - " type="application/atom+xml"/> + <%= most_recent_update(@activities) || to_rfc3339(NaiveDateTime.utc_now()) %> + <%= for activity <- @activities do %> <%= render Phoenix.Controller.view_module(@conn), "_tag_activity.atom", Map.merge(assigns, prepare_activity(activity, actor: true)) %> diff --git a/lib/pleroma/web/templates/feed/feed/tag.rss.eex b/lib/pleroma/web/templates/feed/feed/tag.rss.eex index a87f9bf50..b74a573e5 100644 --- a/lib/pleroma/web/templates/feed/feed/tag.rss.eex +++ b/lib/pleroma/web/templates/feed/feed/tag.rss.eex @@ -1,6 +1,8 @@ @@ -8,7 +10,12 @@ <%= Gettext.dpgettext("static_pages", "tag feed description", "These are public toots tagged with #%{tag}. You can interact with them if you have an account anywhere in the fediverse.", tag: @tag) %> <%= "#{Routes.tag_feed_url(@conn, :feed, @tag)}.rss" %> <%= feed_logo() %> + <%= feed_logo() %> 2b90d9 + <%= with last_build_date when not is_nil(last_build_date) <- most_recent_update(@activities, :rss) do %> + <%= last_build_date %> + <% end %> + <%= Pleroma.Application.named_version() %> <%= for activity <- @activities do %> <%= render Phoenix.Controller.view_module(@conn), "_tag_activity.xml", Map.merge(assigns, prepare_activity(activity)) %> <% end %> diff --git a/lib/pleroma/web/templates/feed/feed/user.atom.eex b/lib/pleroma/web/templates/feed/feed/user.atom.eex index c2c77cfed..cc6883727 100644 --- a/lib/pleroma/web/templates/feed/feed/user.atom.eex +++ b/lib/pleroma/web/templates/feed/feed/user.atom.eex @@ -8,10 +8,12 @@ <%= Routes.user_feed_url(@conn, :feed, @user.nickname) <> ".atom" %> <%= @user.nickname <> "'s timeline" %> - <%= escape(@user.bio) %> + <%= if @user.bio && @user.bio != "" do %> + <%= escape(@user.bio) %> + <% end %> <%= most_recent_update(@activities, @user, :atom) %> <%= logo(@user) %> - " type="application/atom+xml"/> + <%= render Phoenix.Controller.view_module(@conn), "_author.atom", assigns %> diff --git a/lib/pleroma/web/templates/feed/feed/user.rss.eex b/lib/pleroma/web/templates/feed/feed/user.rss.eex index b907a7e57..e2150d8b6 100644 --- a/lib/pleroma/web/templates/feed/feed/user.rss.eex +++ b/lib/pleroma/web/templates/feed/feed/user.rss.eex @@ -1,26 +1,32 @@ <%= @user.nickname <> "'s timeline" %> - <%= "#{Routes.user_feed_url(@conn, :feed, @user.nickname)}.rss" %> - " + + <%= "#{Routes.user_feed_url(@conn, :feed, @user.nickname)}.rss" %> <%= escape(@user.bio) %> <%= logo(@user) %> <%= @user.nickname <> "'s timeline" %> <%= "#{Routes.user_feed_url(@conn, :feed, @user.nickname)}.rss" %> + <%= most_recent_update(@activities, @user, :rss) %> + <%= logo(@user) %> + <%= Pleroma.Application.named_version() %> <%= render Phoenix.Controller.view_module(@conn), "_author.rss", assigns %> <%= if last_activity(@activities) do %> - <%= "#{Routes.user_feed_url(@conn, :feed, @user.nickname)}.rss?max_id=#{last_activity(@activities).id}" %> +