fix(webhooks): merge develop and harden delivery
This commit is contained in:
Lain Soykaf
commit
03e75d3b3e
254 changed files with 12265 additions and 1143 deletions
|
|
@ -1,85 +0,0 @@
|
|||
when:
|
||||
# Temporary PR validation while arm/v7 Docker builds are being brought up.
|
||||
- event: pull_request
|
||||
path: [ "**/*.ex", "**/*.eex", "**/*.exs", "mix.lock", ".woodpecker/**", "Dockerfile" ]
|
||||
- event: push
|
||||
branch: ${CI_REPO_DEFAULT_BRANCH}
|
||||
path: [ "**/*.ex", "**/*.eex", "**/*.exs", "mix.lock", ".woodpecker/**", "Dockerfile" ]
|
||||
- event: tag
|
||||
- event: manual
|
||||
branch: ${CI_REPO_DEFAULT_BRANCH}
|
||||
- event: manual
|
||||
branch: stable
|
||||
|
||||
# Target the CAX21 arm64 host with native arm32 userspace support, not generic arm64 runners.
|
||||
labels:
|
||||
platform: linux/arm
|
||||
armv7: native
|
||||
|
||||
variables:
|
||||
docker_variables: &docker_variables
|
||||
repo: pleroma/pleroma
|
||||
registry: git.pleroma.social
|
||||
armv7_build_settings: &armv7_build_settings
|
||||
<<: *docker_variables
|
||||
build_args:
|
||||
- ELIXIR_IMG=arm32v7/elixir
|
||||
- ELIXIR_TAG=1.17.3-otp-26-alpine
|
||||
- ALPINE_IMG=arm32v7/alpine
|
||||
- ALPINE_VER=3.20
|
||||
extra_opts: --custom-platform=linux/arm/v7
|
||||
armv7_push_settings: &armv7_push_settings
|
||||
<<: *armv7_build_settings
|
||||
username:
|
||||
from_secret: pleroma-ci-user
|
||||
password:
|
||||
from_secret: pleroma-ci-password
|
||||
|
||||
steps:
|
||||
docker-armv7-pr:
|
||||
image: woodpeckerci/plugin-kaniko:2.3.1
|
||||
when:
|
||||
- event: pull_request
|
||||
settings:
|
||||
<<: *armv7_build_settings
|
||||
dry_run: true
|
||||
tags:
|
||||
- armv7-pr-check
|
||||
|
||||
docker-develop-armv7:
|
||||
image: woodpeckerci/plugin-kaniko:2.3.1
|
||||
when:
|
||||
- event: push
|
||||
branch: ${CI_REPO_DEFAULT_BRANCH}
|
||||
- event: manual
|
||||
branch: ${CI_REPO_DEFAULT_BRANCH}
|
||||
settings:
|
||||
<<: *armv7_push_settings
|
||||
tags:
|
||||
- latest-armv7
|
||||
- develop-armv7
|
||||
- ${CI_COMMIT_SHA:0:8}-armv7
|
||||
|
||||
docker-stable-armv7:
|
||||
image: woodpeckerci/plugin-kaniko:2.3.1
|
||||
when:
|
||||
- evaluate: 'CI_PIPELINE_EVENT == "manual" && CI_COMMIT_BRANCH == "stable" && CI_COMMIT_TAG == ""'
|
||||
settings:
|
||||
<<: *armv7_push_settings
|
||||
tags:
|
||||
- latest-armv7
|
||||
- stable-armv7
|
||||
- ${CI_COMMIT_SHA:0:8}-armv7
|
||||
|
||||
docker-stable-tag-armv7:
|
||||
image: woodpeckerci/plugin-kaniko:2.3.1
|
||||
when:
|
||||
- event: tag
|
||||
- evaluate: 'CI_PIPELINE_EVENT == "manual" && CI_COMMIT_BRANCH == "stable" && CI_COMMIT_TAG != ""'
|
||||
settings:
|
||||
<<: *armv7_push_settings
|
||||
tags:
|
||||
- latest-armv7
|
||||
- stable-armv7
|
||||
- ${CI_COMMIT_SHA:0:8}-armv7
|
||||
- ${CI_COMMIT_TAG}-armv7
|
||||
|
|
@ -27,7 +27,7 @@ steps:
|
|||
settings: &docker_settings
|
||||
registry: "git.pleroma.social"
|
||||
image: "pleroma/pleroma"
|
||||
architectures: [amd64, arm64]
|
||||
architectures: [amd64, arm64, armv7]
|
||||
tags:
|
||||
- latest
|
||||
- develop
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ matrix:
|
|||
platform:
|
||||
- linux/amd64
|
||||
- linux/arm64
|
||||
- linux/arm
|
||||
|
||||
# This is needed for the when clauses below.
|
||||
labels:
|
||||
|
|
@ -26,11 +27,19 @@ variables:
|
|||
from_secret: pleroma-ci-user
|
||||
password:
|
||||
from_secret: pleroma-ci-password
|
||||
kaniko_image: &kaniko_image woodpeckerci/plugin-kaniko:2.3.1
|
||||
armv7_docker_variables: &armv7_docker_variables
|
||||
<<: *docker_variables
|
||||
build_args:
|
||||
- ELIXIR_IMG=docker.io/arm32v7/elixir
|
||||
- ELIXIR_TAG=1.17.3-otp-26-alpine
|
||||
- ALPINE_IMG=docker.io/arm32v7/alpine
|
||||
- ALPINE_VER=3.20
|
||||
extra_opts: --custom-platform=linux/arm/v7
|
||||
kaniko_image: &kaniko_image docker.io/woodpeckerci/plugin-kaniko:2.3.2
|
||||
|
||||
steps:
|
||||
docker-develop-amd64:
|
||||
image: woodpeckerci/plugin-kaniko:2.3.1
|
||||
image: *kaniko_image
|
||||
when:
|
||||
- evaluate: 'platform == "linux/amd64" && CI_COMMIT_BRANCH == "${CI_REPO_DEFAULT_BRANCH}"'
|
||||
settings:
|
||||
|
|
@ -41,7 +50,7 @@ steps:
|
|||
- ${CI_COMMIT_SHA:0:8}-amd64
|
||||
|
||||
docker-develop-arm64:
|
||||
image: woodpeckerci/plugin-kaniko:2.3.1
|
||||
image: *kaniko_image
|
||||
when:
|
||||
- evaluate: 'platform == "linux/arm64" && CI_COMMIT_BRANCH == "${CI_REPO_DEFAULT_BRANCH}"'
|
||||
settings:
|
||||
|
|
@ -51,6 +60,17 @@ steps:
|
|||
- develop-arm64
|
||||
- ${CI_COMMIT_SHA:0:8}-arm64
|
||||
|
||||
docker-develop-armv7:
|
||||
image: *kaniko_image
|
||||
when:
|
||||
- evaluate: 'platform == "linux/arm" && CI_COMMIT_BRANCH == "${CI_REPO_DEFAULT_BRANCH}"'
|
||||
settings:
|
||||
<<: *armv7_docker_variables
|
||||
tags:
|
||||
- latest-armv7
|
||||
- develop-armv7
|
||||
- ${CI_COMMIT_SHA:0:8}-armv7
|
||||
|
||||
docker-stable-amd64:
|
||||
image: *kaniko_image
|
||||
when:
|
||||
|
|
@ -106,3 +126,25 @@ steps:
|
|||
tags:
|
||||
- <<: *arm64_tags
|
||||
- ${CI_COMMIT_TAG}-arm64
|
||||
|
||||
docker-stable-armv7:
|
||||
image: *kaniko_image
|
||||
when:
|
||||
- evaluate: 'platform == "linux/arm" && CI_PIPELINE_EVENT == "manual" && CI_COMMIT_BRANCH == "stable" && CI_COMMIT_TAG == ""'
|
||||
settings:
|
||||
<<: *armv7_docker_variables
|
||||
tags: &armv7_tags
|
||||
- latest-armv7
|
||||
- stable-armv7
|
||||
- ${CI_COMMIT_SHA:0:8}-armv7
|
||||
|
||||
docker-stable-tag-armv7:
|
||||
image: *kaniko_image
|
||||
when:
|
||||
- evaluate: 'platform == "linux/arm" && CI_PIPELINE_EVENT == "tag"'
|
||||
- evaluate: 'platform == "linux/arm" && CI_PIPELINE_EVENT == "manual" && CI_COMMIT_BRANCH == "stable" && CI_COMMIT_TAG != ""'
|
||||
settings:
|
||||
<<: *armv7_docker_variables
|
||||
tags:
|
||||
- <<: *armv7_tags
|
||||
- ${CI_COMMIT_TAG}-armv7
|
||||
|
|
|
|||
64
.woodpecker/paradedb-integration.yaml
Normal file
64
.woodpecker/paradedb-integration.yaml
Normal file
|
|
@ -0,0 +1,64 @@
|
|||
when:
|
||||
- event: pull_request
|
||||
path:
|
||||
- ".woodpecker/**"
|
||||
- "lib/pleroma/search/**"
|
||||
- "test/pleroma/search/**"
|
||||
- "mix.exs"
|
||||
- "mix.lock"
|
||||
evaluate: 'CI_COMMIT_SOURCE_BRANCH != "weblate"'
|
||||
- event: push
|
||||
branch: ${CI_REPO_DEFAULT_BRANCH}
|
||||
path:
|
||||
- ".woodpecker/**"
|
||||
- "lib/pleroma/search/**"
|
||||
- "test/pleroma/search/**"
|
||||
- "mix.exs"
|
||||
- "mix.lock"
|
||||
|
||||
depends_on:
|
||||
- lint
|
||||
|
||||
labels:
|
||||
platform: linux/amd64
|
||||
|
||||
variables:
|
||||
script_file_entrypoint: &script_file_entrypoint
|
||||
- /bin/sh
|
||||
- -c
|
||||
- 'printf "%s" "$CI_SCRIPT" | base64 -d > /tmp/ci-script.sh && /bin/sh -xe /tmp/ci-script.sh'
|
||||
|
||||
steps:
|
||||
paradedb-integration:
|
||||
image: docker.io/elixir:1.19-otp-28-alpine
|
||||
entrypoint: *script_file_entrypoint
|
||||
environment:
|
||||
MIX_ENV: test
|
||||
VIX_COMPILATION_MODE: PLATFORM_PROVIDED_LIBVIPS
|
||||
DB_HOST: postgres
|
||||
DB_PORT: 5432
|
||||
PARADEDB_TEST_URL: postgres://postgres:postgres@paradedb:5432/paradedb
|
||||
commands:
|
||||
- apk add --no-cache build-base cmake exiftool ffmpeg vips-dev file-dev git openssl postgresql-client
|
||||
- mix local.hex --force
|
||||
- mix local.rebar --force
|
||||
- mix deps.get
|
||||
- timeout 60 sh -c 'until pg_isready -h postgres -U postgres -d pleroma_test; do sleep 1; done' || { echo "PostgreSQL did not become ready" >&2; exit 1; }
|
||||
- timeout 60 sh -c 'until pg_isready -h paradedb -U postgres -d paradedb; do sleep 1; done' || { echo "ParadeDB did not become ready" >&2; exit 1; }
|
||||
- mix ecto.create
|
||||
- mix ecto.migrate
|
||||
- mix test --include integration test/pleroma/search/parade_db_integration_test.exs
|
||||
|
||||
services:
|
||||
postgres:
|
||||
image: docker.io/postgres:13-alpine
|
||||
environment:
|
||||
POSTGRES_DB: pleroma_test
|
||||
POSTGRES_USER: postgres
|
||||
POSTGRES_PASSWORD: postgres
|
||||
paradedb:
|
||||
image: docker.io/paradedb/paradedb:0.24.3-pg15@sha256:997d62f6c2605becb7c45878af3f8c8c698f355006b675b3ce212d4eb2d14f2e
|
||||
environment:
|
||||
POSTGRES_DB: paradedb
|
||||
POSTGRES_USER: postgres
|
||||
POSTGRES_PASSWORD: postgres
|
||||
|
|
@ -24,10 +24,11 @@ steps:
|
|||
entrypoint: *script_file_entrypoint
|
||||
environment:
|
||||
MIX_ENV: test
|
||||
VIX_COMPILATION_MODE: PLATFORM_PROVIDED_LIBVIPS
|
||||
DB_HOST: postgres
|
||||
DB_PORT: 5432
|
||||
commands:
|
||||
- apk add --no-cache build-base cmake exiftool ffmpeg file-dev git openssl
|
||||
- apk add --no-cache build-base cmake exiftool ffmpeg vips-dev file-dev git openssl
|
||||
- adduser -D -h /home/testuser testuser
|
||||
- mkdir -p /home/testuser/.mix /home/testuser/.hex
|
||||
- chown -R testuser:testuser . /home/testuser
|
||||
|
|
|
|||
|
|
@ -19,15 +19,16 @@ variables:
|
|||
- 'printf "%s" "$CI_SCRIPT" | base64 -d > /tmp/ci-script.sh && /bin/sh -xe /tmp/ci-script.sh'
|
||||
|
||||
steps:
|
||||
unit-testing-elixir-1.18:
|
||||
image: elixir:1.18-otp-27-alpine
|
||||
unit-testing-elixir-1.19:
|
||||
image: elixir:1.19-otp-28-alpine
|
||||
entrypoint: *script_file_entrypoint
|
||||
environment:
|
||||
MIX_ENV: test
|
||||
VIX_COMPILATION_MODE: PLATFORM_PROVIDED_LIBVIPS
|
||||
DB_HOST: postgres
|
||||
DB_PORT: 5432
|
||||
commands:
|
||||
- apk add --no-cache build-base cmake exiftool ffmpeg file-dev git openssl
|
||||
- apk add --no-cache build-base cmake exiftool ffmpeg vips-dev file-dev git openssl
|
||||
- adduser -D -h /home/testuser testuser
|
||||
- mkdir -p /home/testuser/.mix /home/testuser/.hex
|
||||
- chown -R testuser:testuser . /home/testuser
|
||||
|
|
@ -1,8 +1,8 @@
|
|||
# https://hub.docker.com/r/hexpm/elixir/tags
|
||||
ARG ELIXIR_IMG=hexpm/elixir
|
||||
ARG ELIXIR_VER=1.17.3
|
||||
ARG ERLANG_VER=26.2.5.6
|
||||
ARG ALPINE_VER=3.17.9
|
||||
ARG ERLANG_VER=27.3.4.14
|
||||
ARG ALPINE_VER=3.23.5
|
||||
ARG ELIXIR_TAG=${ELIXIR_VER}-erlang-${ERLANG_VER}-alpine-${ALPINE_VER}
|
||||
ARG ALPINE_IMG=alpine
|
||||
|
||||
|
|
|
|||
1
changelog.d/account-favourites-order.fix
Normal file
1
changelog.d/account-favourites-order.fix
Normal file
|
|
@ -0,0 +1 @@
|
|||
Fixed `/api/v1/pleroma/accounts/:id/favourites` ordering and pagination to use the favourite date.
|
||||
1
changelog.d/activitypub-bot-rfc9421-fallback.fix
Normal file
1
changelog.d/activitypub-bot-rfc9421-fallback.fix
Normal file
|
|
@ -0,0 +1 @@
|
|||
Reject unsupported or ambiguous inbox signatures instead of queueing a doomed failed-signature retry, allowing clients to fall back to legacy HTTP Signatures.
|
||||
1
changelog.d/disabling-chats.change
Normal file
1
changelog.d/disabling-chats.change
Normal file
|
|
@ -0,0 +1 @@
|
|||
Allow disabling Chats feature
|
||||
0
changelog.d/docker-arm32v7-pr-forbid-build.skip
Normal file
0
changelog.d/docker-arm32v7-pr-forbid-build.skip
Normal file
1
changelog.d/docker-baseimage-ssl.fix
Normal file
1
changelog.d/docker-baseimage-ssl.fix
Normal file
|
|
@ -0,0 +1 @@
|
|||
Dockerfile: update base images to fix problems with SSL handshakes
|
||||
1
changelog.d/elixir-1-19.add
Normal file
1
changelog.d/elixir-1-19.add
Normal file
|
|
@ -0,0 +1 @@
|
|||
Implemented ConfigDB configuration blacklist and accompanying Mix task
|
||||
1
changelog.d/elixir-1-19.fix
Normal file
1
changelog.d/elixir-1-19.fix
Normal file
|
|
@ -0,0 +1 @@
|
|||
Fixed Elixir 1.19 warnings
|
||||
1
changelog.d/fedidev-federation-compat.add
Normal file
1
changelog.d/fedidev-federation-compat.add
Normal file
|
|
@ -0,0 +1 @@
|
|||
Accept more real-world ActivityPub object shapes collected by fedidev.fun
|
||||
1
changelog.d/fix-preferred-frontend.fix
Normal file
1
changelog.d/fix-preferred-frontend.fix
Normal file
|
|
@ -0,0 +1 @@
|
|||
Fix preferred frontend not working due to InstanceStatic plug prematurely returning HTTP 404
|
||||
1
changelog.d/grouped-notifications-api.add
Normal file
1
changelog.d/grouped-notifications-api.add
Normal file
|
|
@ -0,0 +1 @@
|
|||
Add Mastodon-compatible grouped notifications API endpoints under `/api/v2/notifications` for newly-created notifications
|
||||
0
changelog.d/grouped_notifications_flag.skip
Normal file
0
changelog.d/grouped_notifications_flag.skip
Normal file
1
changelog.d/gun-pool-rework.fix
Normal file
1
changelog.d/gun-pool-rework.fix
Normal file
|
|
@ -0,0 +1 @@
|
|||
Fixed Gun HTTP/1 concurrency, forwarding and authenticated proxy handling, tunnel requests, and streamed connection cleanup. Updated Gun and Cowlib for HTTP client security fixes.
|
||||
1
changelog.d/hashtag-timeline.fix
Normal file
1
changelog.d/hashtag-timeline.fix
Normal file
|
|
@ -0,0 +1 @@
|
|||
Fix hashtag timeline query for additional tag handling
|
||||
1
changelog.d/list-emoji.add
Normal file
1
changelog.d/list-emoji.add
Normal file
|
|
@ -0,0 +1 @@
|
|||
Allow attaching emojis to user lists
|
||||
1
changelog.d/logger-backends.change
Normal file
1
changelog.d/logger-backends.change
Normal file
|
|
@ -0,0 +1 @@
|
|||
Changed how Logger is configured due to changes in Elixir and added deprecation warnings for the old configuration format
|
||||
1
changelog.d/mastodon-admin.add
Normal file
1
changelog.d/mastodon-admin.add
Normal file
|
|
@ -0,0 +1 @@
|
|||
Implement basics of Mastodon admin API
|
||||
1
changelog.d/mediaproxy-content-type.fix
Normal file
1
changelog.d/mediaproxy-content-type.fix
Normal file
|
|
@ -0,0 +1 @@
|
|||
Detect image content types for extensionless MediaProxy responses.
|
||||
1
changelog.d/oauth-user-preload.change
Normal file
1
changelog.d/oauth-user-preload.change
Normal file
|
|
@ -0,0 +1 @@
|
|||
Avoid a duplicate user query during OAuth authentication.
|
||||
1
changelog.d/paradedb-search.add
Normal file
1
changelog.d/paradedb-search.add
Normal file
|
|
@ -0,0 +1 @@
|
|||
Add ParadeDB (pg_search) as an optional external search backend, indexing status content, content warnings, and attachment descriptions.
|
||||
1
changelog.d/previewcard-void-fill.fix
Normal file
1
changelog.d/previewcard-void-fill.fix
Normal file
|
|
@ -0,0 +1 @@
|
|||
Status: Fill all MastoAPI required values in preview cards
|
||||
1
changelog.d/publisher-worker-atoms.fix
Normal file
1
changelog.d/publisher-worker-atoms.fix
Normal file
|
|
@ -0,0 +1 @@
|
|||
Avoid creating atoms from serialized PublisherWorker parameters.
|
||||
1
changelog.d/rss.fix
Normal file
1
changelog.d/rss.fix
Normal file
|
|
@ -0,0 +1 @@
|
|||
Fix RSS feeds to be W3C compliant and render better in RSS readers
|
||||
1
changelog.d/stream-domain-block-following.fix
Normal file
1
changelog.d/stream-domain-block-following.fix
Normal file
|
|
@ -0,0 +1 @@
|
|||
Keep followed users visible in streams when their domain is blocked.
|
||||
0
changelog.d/unit-testing-build-vix.skip
Normal file
0
changelog.d/unit-testing-build-vix.skip
Normal file
1
changelog.d/user-cache-keys.fix
Normal file
1
changelog.d/user-cache-keys.fix
Normal file
|
|
@ -0,0 +1 @@
|
|||
Fix inconsistent User cache keys and invalidate numeric ID aliases.
|
||||
1
changelog.d/woodpecker-armv7-fixups.fix
Normal file
1
changelog.d/woodpecker-armv7-fixups.fix
Normal file
|
|
@ -0,0 +1 @@
|
|||
Include arm32v7 in multi-arch Docker images
|
||||
|
|
@ -133,13 +133,17 @@ config :pleroma, Pleroma.Web.Endpoint,
|
|||
]
|
||||
|
||||
# Configures Elixir's Logger
|
||||
config :logger, backends: [:console]
|
||||
# Primary config
|
||||
config :logger, level: :debug
|
||||
|
||||
config :logger, :console,
|
||||
level: :debug,
|
||||
# Console config
|
||||
config :logger, :default_handler, level: :debug
|
||||
|
||||
config :logger, :default_formatter,
|
||||
format: "\n$time $metadata[$level] $message\n",
|
||||
metadata: [:actor, :path, :type, :user]
|
||||
|
||||
# Syslog config
|
||||
config :logger, :ex_syslogger,
|
||||
level: :debug,
|
||||
ident: "pleroma",
|
||||
|
|
@ -941,6 +945,15 @@ config :pleroma, Pleroma.Search.Meilisearch,
|
|||
private_key: nil,
|
||||
initial_indexing_chunk_size: 100_000
|
||||
|
||||
config :pleroma, Pleroma.Search.ParadeDB,
|
||||
url: nil,
|
||||
table: "pleroma_search_documents",
|
||||
fuzzy_distance: 0
|
||||
|
||||
config :pleroma, Pleroma.Search.ParadeDB.Repo,
|
||||
pool_size: 2,
|
||||
prepare: :unnamed
|
||||
|
||||
config :pleroma, Pleroma.Application,
|
||||
background_migrators: true,
|
||||
internal_fetch: true,
|
||||
|
|
@ -963,6 +976,10 @@ config :pleroma, Pleroma.Search.QdrantSearch,
|
|||
vectors: %{size: 384, distance: "Cosine"}
|
||||
}
|
||||
|
||||
config :pleroma, :database_config_blacklist, [
|
||||
{:pleroma, :logger}
|
||||
]
|
||||
|
||||
config :pleroma, :database_config_whitelist, [
|
||||
{:pleroma},
|
||||
{:cors_plug},
|
||||
|
|
@ -972,6 +989,8 @@ config :pleroma, :database_config_whitelist, [
|
|||
{:web_push_encryption, :vapid_details}
|
||||
]
|
||||
|
||||
config :pleroma, Pleroma.Chat, enabled: true
|
||||
|
||||
# Import environment specific config. This must remain at the bottom
|
||||
# of this file so it overrides the configuration defined above.
|
||||
import_config "#{Mix.env()}.exs"
|
||||
|
|
|
|||
|
|
@ -2726,6 +2726,19 @@ config :pleroma, :config_description, [
|
|||
}
|
||||
]
|
||||
},
|
||||
%{
|
||||
group: :pleroma,
|
||||
key: Pleroma.Chat,
|
||||
type: :group,
|
||||
description: "Pleroma Chat settings",
|
||||
children: [
|
||||
%{
|
||||
key: :enabled,
|
||||
type: :boolean,
|
||||
description: "Enables the chats API."
|
||||
}
|
||||
]
|
||||
},
|
||||
%{
|
||||
group: :pleroma,
|
||||
key: :http,
|
||||
|
|
@ -3085,9 +3098,16 @@ config :pleroma, :config_description, [
|
|||
%{
|
||||
key: :max_connections,
|
||||
type: :integer,
|
||||
description: "Maximum number of connections in the pool. Default: 250 connections.",
|
||||
description:
|
||||
"Maximum total number of connections in the pool. HTTP/1 origins may use multiple connections within this limit, while HTTP/2 connections are multiplexed. Default: 250 connections.",
|
||||
suggestions: [250]
|
||||
},
|
||||
%{
|
||||
key: :max_idle_time,
|
||||
type: :integer,
|
||||
description: "Time before an unused connection is closed. Default: 30000ms.",
|
||||
suggestions: [30_000]
|
||||
},
|
||||
%{
|
||||
key: :connect_timeout,
|
||||
type: :integer,
|
||||
|
|
@ -3508,7 +3528,7 @@ config :pleroma, :config_description, [
|
|||
key: Pleroma.Webhook.Notify,
|
||||
type: :keyword,
|
||||
description: "Concurrent limits configuration for webhooks.",
|
||||
suggestions: [max_running: 5, max_waiting: 5],
|
||||
suggestions: [max_running: 5, max_waiting: 200],
|
||||
children: [
|
||||
%{
|
||||
key: :max_running,
|
||||
|
|
@ -3520,7 +3540,7 @@ config :pleroma, :config_description, [
|
|||
key: :max_waiting,
|
||||
type: :integer,
|
||||
description: "Max waiting jobs.",
|
||||
suggestion: [5]
|
||||
suggestion: [200]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -3536,7 +3556,12 @@ config :pleroma, :config_description, [
|
|||
key: :module,
|
||||
type: :keyword,
|
||||
description: "Selected search module.",
|
||||
suggestion: [Pleroma.Search.DatabaseSearch, Pleroma.Search.Meilisearch]
|
||||
suggestion: [
|
||||
Pleroma.Search.DatabaseSearch,
|
||||
Pleroma.Search.Meilisearch,
|
||||
Pleroma.Search.QdrantSearch,
|
||||
Pleroma.Search.ParadeDB
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
|
|
@ -3569,6 +3594,34 @@ config :pleroma, :config_description, [
|
|||
}
|
||||
]
|
||||
},
|
||||
%{
|
||||
group: :pleroma,
|
||||
key: Pleroma.Search.ParadeDB,
|
||||
type: :group,
|
||||
description: "ParadeDB settings.",
|
||||
children: [
|
||||
%{
|
||||
key: :url,
|
||||
type: :string,
|
||||
description:
|
||||
"ParadeDB database URL (separate Postgres instance). " <>
|
||||
"Can also be set via `PARADEDB_DATABASE_URL`.",
|
||||
suggestion: ["postgres://postgres:postgres@127.0.0.1:5432/paradedb"]
|
||||
},
|
||||
%{
|
||||
key: :table,
|
||||
type: :string,
|
||||
description: "Table name used for search documents.",
|
||||
suggestion: ["pleroma_search_documents"]
|
||||
},
|
||||
%{
|
||||
key: :fuzzy_distance,
|
||||
type: :integer,
|
||||
description: "Edit-distance used for fuzzy token matching (0 disables fuzziness, max 2).",
|
||||
suggestion: [0, 1, 2]
|
||||
}
|
||||
]
|
||||
},
|
||||
%{
|
||||
group: :pleroma,
|
||||
key: Pleroma.Language.LanguageDetector,
|
||||
|
|
|
|||
|
|
@ -36,7 +36,7 @@ config :pleroma, Pleroma.Emails.Mailer, adapter: Swoosh.Adapters.Local
|
|||
# different ports.
|
||||
|
||||
# Do not include timestamps in development logs
|
||||
config :logger, Logger.Backends.Console, format: "$metadata[$level] $message\n"
|
||||
config :logger, :default_formatter, format: "$metadata[$level] $message\n"
|
||||
|
||||
# Set a higher stacktrace during development. Avoid configuring such
|
||||
# in production as building large stacktraces may be expensive.
|
||||
|
|
|
|||
|
|
@ -20,8 +20,7 @@ config :pleroma, Pleroma.Web.Endpoint,
|
|||
config :phoenix, serve_endpoints: true
|
||||
|
||||
# Do not print debug messages in production
|
||||
config :logger, Logger.Backends.Console, level: :info
|
||||
config :logger, :console, level: :info
|
||||
config :logger, :default_handler, level: :info
|
||||
config :logger, :ex_syslogger, level: :info
|
||||
|
||||
# ## SSL Support
|
||||
|
|
|
|||
|
|
@ -15,9 +15,11 @@ config :pleroma, Pleroma.Captcha,
|
|||
method: Pleroma.Captcha.Mock
|
||||
|
||||
# Print only warnings and errors during test
|
||||
config :logger, :console,
|
||||
level: :warning,
|
||||
format: "\n[$level] $message\n"
|
||||
config :logger, :default_handler, level: :warning
|
||||
|
||||
config :logger, :default_formatter,
|
||||
format: "\n[$level] $message\n",
|
||||
colors: [enabled: false]
|
||||
|
||||
config :pleroma, :auth, oauth_consumer_strategies: []
|
||||
|
||||
|
|
|
|||
|
|
@ -573,7 +573,8 @@ Settings for HTTP connection pool.
|
|||
|
||||
* `:connection_acquisition_wait` - Timeout to acquire a connection from pool.The total max time is this value multiplied by the number of retries.
|
||||
* `connection_acquisition_retries` - Number of attempts to acquire the connection from the pool if it is overloaded. Each attempt is timed `:connection_acquisition_wait` apart.
|
||||
* `:max_connections` - Maximum number of connections in the pool.
|
||||
* `:max_connections` - Maximum total number of connections in the pool. HTTP/1 origins may use multiple connections within this limit, while HTTP/2 connections are multiplexed.
|
||||
* `:max_idle_time` - Time before an unused connection is closed.
|
||||
* `:connect_timeout` - Timeout to connect to the host.
|
||||
* `:reclaim_multiplier` - Multiplied by `:max_connections` this will be the maximum number of idle connections that will be reclaimed in case the pool is overloaded.
|
||||
|
||||
|
|
@ -858,21 +859,23 @@ Web Push Notifications configuration. You can use the mix task `mix web_push.gen
|
|||
* ``private_key``: VAPID private key
|
||||
|
||||
## :logger
|
||||
* `backends`: `:console` is used to send logs to stdout, `{ExSyslogger, :ex_syslogger}` to log to syslog
|
||||
* Logging to console/stdout is done by default, use `{ExSyslogger, :ex_syslogger}` to log to syslog
|
||||
|
||||
An example to enable ONLY ExSyslogger (f/ex in ``prod.secret.exs``) with info and debug suppressed:
|
||||
```elixir
|
||||
config :logger,
|
||||
config :pleroma, :logger,
|
||||
backends: [{ExSyslogger, :ex_syslogger}]
|
||||
|
||||
config :logger, default_handler: false
|
||||
|
||||
config :logger, :ex_syslogger,
|
||||
level: :warning
|
||||
```
|
||||
|
||||
Another example, keeping console output and adding the pid to syslog output:
|
||||
```elixir
|
||||
config :logger,
|
||||
backends: [:console, {ExSyslogger, :ex_syslogger}]
|
||||
config :pleroma, :logger,
|
||||
backends: [{ExSyslogger, :ex_syslogger}]
|
||||
|
||||
config :logger, :ex_syslogger,
|
||||
level: :warning,
|
||||
|
|
@ -883,23 +886,22 @@ See: [logger’s documentation](https://hexdocs.pm/logger/Logger.html) and [ex_s
|
|||
|
||||
An example of logging info to local syslog, but debug to console:
|
||||
```elixir
|
||||
config :logger,
|
||||
backends: [ {ExSyslogger, :ex_syslogger}, :console ],
|
||||
level: :info
|
||||
config :pleroma, :logger,
|
||||
backends: [{ExSyslogger, :ex_syslogger}]
|
||||
|
||||
config :logger, :ex_syslogger,
|
||||
level: :info,
|
||||
ident: "pleroma",
|
||||
format: "$metadata[$level] $message"
|
||||
|
||||
config :logger, :console,
|
||||
level: :debug,
|
||||
config :logger, :default_handler,
|
||||
level: :debug
|
||||
|
||||
config :logger, :default_formatter,
|
||||
format: "\n$time $metadata[$level] $message\n",
|
||||
metadata: [:request_id]
|
||||
```
|
||||
|
||||
|
||||
|
||||
## Database options
|
||||
|
||||
### RUM indexing for full text search
|
||||
|
|
|
|||
|
|
@ -10,3 +10,6 @@ The other way to do it, for example, with Tor you would most likely add somethin
|
|||
```
|
||||
config :pleroma, :http, proxy_url: {:socks5, :localhost, 9050}
|
||||
```
|
||||
|
||||
The Gun HTTP adapter supports SOCKS5. SOCKS4 proxies require another HTTP adapter.
|
||||
For regular HTTP proxies, Gun forwards plain HTTP requests directly and uses CONNECT tunnels for HTTPS requests.
|
||||
|
|
|
|||
|
|
@ -85,7 +85,7 @@ You will see a "Default Admin API Key", this is the key you actually put into yo
|
|||
|
||||
### Initial indexing
|
||||
|
||||
After setting up the configuration, you'll want to index all of your already existing posts. Only public posts are indexed. You'll only
|
||||
After setting up the configuration, you'll want to index all of your already existing posts. Only public/unlisted posts are indexed. You'll only
|
||||
have to do it one time, but it might take a while, depending on the amount of posts your instance has seen. This is also a fairly RAM
|
||||
consuming process for `meilisearch`, and it will take a lot of RAM when running if you have a lot of posts (seems to be around 5G for ~1.2
|
||||
million posts while idle and up to 7G while indexing initially, but your experience may be different).
|
||||
|
|
@ -145,3 +145,155 @@ This will clear **all** the posts from the search index. Note, that deleted post
|
|||
there is no need to actually clear the whole index, unless you want **all** of it gone. That said, the index does not hold any information
|
||||
that cannot be re-created from the database, it should also generally be a lot smaller than the size of your database. Still, the size
|
||||
depends on the amount of text in posts.
|
||||
|
||||
## ParadeDB
|
||||
|
||||
[ParadeDB](https://www.paradedb.com/) is a Postgres extension that provides BM25 full-text search. In Pleroma, it can be used as an
|
||||
external search backend by pointing Pleroma at a separate Postgres instance with the `pg_search` extension installed.
|
||||
Pleroma's integration suite tests `pg_search` 0.24.3 on PostgreSQL 15.
|
||||
|
||||
Pleroma will maintain a small `pleroma_search_documents` table in that database (via the existing search indexing queue) and run search
|
||||
queries against it.
|
||||
The indexed text includes status content, content warnings, and attachment descriptions.
|
||||
|
||||
### Configuration
|
||||
|
||||
Set the search module to `Pleroma.Search.ParadeDB` and configure the ParadeDB database URL. These settings must be in static/runtime config
|
||||
(or `PARADEDB_DATABASE_URL`) so the dedicated ParadeDB Repo is started with the application. They cannot be migrated to or changed through
|
||||
ConfigDB.
|
||||
|
||||
> config :pleroma, Pleroma.Search, module: Pleroma.Search.ParadeDB
|
||||
>
|
||||
> config :pleroma, Pleroma.Search.ParadeDB,
|
||||
> url: System.get_env("PARADEDB_DATABASE_URL"),
|
||||
> table: "pleroma_search_documents",
|
||||
> fuzzy_distance: 0
|
||||
>
|
||||
> config :pleroma, Pleroma.Search.ParadeDB.Repo,
|
||||
> pool_size: 2,
|
||||
> prepare: :unnamed
|
||||
|
||||
When connecting over an untrusted network, enable certificate verification in the Repo configuration. For example:
|
||||
|
||||
> config :pleroma, Pleroma.Search.ParadeDB.Repo,
|
||||
> ssl: [
|
||||
> verify: :verify_peer,
|
||||
> cacertfile: "/etc/pleroma/paradedb-ca.pem",
|
||||
> server_name_indication: ~c"paradedb.example.com"
|
||||
> ]
|
||||
|
||||
### Initial rollout
|
||||
|
||||
The `pg_search` extension must already be available in the database. Creating the extension normally requires an administrative database
|
||||
role. The Pleroma runtime role requires `CONNECT` on the database, `USAGE` on the table's schema and the `pdb` schema, and `SELECT`, `INSERT`, `UPDATE`,
|
||||
and `DELETE` on the generated table. For example, after creating the table as its owner:
|
||||
|
||||
```sql
|
||||
GRANT CONNECT ON DATABASE paradedb TO pleroma_search;
|
||||
GRANT USAGE ON SCHEMA public, pdb TO pleroma_search;
|
||||
GRANT SELECT, INSERT, UPDATE, DELETE ON TABLE pleroma_search_documents TO pleroma_search;
|
||||
```
|
||||
|
||||
Use this rollout order to prevent jobs from being handled by the previous backend:
|
||||
|
||||
1. Install `pg_search` 0.24.3 on PostgreSQL 15 or another deliberately tested combination.
|
||||
2. Stop Pleroma so no search indexing jobs are running.
|
||||
3. Put the ParadeDB backend in static configuration.
|
||||
4. Create the table and BM25 index in a one-shot offline process using `PARADEDB_DATABASE_URL` with the table owner/setup credential.
|
||||
5. Apply the runtime grants shown above, set the service's `PARADEDB_DATABASE_URL` to the restricted runtime credential, and start Pleroma.
|
||||
6. Start the initial backfill and monitor the `search_indexing` queue.
|
||||
|
||||
Create the table and BM25 index once with:
|
||||
|
||||
=== "OTP"
|
||||
```sh
|
||||
PARADEDB_DATABASE_URL='OWNER_URL' PLEROMA_CTL_RPC_DISABLED=true \
|
||||
./bin/pleroma_ctl search.indexer create_index
|
||||
```
|
||||
|
||||
=== "From Source"
|
||||
```sh
|
||||
PARADEDB_DATABASE_URL='OWNER_URL' mix pleroma.search.indexer create_index
|
||||
```
|
||||
|
||||
The task currently prints an error without returning a failing process status. Verify that it prints `Index created` before continuing.
|
||||
|
||||
### Backfill
|
||||
|
||||
Run an initial indexing pass to enqueue existing posts:
|
||||
|
||||
=== "OTP"
|
||||
```sh
|
||||
./bin/pleroma_ctl search.indexer index
|
||||
```
|
||||
|
||||
=== "From Source"
|
||||
```sh
|
||||
PARADEDB_DATABASE_URL='RUNTIME_URL' mix pleroma.search.indexer index
|
||||
```
|
||||
|
||||
The command enqueues up to 100,000 Create activities by default. It uses keyset pagination, so new activities arriving during the backfill
|
||||
do not shift the remaining pages. The defaults and accepted options are:
|
||||
|
||||
* `--limit`: maximum activities to enqueue in this run; defaults to `100000` and must be positive
|
||||
* `--step`: activities fetched from Pleroma's database per page; defaults to `1000` and must be positive
|
||||
* `--chunk`: jobs inserted into Oban per batch; defaults to `100` and must be positive
|
||||
* `--before`: resume before the checkpoint activity ID printed by an earlier run; the cursor is exclusive
|
||||
|
||||
The checkpoint is the last activity enqueued, not confirmation that its job succeeded. Repeat the command with its last `--before`
|
||||
checkpoint only when it reports that more activities remain. Inspect failed or discarded Oban jobs as well as waiting jobs before declaring
|
||||
the backfill complete.
|
||||
|
||||
The same operation can be run from an attached console while Pleroma is online:
|
||||
|
||||
```elixir
|
||||
Pleroma.Search.Backfill.run(limit: 10_000)
|
||||
```
|
||||
|
||||
The returned `next_cursor` can be supplied as `before` when `exhausted` is `false`. The attached API also accepts `limit: :infinity`, but this
|
||||
can create an unbounded number of Oban rows; bounded runs are safer on large instances. An `on_page` callback may be supplied for progress
|
||||
reporting.
|
||||
|
||||
Note: Like the other external search backends, only public/unlisted Notes are indexed.
|
||||
|
||||
Set `fuzzy_distance` to `1` or `2` to allow typo-tolerant matching (`2` is the maximum ParadeDB supports).
|
||||
|
||||
Indexing runs via the Oban `search_indexing` queue. Its stock configuration starts paused so the search health monitor can control it; verify
|
||||
that the queue is running before backfilling. A manually managed queue can be enabled with `search_indexing: 10` in the Oban `queues`
|
||||
configuration instead of a `paused: true` entry.
|
||||
Monitor that queue until the backfill jobs have drained before evaluating search completeness.
|
||||
|
||||
### Rebuilding
|
||||
|
||||
Stop Pleroma, or otherwise ensure no search indexing jobs are running, before rebuilding. While ParadeDB is still the configured backend,
|
||||
run `drop_index`, then `create_index` with the table owner/setup credential. Reapply runtime grants after recreating the table, restart
|
||||
Pleroma, and begin a new backfill. The DDL tasks print failures without returning a failing process status, so verify the exact `Index
|
||||
dropped` and `Index created` success messages. Dropping the table removes only derived search data; Pleroma activities remain in the primary
|
||||
database.
|
||||
|
||||
### Rolling back
|
||||
|
||||
To retain the ParadeDB table for a later retry, configure `Pleroma.Search.DatabaseSearch` (or the previous external backend) in static config
|
||||
and restart Pleroma.
|
||||
|
||||
Before switching backends, inspect the `search_indexing` queue. Drain or cancel pending backfill jobs, or explicitly accept that they will run
|
||||
against the newly configured backend after restart.
|
||||
|
||||
To remove the table during rollback, first stop Pleroma and run `drop_index` with the table owner/setup credential while ParadeDB is still
|
||||
configured:
|
||||
|
||||
=== "OTP"
|
||||
```sh
|
||||
PARADEDB_DATABASE_URL='OWNER_URL' PLEROMA_CTL_RPC_DISABLED=true \
|
||||
./bin/pleroma_ctl search.indexer drop_index
|
||||
```
|
||||
|
||||
=== "From Source"
|
||||
```sh
|
||||
PARADEDB_DATABASE_URL='OWNER_URL' mix pleroma.search.indexer drop_index
|
||||
```
|
||||
|
||||
Verify that the task prints `Index dropped`. Then change the backend in static config and restart. The ParadeDB environment variable may be
|
||||
removed after removing the ParadeDB stanza or switching to a backend that does not start its Repo. After switching away from ParadeDB, the
|
||||
generic `drop_index` task targets the newly configured backend and will not remove the ParadeDB table; remove it directly with the database
|
||||
owner instead.
|
||||
|
|
|
|||
|
|
@ -1,8 +1,8 @@
|
|||
## Required dependencies
|
||||
|
||||
* PostgreSQL >=11.0
|
||||
* Elixir >=1.15.0 <1.19
|
||||
* Erlang OTP >=23.0.0 (supported: <28)
|
||||
* Elixir >=1.15.0, <1.20
|
||||
* Erlang OTP >=24.0.0 (supported: <29)
|
||||
* git
|
||||
* file / libmagic
|
||||
* gcc or clang
|
||||
|
|
|
|||
|
|
@ -62,7 +62,8 @@ defmodule Mix.Pleroma do
|
|||
{Majic.Pool,
|
||||
[name: Pleroma.MajicPool, pool_size: Pleroma.Config.get([:majic_pool, :size], 2)]}
|
||||
] ++
|
||||
http_children(adapter)
|
||||
http_children(adapter) ++
|
||||
search_children()
|
||||
|
||||
cachex_children = Enum.map(@cachex_children, &Pleroma.Application.build_cachex(&1, []))
|
||||
|
||||
|
|
@ -141,4 +142,14 @@ defmodule Mix.Pleroma do
|
|||
end
|
||||
|
||||
defp http_children(_), do: []
|
||||
|
||||
defp search_children do
|
||||
case Pleroma.Config.get([Pleroma.Search, :module]) do
|
||||
Pleroma.Search.ParadeDB ->
|
||||
[Pleroma.Search.ParadeDB.Repo]
|
||||
|
||||
_ ->
|
||||
[]
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
|
|||
|
|
@ -252,40 +252,43 @@ defmodule Mix.Tasks.Pleroma.Config do
|
|||
if whitelisted_configs in [nil, false] do
|
||||
shell_error("No unwanted settings in ConfigDB. No changes made.")
|
||||
else
|
||||
whitelisted_groups =
|
||||
whitelisted_configs
|
||||
|> Enum.filter(fn
|
||||
{_group} -> true
|
||||
_ -> false
|
||||
end)
|
||||
|> Enum.map(fn {group} -> group end)
|
||||
|
||||
whitelisted_keys =
|
||||
whitelisted_configs
|
||||
|> Enum.filter(fn
|
||||
{_group, _key} -> true
|
||||
_ -> false
|
||||
end)
|
||||
{whitelisted_groups, whitelisted_keys} = get_filters(whitelisted_configs)
|
||||
|
||||
filtered =
|
||||
from(c in ConfigDB)
|
||||
|> Repo.all()
|
||||
|> Enum.filter(¬_whitelisted?(&1, whitelisted_groups, whitelisted_keys))
|
||||
|
||||
if not Enum.empty?(filtered) do
|
||||
shell_info("The following settings will be removed from ConfigDB:\n")
|
||||
Enum.each(filtered, &dump(&1))
|
||||
|
||||
if force or shell_prompt("Are you sure you want to continue?", "n") in ~w(Yn Y y) do
|
||||
filtered_ids = Enum.map(filtered, fn %{id: id} -> id end)
|
||||
|
||||
Repo.delete_all(from(c in ConfigDB, where: c.id in ^filtered_ids))
|
||||
else
|
||||
shell_error("No changes made.")
|
||||
do_config_filter(filtered, force)
|
||||
end
|
||||
else
|
||||
end
|
||||
|
||||
# Removes blacklisted configuration sections
|
||||
def run(["filter_blacklisted" | rest]) do
|
||||
{options, [], []} =
|
||||
OptionParser.parse(
|
||||
rest,
|
||||
strict: [force: :boolean],
|
||||
aliases: [f: :force]
|
||||
)
|
||||
|
||||
force = Keyword.get(options, :force, false)
|
||||
|
||||
start_pleroma()
|
||||
|
||||
blacklisted_configs = Pleroma.Config.get(:database_config_blacklist)
|
||||
|
||||
if blacklisted_configs in [nil, false] do
|
||||
shell_error("No unwanted settings in ConfigDB. No changes made.")
|
||||
end
|
||||
else
|
||||
{blacklisted_groups, blacklisted_keys} = get_filters(blacklisted_configs)
|
||||
|
||||
filtered =
|
||||
from(c in ConfigDB)
|
||||
|> Repo.all()
|
||||
|> Enum.filter(&blacklisted?(&1, blacklisted_groups, blacklisted_keys))
|
||||
|
||||
do_config_filter(filtered, force)
|
||||
end
|
||||
end
|
||||
|
||||
|
|
@ -332,9 +335,12 @@ defmodule Mix.Tasks.Pleroma.Config do
|
|||
group
|
||||
|> Pleroma.Config.Loader.filter_group(settings)
|
||||
|> Enum.each(fn {key, value} ->
|
||||
if ConfigDB.static?(group, key) do
|
||||
shell_info("Settings for key #{key} must remain in static configuration; skipping.")
|
||||
else
|
||||
{:ok, _} = ConfigDB.update_or_create(%{group: group, key: key, value: value})
|
||||
|
||||
shell_info("Settings for key #{key} migrated.")
|
||||
end
|
||||
end)
|
||||
|
||||
shell_info("Settings for group #{inspect(group)} migrated.")
|
||||
|
|
@ -490,8 +496,49 @@ defmodule Mix.Tasks.Pleroma.Config do
|
|||
Ecto.Adapters.SQL.query!(Repo, "ALTER SEQUENCE config_id_seq RESTART;")
|
||||
end
|
||||
|
||||
defp get_filters(config) do
|
||||
groups =
|
||||
config
|
||||
|> Enum.filter(fn
|
||||
{_group} -> true
|
||||
_ -> false
|
||||
end)
|
||||
|> Enum.map(fn {group} -> group end)
|
||||
|
||||
keys =
|
||||
config
|
||||
|> Enum.filter(fn
|
||||
{_group, _key} -> true
|
||||
_ -> false
|
||||
end)
|
||||
|
||||
{groups, keys}
|
||||
end
|
||||
|
||||
defp not_whitelisted?(%{group: group, key: key}, whitelisted_groups, whitelisted_keys) do
|
||||
not Enum.member?(whitelisted_groups, group) and
|
||||
not Enum.member?(whitelisted_keys, {group, key})
|
||||
end
|
||||
|
||||
defp blacklisted?(%{group: group, key: key}, blacklisted_groups, blacklisted_keys) do
|
||||
Enum.member?(blacklisted_groups, group) or
|
||||
Enum.member?(blacklisted_keys, {group, key})
|
||||
end
|
||||
|
||||
defp do_config_filter(filtered, _) when length(filtered) == 0 do
|
||||
shell_error("No unwanted settings in ConfigDB. No changes made.")
|
||||
end
|
||||
|
||||
defp do_config_filter(filtered, force) do
|
||||
shell_info("The following settings will be removed from ConfigDB:\n")
|
||||
Enum.each(filtered, &dump(&1))
|
||||
|
||||
if force or shell_prompt("Are you sure you want to continue?", "n") in ~w(Yn Y y) do
|
||||
filtered_ids = Enum.map(filtered, fn %{id: id} -> id end)
|
||||
|
||||
Repo.delete_all(from(c in ConfigDB, where: c.id in ^filtered_ids))
|
||||
else
|
||||
shell_error("No changes made.")
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
|
|||
|
|
@ -4,9 +4,6 @@
|
|||
|
||||
defmodule Mix.Tasks.Pleroma.Search.Indexer do
|
||||
import Mix.Pleroma
|
||||
import Ecto.Query
|
||||
|
||||
alias Pleroma.Workers.SearchIndexingWorker
|
||||
|
||||
def run(["create_index"]) do
|
||||
start_pleroma()
|
||||
|
|
@ -35,49 +32,30 @@ defmodule Mix.Tasks.Pleroma.Search.Indexer do
|
|||
strict: [
|
||||
chunk: :integer,
|
||||
limit: :integer,
|
||||
step: :integer
|
||||
step: :integer,
|
||||
before: :string
|
||||
]
|
||||
)
|
||||
|
||||
start_pleroma()
|
||||
|
||||
chunk_size = Keyword.get(options, :chunk, 100)
|
||||
limit = Keyword.get(options, :limit, 100_000)
|
||||
per_step = Keyword.get(options, :step, 1000)
|
||||
|
||||
chunks = max(div(limit, per_step), 1)
|
||||
|
||||
1..chunks
|
||||
|> Enum.each(fn step ->
|
||||
q =
|
||||
from(a in Pleroma.Activity,
|
||||
limit: ^per_step,
|
||||
offset: ^per_step * (^step - 1),
|
||||
select: [:id],
|
||||
order_by: [desc: :id]
|
||||
result =
|
||||
Pleroma.Search.Backfill.run(
|
||||
options ++
|
||||
[
|
||||
on_page: fn page ->
|
||||
IO.puts(
|
||||
"Queued #{page.count} activities (#{page.enqueued} total), " <>
|
||||
"checkpoint: #{page.next_cursor}"
|
||||
)
|
||||
end
|
||||
]
|
||||
)
|
||||
|
||||
{:ok, ids} =
|
||||
Pleroma.Repo.transaction(fn ->
|
||||
Pleroma.Repo.stream(q, timeout: :infinity)
|
||||
|> Enum.map(fn a ->
|
||||
a.id
|
||||
end)
|
||||
end)
|
||||
IO.puts("Queued #{result.enqueued} activities")
|
||||
|
||||
IO.puts("Got #{length(ids)} activities, adding to indexer")
|
||||
|
||||
ids
|
||||
|> Enum.chunk_every(chunk_size)
|
||||
|> Enum.each(fn chunk ->
|
||||
IO.puts("Adding #{length(chunk)} activities to indexing queue")
|
||||
|
||||
chunk
|
||||
|> Enum.map(fn id ->
|
||||
SearchIndexingWorker.new(%{"op" => "add_to_index", "activity" => id})
|
||||
end)
|
||||
|> Oban.insert_all()
|
||||
end)
|
||||
end)
|
||||
unless result.exhausted do
|
||||
IO.puts("Resume with --before #{result.next_cursor}")
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
|
|||
|
|
@ -43,6 +43,7 @@ defmodule Pleroma.Application do
|
|||
# every time the application is restarted, so we disable module
|
||||
# conflicts at runtime
|
||||
Code.compiler_options(ignore_module_conflict: true)
|
||||
configure_logger()
|
||||
Pleroma.Telemetry.Logger.attach()
|
||||
Config.Holder.save_default()
|
||||
Pleroma.HTML.compile_scrubbers()
|
||||
|
|
@ -80,7 +81,10 @@ defmodule Pleroma.Application do
|
|||
[
|
||||
Pleroma.PromEx,
|
||||
Pleroma.LDAP,
|
||||
Pleroma.Repo,
|
||||
Pleroma.Repo
|
||||
] ++
|
||||
search_children() ++
|
||||
[
|
||||
Config.TransferTask,
|
||||
Pleroma.Emoji,
|
||||
Pleroma.Web.Plugs.RateLimiter.Supervisor,
|
||||
|
|
@ -113,6 +117,44 @@ defmodule Pleroma.Application do
|
|||
Supervisor.start_link(children, opts)
|
||||
end
|
||||
|
||||
def configure_logger do
|
||||
Config.get([:logger, :backends], [])
|
||||
|> Enum.each(fn backend ->
|
||||
case backend_to_logger(backend) do
|
||||
{:ok, logger} ->
|
||||
add_logger(logger)
|
||||
|
||||
{:error, :console_backend} ->
|
||||
Logger.warning(":console is no longer considered a backend and is enabled by default")
|
||||
end
|
||||
end)
|
||||
end
|
||||
|
||||
defp add_logger(backend) do
|
||||
case LoggerBackends.add(backend) do
|
||||
{:ok, _} ->
|
||||
Logger.debug("Successfully added logger backend: #{inspect(backend)}")
|
||||
|
||||
{:error, :already_present} ->
|
||||
:ok
|
||||
|
||||
{:error, reason} ->
|
||||
Logger.error("Failed to add logger backend #{inspect(backend)}: #{inspect(reason)}")
|
||||
end
|
||||
end
|
||||
|
||||
defp backend_to_logger(:console), do: {:error, :console_backend}
|
||||
|
||||
defp backend_to_logger({:ex_syslogger = backend, name}) do
|
||||
Logger.warning(
|
||||
"Configuration {:#{backend}, :#{name}} is incorrect. Use {ExSyslogger, :#{name}} instead!"
|
||||
)
|
||||
|
||||
{:ok, {ExSyslogger, name}}
|
||||
end
|
||||
|
||||
defp backend_to_logger(backend), do: {:ok, backend}
|
||||
|
||||
def load_custom_modules do
|
||||
dir = Config.get([:modules, :runtime_dir])
|
||||
|
||||
|
|
@ -159,6 +201,16 @@ defmodule Pleroma.Application do
|
|||
]
|
||||
end
|
||||
|
||||
defp search_children do
|
||||
case Config.get([Pleroma.Search, :module]) do
|
||||
Pleroma.Search.ParadeDB ->
|
||||
[Pleroma.Search.ParadeDB.Repo]
|
||||
|
||||
_ ->
|
||||
[]
|
||||
end
|
||||
end
|
||||
|
||||
defp emoji_packs_expiration,
|
||||
do: expiration(default: :timer.seconds(5 * 60), interval: :timer.seconds(60))
|
||||
|
||||
|
|
|
|||
|
|
@ -108,4 +108,6 @@ defmodule Pleroma.Chat do
|
|||
|> cast(%{pinned: false}, [:pinned])
|
||||
|> Repo.update()
|
||||
end
|
||||
|
||||
def enabled?, do: Pleroma.Config.get([__MODULE__, :enabled], true)
|
||||
end
|
||||
|
|
|
|||
|
|
@ -20,6 +20,8 @@ defmodule Pleroma.Config.DeprecationWarnings do
|
|||
"\n* `config :pleroma, :instance, mrf_transparency_exclusions` is now `config :pleroma, :mrf, transparency_exclusions`"}
|
||||
]
|
||||
|
||||
@logger_formatter_config_knobs [:colors, :format, :metadata, :truncate, :utc_log]
|
||||
|
||||
def check_exiftool_filter do
|
||||
filters = Config.get([Pleroma.Upload]) |> Keyword.get(:filters, [])
|
||||
|
||||
|
|
@ -218,7 +220,8 @@ defmodule Pleroma.Config.DeprecationWarnings do
|
|||
check_quarantined_instances_tuples(),
|
||||
check_transparency_exclusions_tuples(),
|
||||
check_simple_policy_tuples(),
|
||||
check_exiftool_filter()
|
||||
check_exiftool_filter(),
|
||||
check_deprecated_logger_config()
|
||||
]
|
||||
|> Enum.reduce(:ok, fn
|
||||
:ok, :ok -> :ok
|
||||
|
|
@ -415,4 +418,72 @@ defmodule Pleroma.Config.DeprecationWarnings do
|
|||
:ok
|
||||
end
|
||||
end
|
||||
|
||||
defp merge_deprecated_logger_config(config) do
|
||||
formatter_config = Enum.filter(config, fn {k, _} -> k in @logger_formatter_config_knobs end)
|
||||
formatter = Logger.default_formatter(formatter_config)
|
||||
log_level = Keyword.get(config, :level, nil)
|
||||
|
||||
Logger.debug("""
|
||||
Reconfiguring console Logger with deprecated configuration syntax.
|
||||
Handler configuration:
|
||||
log_level: #{inspect(log_level)}
|
||||
|
||||
Formatter:
|
||||
#{inspect(formatter)}
|
||||
""")
|
||||
|
||||
if Config.get(:env) != :test do
|
||||
if log_level, do: :logger.update_handler_config(:default, :level, log_level)
|
||||
:logger.set_handler_config(:default, :formatter, formatter)
|
||||
end
|
||||
end
|
||||
|
||||
@spec check_deprecated_logger_config() :: :ok | :error
|
||||
def check_deprecated_logger_config do
|
||||
backends_config = Application.get_env(:logger, :backends)
|
||||
console_config = Application.get_env(:logger, :console)
|
||||
|
||||
# NOTE: No need to merge the old backends config since it still works.
|
||||
# And new configuration will just add new Logger backends.
|
||||
backend =
|
||||
if backends_config do
|
||||
Logger.warning("""
|
||||
!!!DEPRECATION WARNING!!!
|
||||
Your configuration is using deprecated syntax for configuring backends of Elixir's logger.
|
||||
`config :logger, backends: [...]` is deprecated syntax due to changes in Elixir.
|
||||
Please update your configuration at your earliest convenience to use:
|
||||
`config :pleroma, :logger, backends: [...]`
|
||||
|
||||
Note: `:console` is no longer considered a backend and is used by default, you can disable it using:
|
||||
`config :logger, :default_handler: false`
|
||||
""")
|
||||
|
||||
true
|
||||
else
|
||||
false
|
||||
end
|
||||
|
||||
console =
|
||||
if console_config do
|
||||
Logger.warning("""
|
||||
!!!DEPRECATION WARNING!!!
|
||||
Your configuration is using deprecated syntax for configuring logging to console.
|
||||
`config :logger, :console` is deprecated syntax due to changes in Elixir.
|
||||
Please update your configuration at your earliest convenience to use
|
||||
`config :logger, default_handler` and `config :logger, :default_formatter`.
|
||||
|
||||
Note: `:default_handler` is used only for the `level` setting. All other configurations go under
|
||||
`:default_formatter`. For more info visit: https://hexdocs.pm/logger/Logger.html#module-backends-and-backwards-compatibility
|
||||
""")
|
||||
|
||||
merge_deprecated_logger_config(console_config)
|
||||
|
||||
true
|
||||
else
|
||||
false
|
||||
end
|
||||
|
||||
if backend or console, do: :error, else: :ok
|
||||
end
|
||||
end
|
||||
|
|
|
|||
|
|
@ -47,11 +47,12 @@ defmodule Pleroma.Config.TransferTask do
|
|||
|
||||
settings =
|
||||
(Repo.all(ConfigDB) ++ deleted_settings)
|
||||
|> Enum.reject(&ConfigDB.static?/1)
|
||||
|> Enum.map(&merge_with_default/1)
|
||||
|
||||
started_applications = Application.started_applications()
|
||||
|
||||
reject = [nil, :postgrex]
|
||||
reject = [nil, :postgrex, :logger]
|
||||
|
||||
reject =
|
||||
if restart_pleroma? do
|
||||
|
|
|
|||
|
|
@ -23,6 +23,12 @@ defmodule Pleroma.ConfigDB do
|
|||
{:pleroma, :mrf_keyword, :replace}
|
||||
]
|
||||
|
||||
@static_config [
|
||||
{:pleroma, Pleroma.Search},
|
||||
{:pleroma, Pleroma.Search.ParadeDB},
|
||||
{:pleroma, Pleroma.Search.ParadeDB.Repo}
|
||||
]
|
||||
|
||||
schema "config" do
|
||||
field(:key, Pleroma.EctoType.Config.Atom)
|
||||
field(:group, Pleroma.EctoType.Config.Atom)
|
||||
|
|
@ -32,16 +38,32 @@ defmodule Pleroma.ConfigDB do
|
|||
timestamps()
|
||||
end
|
||||
|
||||
@spec get_all_as_keyword() :: keyword()
|
||||
def get_all_as_keyword do
|
||||
@spec get_all_as_keyword(keyword()) :: keyword()
|
||||
def get_all_as_keyword(options \\ []) do
|
||||
ConfigDB
|
||||
|> select([c], {c.group, c.key, c.value})
|
||||
|> Repo.all()
|
||||
|> maybe_reject_static(options)
|
||||
|> Enum.reduce([], fn {group, key, value}, acc ->
|
||||
Keyword.update(acc, group, [{key, value}], &Keyword.merge(&1, [{key, value}]))
|
||||
end)
|
||||
end
|
||||
|
||||
def static?(%{group: group, key: key}), do: static?(group, key)
|
||||
def static?(_config), do: false
|
||||
|
||||
def static?(group, key) do
|
||||
Enum.any?(@static_config, fn {static_group, static_key} ->
|
||||
group in [static_group, inspect(static_group)] and key in [static_key, inspect(static_key)]
|
||||
end)
|
||||
end
|
||||
|
||||
defp maybe_reject_static(configs, exclude_static: true) do
|
||||
Enum.reject(configs, fn {group, key, _value} -> static?(group, key) end)
|
||||
end
|
||||
|
||||
defp maybe_reject_static(configs, _options), do: configs
|
||||
|
||||
@spec get_all_by_group(atom() | String.t()) :: [t()]
|
||||
def get_all_by_group(group) do
|
||||
from(c in ConfigDB, where: c.group == ^group) |> Repo.all()
|
||||
|
|
|
|||
|
|
@ -12,13 +12,27 @@ defmodule Pleroma.Docs.Generator do
|
|||
|
||||
@spec list_behaviour_implementations(behaviour :: module()) :: [module()]
|
||||
def list_behaviour_implementations(behaviour) do
|
||||
:code.all_loaded()
|
||||
|> Enum.filter(fn {module, _} ->
|
||||
# :code.all_loaded() no longer works on Elixir 1.19 and newer since modules are lazily loaded.
|
||||
# We can get a complete list of modules by interating over Application.loaded_applications/0,
|
||||
# but that includes every module which would get loaded below, which is very slow.
|
||||
# Hardcode the Pleroma application instead, which is good enough for now.
|
||||
application_modules =
|
||||
case :application.get_key(:pleroma, :modules) do
|
||||
{:ok, modules} -> modules
|
||||
:undefined -> []
|
||||
end
|
||||
|
||||
# At compile time or in tests some modules will be outside of the Pleroma application.
|
||||
loaded_modules = Enum.map(:code.all_loaded(), fn {module, _} -> module end)
|
||||
|
||||
(application_modules ++ loaded_modules)
|
||||
|> Enum.uniq()
|
||||
|> Enum.filter(fn module ->
|
||||
Code.ensure_loaded(module)
|
||||
|
||||
# This shouldn't be needed as all modules are expected to have module_info/1,
|
||||
# but in test environments some transient modules `:elixir_compiler_XX`
|
||||
# are loaded for some reason (where XX is a random integer).
|
||||
Code.ensure_loaded(module)
|
||||
|
||||
if function_exported?(module, :module_info, 1) do
|
||||
module.module_info(:attributes)
|
||||
|> Keyword.get_values(:behaviour)
|
||||
|
|
@ -26,7 +40,6 @@ defmodule Pleroma.Docs.Generator do
|
|||
|> Enum.member?(behaviour)
|
||||
end
|
||||
end)
|
||||
|> Enum.map(fn {module, _} -> module end)
|
||||
end
|
||||
|
||||
@doc """
|
||||
|
|
|
|||
|
|
@ -6,9 +6,12 @@ defmodule Pleroma.Gun do
|
|||
@callback open(charlist(), pos_integer(), map()) :: {:ok, pid()}
|
||||
@callback info(pid()) :: map()
|
||||
@callback close(pid()) :: :ok
|
||||
@callback cancel(pid(), reference() | [reference()]) :: :ok
|
||||
@callback await_up(pid, pos_integer()) :: {:ok, atom()} | {:error, atom()}
|
||||
@callback connect(pid(), map()) :: reference()
|
||||
@callback await(pid(), reference()) :: {:response, :fin, 200, []}
|
||||
@callback await(pid(), reference()) :: tuple()
|
||||
@callback await_tunnel_up(pid(), reference() | :undefined, timeout()) ::
|
||||
{:ok, atom()} | {:error, term()}
|
||||
@callback set_owner(pid(), pid()) :: :ok
|
||||
|
||||
defp api, do: Pleroma.Config.get([Pleroma.Gun], Pleroma.Gun.API)
|
||||
|
|
@ -19,11 +22,15 @@ defmodule Pleroma.Gun do
|
|||
|
||||
def close(pid), do: api().close(pid)
|
||||
|
||||
def cancel(pid, stream), do: api().cancel(pid, stream)
|
||||
|
||||
def await_up(pid, timeout \\ 5_000), do: api().await_up(pid, timeout)
|
||||
|
||||
def connect(pid, opts), do: api().connect(pid, opts)
|
||||
|
||||
def await(pid, ref), do: api().await(pid, ref)
|
||||
|
||||
def await_tunnel_up(pid, ref, timeout), do: api().await_tunnel_up(pid, ref, timeout)
|
||||
|
||||
def set_owner(pid, owner), do: api().set_owner(pid, owner)
|
||||
end
|
||||
|
|
|
|||
|
|
@ -32,6 +32,9 @@ defmodule Pleroma.Gun.API do
|
|||
@impl Gun
|
||||
defdelegate close(pid), to: :gun
|
||||
|
||||
@impl Gun
|
||||
defdelegate cancel(pid, stream), to: :gun
|
||||
|
||||
@impl Gun
|
||||
defdelegate await_up(pid, timeout \\ 5_000), to: :gun
|
||||
|
||||
|
|
@ -41,6 +44,28 @@ defmodule Pleroma.Gun.API do
|
|||
@impl Gun
|
||||
defdelegate await(pid, ref), to: :gun
|
||||
|
||||
@impl Gun
|
||||
def await_tunnel_up(pid, stream_ref, timeout) do
|
||||
monitor_ref = Process.monitor(pid)
|
||||
|
||||
result =
|
||||
receive do
|
||||
{:gun_tunnel_up, ^pid, ^stream_ref, protocol} ->
|
||||
{:ok, protocol}
|
||||
|
||||
{:gun_down, ^pid, _protocol, reason, _killed_streams} ->
|
||||
{:error, {:down, reason}}
|
||||
|
||||
{:DOWN, ^monitor_ref, :process, ^pid, reason} ->
|
||||
{:error, {:down, reason}}
|
||||
after
|
||||
timeout -> {:error, :timeout}
|
||||
end
|
||||
|
||||
Process.demonitor(monitor_ref, [:flush])
|
||||
result
|
||||
end
|
||||
|
||||
@impl Gun
|
||||
defdelegate set_owner(pid, owner), to: :gun
|
||||
end
|
||||
|
|
|
|||
|
|
@ -42,18 +42,34 @@ defmodule Pleroma.Gun.Conn do
|
|||
Map.put(opts, :tls_opts, tls_opts)
|
||||
end
|
||||
|
||||
defp do_open(%URI{scheme: "http"} = uri, %{proxy: {proxy_host, proxy_port}} = opts) do
|
||||
with open_opts <- opts |> proxy_open_opts() |> Map.put(:protocols, [:http]),
|
||||
{:ok, conn} <- Gun.open(proxy_host, proxy_port, open_opts),
|
||||
{:ok, protocol} <- await_up(conn, opts[:connect_timeout]) do
|
||||
{:ok, conn, protocol, :forward_proxy}
|
||||
else
|
||||
error ->
|
||||
Logger.warning(
|
||||
"Opening forward proxy connection to #{compose_uri_log(uri)} failed with error #{inspect(error)}"
|
||||
)
|
||||
|
||||
error
|
||||
end
|
||||
end
|
||||
|
||||
defp do_open(uri, %{proxy: {proxy_host, proxy_port}} = opts) do
|
||||
connect_opts =
|
||||
uri
|
||||
|> destination_opts()
|
||||
|> add_http2_opts(uri.scheme, Map.get(opts, :tls_opts, []))
|
||||
|> add_proxy_auth(opts)
|
||||
|
||||
with open_opts <- Map.delete(opts, :tls_opts),
|
||||
with open_opts <- proxy_open_opts(opts),
|
||||
{:ok, conn} <- Gun.open(proxy_host, proxy_port, open_opts),
|
||||
{:ok, protocol} <- Gun.await_up(conn, opts[:connect_timeout]),
|
||||
stream <- Gun.connect(conn, connect_opts),
|
||||
{:response, :fin, 200, _} <- Gun.await(conn, stream) do
|
||||
{:ok, conn, protocol}
|
||||
{:ok, _proxy_protocol} <- await_up(conn, opts[:connect_timeout]) do
|
||||
conn
|
||||
|> Gun.connect(connect_opts)
|
||||
|> await_proxy_connect(conn, opts[:connect_timeout])
|
||||
else
|
||||
error ->
|
||||
Logger.warning(
|
||||
|
|
@ -64,30 +80,29 @@ defmodule Pleroma.Gun.Conn do
|
|||
end
|
||||
end
|
||||
|
||||
defp do_open(uri, %{proxy: {proxy_type, proxy_host, proxy_port}} = opts) do
|
||||
version =
|
||||
proxy_type
|
||||
|> to_string()
|
||||
|> String.last()
|
||||
|> case do
|
||||
"4" -> 4
|
||||
_ -> 5
|
||||
defp do_open(_uri, %{proxy: {:socks4, _proxy_host, _proxy_port}}) do
|
||||
{:error, :socks4_unsupported}
|
||||
end
|
||||
|
||||
defp do_open(uri, %{proxy: {proxy_type, proxy_host, proxy_port}} = opts)
|
||||
when proxy_type in [:socks, :socks5] do
|
||||
socks_opts =
|
||||
uri
|
||||
|> destination_opts()
|
||||
|> add_http2_opts(uri.scheme, Map.get(opts, :tls_opts, []))
|
||||
|> Map.put(:version, version)
|
||||
|> Map.put(:version, 5)
|
||||
|> add_socks_proxy_auth(opts)
|
||||
|
||||
opts =
|
||||
open_opts =
|
||||
opts
|
||||
|> proxy_open_opts()
|
||||
|> Map.put(:protocols, [:socks])
|
||||
|> Map.put(:socks_opts, socks_opts)
|
||||
|
||||
with {:ok, conn} <- Gun.open(proxy_host, proxy_port, opts),
|
||||
{:ok, protocol} <- Gun.await_up(conn, opts[:connect_timeout]) do
|
||||
{:ok, conn, protocol}
|
||||
with {:ok, conn} <- Gun.open(proxy_host, proxy_port, open_opts),
|
||||
{:ok, :socks} <- await_up(conn, opts[:connect_timeout]),
|
||||
{:ok, protocol} <- await_tunnel_up(conn, :undefined, opts[:connect_timeout]) do
|
||||
{:ok, conn, protocol, nil}
|
||||
else
|
||||
error ->
|
||||
Logger.warning(
|
||||
|
|
@ -100,10 +115,11 @@ defmodule Pleroma.Gun.Conn do
|
|||
|
||||
defp do_open(%URI{host: host, port: port} = uri, opts) do
|
||||
host = Pleroma.HTTP.AdapterHelper.parse_host(host)
|
||||
opts = Map.put(opts, :transport, transport(uri.scheme))
|
||||
|
||||
with {:ok, conn} <- Gun.open(host, port, opts),
|
||||
{:ok, protocol} <- Gun.await_up(conn, opts[:connect_timeout]) do
|
||||
{:ok, conn, protocol}
|
||||
{:ok, protocol} <- await_up(conn, opts[:connect_timeout]) do
|
||||
{:ok, conn, protocol, nil}
|
||||
else
|
||||
error ->
|
||||
Logger.warning(
|
||||
|
|
@ -120,11 +136,90 @@ defmodule Pleroma.Gun.Conn do
|
|||
end
|
||||
|
||||
defp add_http2_opts(opts, "https", tls_opts) do
|
||||
Map.merge(opts, %{protocols: [:http2], transport: :tls, tls_opts: tls_opts})
|
||||
Map.merge(opts, %{protocols: [:http2, :http], transport: :tls, tls_opts: tls_opts})
|
||||
end
|
||||
|
||||
defp add_http2_opts(opts, _, _), do: opts
|
||||
|
||||
defp add_proxy_auth(connect_opts, %{proxy_auth: {username, password}})
|
||||
when is_binary(username) and is_binary(password) do
|
||||
Map.merge(connect_opts, %{username: username, password: password})
|
||||
end
|
||||
|
||||
defp add_proxy_auth(connect_opts, _), do: connect_opts
|
||||
|
||||
defp add_socks_proxy_auth(socks_opts, %{proxy_auth: {username, password}})
|
||||
when is_binary(username) and is_binary(password) do
|
||||
Map.put(socks_opts, :auth, [{:username_password, username, password}])
|
||||
end
|
||||
|
||||
defp add_socks_proxy_auth(socks_opts, _), do: socks_opts
|
||||
|
||||
defp proxy_open_opts(opts) do
|
||||
proxy_tls_opts = Map.get(opts, :proxy_tls_opts, [])
|
||||
|
||||
opts =
|
||||
opts
|
||||
|> Map.drop([:proxy, :proxy_auth, :proxy_tls_opts, :tls_opts])
|
||||
|> Map.put_new(:transport, :tcp)
|
||||
|
||||
if opts.transport == :tls do
|
||||
Map.put(opts, :tls_opts, proxy_tls_opts)
|
||||
else
|
||||
opts
|
||||
end
|
||||
end
|
||||
|
||||
defp await_up(conn, timeout) do
|
||||
case Gun.await_up(conn, timeout) do
|
||||
{:ok, protocol} ->
|
||||
{:ok, protocol}
|
||||
|
||||
error ->
|
||||
Gun.close(conn)
|
||||
error
|
||||
end
|
||||
end
|
||||
|
||||
defp await_proxy_connect(stream, conn, timeout) do
|
||||
case Gun.await(conn, stream) do
|
||||
{:response, :fin, 200, _headers} ->
|
||||
case await_tunnel_up(conn, stream, timeout) do
|
||||
{:ok, protocol} -> {:ok, conn, protocol, stream}
|
||||
error -> error
|
||||
end
|
||||
|
||||
{:response, _fin, 407, _headers} ->
|
||||
close_with_error(conn, :proxy_auth_failed)
|
||||
|
||||
{:response, _fin, status, _headers} when is_integer(status) ->
|
||||
close_with_error(conn, {:proxy_connect_failed, status})
|
||||
|
||||
error ->
|
||||
Gun.close(conn)
|
||||
error
|
||||
end
|
||||
end
|
||||
|
||||
defp close_with_error(conn, reason) do
|
||||
Gun.close(conn)
|
||||
{:error, reason}
|
||||
end
|
||||
|
||||
defp transport("https"), do: :tls
|
||||
defp transport(_), do: :tcp
|
||||
|
||||
defp await_tunnel_up(conn, stream_ref, timeout) do
|
||||
case Gun.await_tunnel_up(conn, stream_ref, timeout) do
|
||||
{:ok, protocol} ->
|
||||
{:ok, protocol}
|
||||
|
||||
error ->
|
||||
Gun.close(conn)
|
||||
error
|
||||
end
|
||||
end
|
||||
|
||||
def compose_uri_log(%URI{scheme: scheme, host: host, path: path}) do
|
||||
"#{scheme}://#{host}#{path}"
|
||||
end
|
||||
|
|
|
|||
|
|
@ -4,53 +4,132 @@
|
|||
|
||||
defmodule Pleroma.Gun.ConnectionPool do
|
||||
@registry __MODULE__
|
||||
@locks Module.concat(__MODULE__, Locks)
|
||||
|
||||
alias Pleroma.Gun.ConnectionPool.WorkerSupervisor
|
||||
|
||||
@connection_option_keys [
|
||||
:http_opts,
|
||||
:http2_opts,
|
||||
:protocols,
|
||||
:proxy,
|
||||
:proxy_auth,
|
||||
:proxy_tls_opts,
|
||||
:retry,
|
||||
:retry_timeout,
|
||||
:socks_opts,
|
||||
:tcp_opts,
|
||||
:tls_opts,
|
||||
:transport,
|
||||
:ws_opts
|
||||
]
|
||||
|
||||
def children do
|
||||
[
|
||||
{Registry, keys: :unique, name: @registry},
|
||||
{Registry, keys: :duplicate, name: @registry},
|
||||
{Registry, keys: :unique, name: @locks},
|
||||
Pleroma.Gun.ConnectionPool.WorkerSupervisor
|
||||
]
|
||||
end
|
||||
|
||||
@spec get_conn(URI.t(), keyword()) :: {:ok, pid()} | {:error, term()}
|
||||
def get_conn(uri, opts) do
|
||||
key = "#{uri.scheme}:#{uri.host}:#{uri.port}"
|
||||
key = connection_key(uri, opts)
|
||||
|
||||
case Registry.lookup(@registry, key) do
|
||||
# The key has already been registered, but connection is not up yet
|
||||
[{worker_pid, nil}] ->
|
||||
get_gun_pid_from_worker(worker_pid, true)
|
||||
|
||||
[{worker_pid, {gun_pid, _used_by, _crf, _last_reference}}] ->
|
||||
GenServer.call(worker_pid, :add_client)
|
||||
case checkout_existing(Registry.lookup(@registry, key)) do
|
||||
{:ok, gun_pid} ->
|
||||
{:ok, gun_pid}
|
||||
|
||||
[] ->
|
||||
# :gun.set_owner fails in :connected state for whatevever reason,
|
||||
# so we open the connection in the process directly and send it's pid back
|
||||
# We trust gun to handle timeouts by itself
|
||||
:not_available ->
|
||||
provision_or_wait(key, uri, opts)
|
||||
end
|
||||
end
|
||||
|
||||
defp checkout_existing([]), do: :not_available
|
||||
|
||||
defp checkout_existing([{_worker_pid, nil} | workers]), do: checkout_existing(workers)
|
||||
|
||||
defp checkout_existing([{worker_pid, _value} | workers]) do
|
||||
case checkout_worker(worker_pid) do
|
||||
{:ok, gun_pid} -> {:ok, gun_pid}
|
||||
_ -> checkout_existing(workers)
|
||||
end
|
||||
end
|
||||
|
||||
defp provision_or_wait(key, uri, opts) do
|
||||
result = with_route_lock(key, fn -> provision(key, uri, opts) end)
|
||||
|
||||
case result do
|
||||
{:ok, gun_pid} ->
|
||||
{:ok, gun_pid}
|
||||
|
||||
{:new, worker_pid} ->
|
||||
get_gun_pid_from_worker(worker_pid)
|
||||
|
||||
{:wait, worker_pid} ->
|
||||
case checkout_worker(worker_pid, :infinity) do
|
||||
{:ok, gun_pid} -> {:ok, gun_pid}
|
||||
:busy -> get_conn(uri, opts)
|
||||
error -> error
|
||||
end
|
||||
|
||||
error ->
|
||||
error
|
||||
end
|
||||
end
|
||||
|
||||
defp provision(key, uri, opts) do
|
||||
workers = Registry.lookup(@registry, key)
|
||||
|
||||
case checkout_existing(workers) do
|
||||
{:ok, gun_pid} ->
|
||||
{:ok, gun_pid}
|
||||
|
||||
:not_available ->
|
||||
case Enum.find(workers, fn {_worker_pid, value} -> is_nil(value) end) do
|
||||
{worker_pid, nil} -> {:wait, worker_pid}
|
||||
nil -> start_worker(key, uri, opts)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
defp with_route_lock(key, function) do
|
||||
case Registry.register(@locks, key, nil) do
|
||||
{:ok, _} ->
|
||||
try do
|
||||
function.()
|
||||
after
|
||||
Registry.unregister(@locks, key)
|
||||
end
|
||||
|
||||
{:error, {:already_registered, _owner}} ->
|
||||
Process.sleep(1)
|
||||
with_route_lock(key, function)
|
||||
end
|
||||
end
|
||||
|
||||
defp start_worker(key, uri, opts) do
|
||||
# :gun.set_owner fails in :connected state, so the worker opens the connection itself.
|
||||
case WorkerSupervisor.start_worker([key, uri, opts, self()]) do
|
||||
{:ok, worker_pid} ->
|
||||
get_gun_pid_from_worker(worker_pid, false)
|
||||
|
||||
{:error, {:already_started, worker_pid}} ->
|
||||
get_gun_pid_from_worker(worker_pid, true)
|
||||
|
||||
err ->
|
||||
err
|
||||
end
|
||||
{:ok, worker_pid} -> {:new, worker_pid}
|
||||
error -> error
|
||||
end
|
||||
end
|
||||
|
||||
defp get_gun_pid_from_worker(worker_pid, register) do
|
||||
defp checkout_worker(worker_pid, timeout \\ 5_000) do
|
||||
try do
|
||||
GenServer.call(worker_pid, :checkout, timeout)
|
||||
catch
|
||||
:exit, reason -> {:error, {:connection_worker_exit, reason}}
|
||||
end
|
||||
end
|
||||
|
||||
defp get_gun_pid_from_worker(worker_pid) do
|
||||
# GenServer.call will block the process for timeout length if
|
||||
# the server crashes on startup (which will happen if gun fails to connect)
|
||||
# so instead we use cast + monitor
|
||||
|
||||
ref = Process.monitor(worker_pid)
|
||||
if register, do: GenServer.cast(worker_pid, {:add_client, self()})
|
||||
|
||||
receive do
|
||||
{:conn_pid, pid} ->
|
||||
|
|
@ -68,19 +147,115 @@ defmodule Pleroma.Gun.ConnectionPool do
|
|||
|
||||
@spec release_conn(pid()) :: :ok
|
||||
def release_conn(conn_pid) do
|
||||
# :ets.fun2ms(fn {_, {worker_pid, {gun_pid, _, _, _}}} when gun_pid == conn_pid ->
|
||||
# :ets.fun2ms(fn {_, {worker_pid, {gun_pid, _tunnel}}} when gun_pid == conn_pid ->
|
||||
# worker_pid end)
|
||||
query_result =
|
||||
Registry.select(@registry, [
|
||||
{{:_, :"$1", {:"$2", :_, :_, :_}}, [{:==, :"$2", conn_pid}], [:"$1"]}
|
||||
{{:_, :"$1", {:"$2", :_}}, [{:==, :"$2", conn_pid}], [:"$1"]}
|
||||
])
|
||||
|
||||
case query_result do
|
||||
[worker_pid] ->
|
||||
try do
|
||||
GenServer.call(worker_pid, :remove_client)
|
||||
catch
|
||||
:exit, _ -> :ok
|
||||
end
|
||||
|
||||
[] ->
|
||||
:ok
|
||||
end
|
||||
end
|
||||
|
||||
@spec discard_conn(pid()) :: :ok
|
||||
def discard_conn(conn_pid) do
|
||||
case worker_for_conn(conn_pid) do
|
||||
nil ->
|
||||
:ok
|
||||
|
||||
worker_pid ->
|
||||
DynamicSupervisor.terminate_child(WorkerSupervisor, worker_pid)
|
||||
end
|
||||
end
|
||||
|
||||
@spec cancel_stream(pid(), reference() | [reference()]) :: :ok
|
||||
def cancel_stream(conn_pid, stream) do
|
||||
case worker_for_conn(conn_pid) do
|
||||
nil ->
|
||||
Pleroma.Gun.cancel(conn_pid, stream)
|
||||
|
||||
worker_pid ->
|
||||
try do
|
||||
GenServer.call(worker_pid, {:cancel_stream, stream})
|
||||
catch
|
||||
:exit, _ -> :ok
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
@spec register_stream(pid(), reference() | [reference()]) :: :ok
|
||||
def register_stream(conn_pid, stream) do
|
||||
call_worker(conn_pid, {:register_stream, stream})
|
||||
end
|
||||
|
||||
@spec finish_stream(pid(), reference() | [reference()]) :: :ok
|
||||
def finish_stream(conn_pid, stream) do
|
||||
call_worker(conn_pid, {:finish_stream, stream})
|
||||
end
|
||||
|
||||
@spec release_stream(pid(), reference() | [reference()]) :: :ok
|
||||
def release_stream(conn_pid, stream) do
|
||||
call_worker(conn_pid, {:release_stream, stream})
|
||||
end
|
||||
|
||||
@spec tunnel_ref(pid()) :: reference() | :forward_proxy | nil
|
||||
def tunnel_ref(conn_pid) do
|
||||
case Registry.select(@registry, [
|
||||
{{:_, :_, {:"$1", :"$2"}}, [{:==, :"$1", conn_pid}], [:"$2"]}
|
||||
]) do
|
||||
[tunnel] -> tunnel
|
||||
[] -> nil
|
||||
end
|
||||
end
|
||||
|
||||
defp worker_for_conn(conn_pid) do
|
||||
case Registry.select(@registry, [
|
||||
{{:_, :"$1", {:"$2", :_}}, [{:==, :"$2", conn_pid}], [:"$1"]}
|
||||
]) do
|
||||
[worker_pid] -> worker_pid
|
||||
[] -> nil
|
||||
end
|
||||
end
|
||||
|
||||
defp call_worker(conn_pid, message) do
|
||||
case worker_for_conn(conn_pid) do
|
||||
nil ->
|
||||
:ok
|
||||
|
||||
worker_pid ->
|
||||
try do
|
||||
GenServer.call(worker_pid, message)
|
||||
catch
|
||||
:exit, _ -> :ok
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
defp connection_key(uri, opts) do
|
||||
origin = {
|
||||
String.downcase(uri.scheme),
|
||||
String.downcase(uri.host),
|
||||
uri.port
|
||||
}
|
||||
|
||||
profile =
|
||||
opts
|
||||
|> Map.new()
|
||||
|> Map.take(@connection_option_keys)
|
||||
|> :erlang.term_to_binary()
|
||||
|> then(&:crypto.hash(:sha256, &1))
|
||||
|
||||
fingerprint = Base.encode16(profile, case: :lower)
|
||||
Enum.join(Tuple.to_list(origin) ++ [fingerprint], ":")
|
||||
end
|
||||
end
|
||||
|
|
|
|||
|
|
@ -9,11 +9,11 @@ defmodule Pleroma.Gun.ConnectionPool.Reclaimer do
|
|||
|
||||
def start_monitor do
|
||||
pid =
|
||||
case GenServer.start(__MODULE__, [], name: {:via, Registry, {registry(), "reclaimer"}}) do
|
||||
case GenServer.start(__MODULE__, [], name: __MODULE__) do
|
||||
{:ok, pid} ->
|
||||
pid
|
||||
|
||||
{:error, {:already_registered, pid}} ->
|
||||
{:error, {:already_started, pid}} ->
|
||||
pid
|
||||
end
|
||||
|
||||
|
|
@ -41,17 +41,16 @@ defmodule Pleroma.Gun.ConnectionPool.Reclaimer do
|
|||
reclaim_max: reclaim_max
|
||||
})
|
||||
|
||||
# :ets.fun2ms(
|
||||
# fn {_, {worker_pid, {_, used_by, crf, last_reference}}} when used_by == [] ->
|
||||
# {worker_pid, crf, last_reference} end)
|
||||
unused_conns =
|
||||
workers =
|
||||
Registry.select(
|
||||
registry(),
|
||||
[
|
||||
{{:_, :"$1", {:_, :"$2", :"$3", :"$4"}}, [{:==, :"$2", []}], [{{:"$1", :"$3", :"$4"}}]}
|
||||
{{:_, :"$1", {:_, :_}}, [], [:"$1"]}
|
||||
]
|
||||
)
|
||||
|
||||
unused_conns = Enum.flat_map(workers, &reclaim_info/1)
|
||||
|
||||
case unused_conns do
|
||||
[] ->
|
||||
:telemetry.execute(
|
||||
|
|
@ -65,25 +64,43 @@ defmodule Pleroma.Gun.ConnectionPool.Reclaimer do
|
|||
{:stop, :no_unused_conns, nil}
|
||||
|
||||
unused_conns ->
|
||||
reclaimed =
|
||||
reclaimed_count =
|
||||
unused_conns
|
||||
|> Enum.sort(fn {_pid1, crf1, last_reference1}, {_pid2, crf2, last_reference2} ->
|
||||
crf1 <= crf2 and last_reference1 <= last_reference2
|
||||
end)
|
||||
|> Enum.take(reclaim_max)
|
||||
|> Enum.sort_by(fn {_pid, crf, last_reference} -> {crf, last_reference} end)
|
||||
|> Enum.reduce_while(0, fn
|
||||
_worker, count when count == reclaim_max ->
|
||||
{:halt, count}
|
||||
|
||||
reclaimed
|
||||
|> Enum.each(fn {pid, _, _} ->
|
||||
DynamicSupervisor.terminate_child(Pleroma.Gun.ConnectionPool.WorkerSupervisor, pid)
|
||||
{worker_pid, _crf, _last_reference}, count ->
|
||||
if reclaim(worker_pid), do: {:cont, count + 1}, else: {:cont, count}
|
||||
end)
|
||||
|
||||
:telemetry.execute(
|
||||
[:pleroma, :connection_pool, :reclaim, :stop],
|
||||
%{reclaimed_count: Enum.count(reclaimed)},
|
||||
%{reclaimed_count: reclaimed_count},
|
||||
%{max_connections: max_connections}
|
||||
)
|
||||
|
||||
{:stop, :normal, nil}
|
||||
end
|
||||
end
|
||||
|
||||
defp reclaim_info(worker_pid) do
|
||||
try do
|
||||
case GenServer.call(worker_pid, :reclaim_info) do
|
||||
{:idle, crf, last_reference} -> [{worker_pid, crf, last_reference}]
|
||||
:in_use -> []
|
||||
end
|
||||
catch
|
||||
:exit, _ -> []
|
||||
end
|
||||
end
|
||||
|
||||
defp reclaim(worker_pid) do
|
||||
try do
|
||||
GenServer.call(worker_pid, :reclaim) == :reclaimed
|
||||
catch
|
||||
:exit, _ -> false
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
|
|||
|
|
@ -8,25 +8,22 @@ defmodule Pleroma.Gun.ConnectionPool.Worker do
|
|||
|
||||
defp registry, do: Pleroma.Gun.ConnectionPool
|
||||
|
||||
def start_link([key | _] = opts) do
|
||||
GenServer.start_link(__MODULE__, opts, name: {:via, Registry, {registry(), key}})
|
||||
end
|
||||
def start_link(opts), do: GenServer.start_link(__MODULE__, opts)
|
||||
|
||||
@impl true
|
||||
def init([_key, _uri, _opts, _client_pid] = opts) do
|
||||
def init([key, _uri, _opts, _client_pid] = opts) do
|
||||
{:ok, _} = Registry.register(registry(), key, nil)
|
||||
{:ok, nil, {:continue, {:connect, opts}}}
|
||||
end
|
||||
|
||||
@impl true
|
||||
def handle_continue({:connect, [key, uri, opts, client_pid]}, _) do
|
||||
with {:ok, conn_pid, protocol} <- Gun.Conn.open(uri, opts),
|
||||
with {:ok, conn_pid, protocol, tunnel} <- Gun.Conn.open(uri, opts),
|
||||
Process.link(conn_pid) do
|
||||
time = :erlang.monotonic_time(:millisecond)
|
||||
|
||||
{_, _} =
|
||||
Registry.update_value(registry(), key, fn _ ->
|
||||
{conn_pid, [client_pid], 1, time}
|
||||
end)
|
||||
Registry.unregister(registry(), key)
|
||||
{:ok, _} = Registry.register(registry(), key, {conn_pid, tunnel})
|
||||
|
||||
send(client_pid, {:conn_pid, conn_pid})
|
||||
|
||||
|
|
@ -34,8 +31,14 @@ defmodule Pleroma.Gun.ConnectionPool.Worker do
|
|||
%{
|
||||
key: key,
|
||||
timer: nil,
|
||||
client_monitors: %{client_pid => Process.monitor(client_pid)},
|
||||
protocol: protocol
|
||||
clients: %{
|
||||
client_pid => %{count: 1, monitor: Process.monitor(client_pid), streams: MapSet.new()}
|
||||
},
|
||||
conn_pid: conn_pid,
|
||||
protocol: protocol,
|
||||
proxied: Keyword.has_key?(opts, :proxy),
|
||||
crf: 1,
|
||||
last_reference: time
|
||||
}, :hibernate}
|
||||
else
|
||||
err ->
|
||||
|
|
@ -44,106 +47,225 @@ defmodule Pleroma.Gun.ConnectionPool.Worker do
|
|||
end
|
||||
|
||||
@impl true
|
||||
def handle_cast({:add_client, client_pid}, state) do
|
||||
case handle_call(:add_client, {client_pid, nil}, state) do
|
||||
{:reply, conn_pid, state, :hibernate} ->
|
||||
send(client_pid, {:conn_pid, conn_pid})
|
||||
{:noreply, state, :hibernate}
|
||||
end
|
||||
def handle_call(:checkout, _from, %{protocol: protocol, clients: clients} = state)
|
||||
when protocol != :http2 and map_size(clients) > 0 do
|
||||
{:reply, :busy, state, :hibernate}
|
||||
end
|
||||
|
||||
@impl true
|
||||
def handle_cast({:remove_client, client_pid}, state) do
|
||||
case handle_call(:remove_client, {client_pid, nil}, state) do
|
||||
{:reply, _, state, :hibernate} ->
|
||||
{:noreply, state, :hibernate}
|
||||
end
|
||||
end
|
||||
|
||||
@impl true
|
||||
def handle_call(:add_client, {client_pid, _}, %{key: key, protocol: protocol} = state) do
|
||||
def handle_call(:checkout, {client_pid, _}, %{key: key, protocol: protocol} = state) do
|
||||
time = :erlang.monotonic_time(:millisecond)
|
||||
|
||||
{{conn_pid, used_by, _, _}, _} =
|
||||
Registry.update_value(registry(), key, fn {conn_pid, used_by, crf, last_reference} ->
|
||||
{conn_pid, [client_pid | used_by], crf(time - last_reference, crf), time}
|
||||
end)
|
||||
[{_, {conn_pid, _tunnel}}] =
|
||||
Registry.lookup(registry(), key) |> Enum.filter(&(elem(&1, 0) == self()))
|
||||
|
||||
state =
|
||||
state
|
||||
|> cancel_idle_timer()
|
||||
|> add_client(client_pid)
|
||||
|> Map.put(:crf, crf(time - state.last_reference, state.crf))
|
||||
|> Map.put(:last_reference, time)
|
||||
|
||||
:telemetry.execute(
|
||||
[:pleroma, :connection_pool, :client, :add],
|
||||
%{client_pid: client_pid, clients: used_by},
|
||||
%{client_pid: client_pid, clients: Map.keys(state.clients)},
|
||||
%{key: state.key, protocol: protocol}
|
||||
)
|
||||
|
||||
state =
|
||||
if state.timer != nil do
|
||||
Process.cancel_timer(state[:timer])
|
||||
%{state | timer: nil}
|
||||
else
|
||||
state
|
||||
end
|
||||
|
||||
ref = Process.monitor(client_pid)
|
||||
|
||||
state = put_in(state.client_monitors[client_pid], ref)
|
||||
{:reply, conn_pid, state, :hibernate}
|
||||
{:reply, {:ok, conn_pid}, state, :hibernate}
|
||||
end
|
||||
|
||||
@impl true
|
||||
def handle_call(:remove_client, {client_pid, _}, %{key: key} = state) do
|
||||
{{_conn_pid, used_by, _crf, _last_reference}, _} =
|
||||
Registry.update_value(registry(), key, fn {conn_pid, used_by, crf, last_reference} ->
|
||||
{conn_pid, List.delete(used_by, client_pid), crf, last_reference}
|
||||
end)
|
||||
def handle_call(:remove_client, {client_pid, _}, state) do
|
||||
case active_streams(state, client_pid) do
|
||||
[] ->
|
||||
{:reply, :ok, remove_client(state, client_pid, true), :hibernate}
|
||||
|
||||
{ref, state} = pop_in(state.client_monitors[client_pid])
|
||||
streams when state.protocol == :http2 ->
|
||||
Enum.each(streams, &(:ok = Gun.cancel(state.conn_pid, &1)))
|
||||
{:reply, :ok, remove_client(state, client_pid, true), :hibernate}
|
||||
|
||||
Process.demonitor(ref, [:flush])
|
||||
|
||||
timer =
|
||||
if used_by == [] do
|
||||
max_idle = Pleroma.Config.get([:connections_pool, :max_idle_time], 30_000)
|
||||
Process.send_after(self(), :idle_close, max_idle)
|
||||
else
|
||||
nil
|
||||
_streams ->
|
||||
{:stop, :normal, :ok, state}
|
||||
end
|
||||
end
|
||||
|
||||
{:reply, :ok, %{state | timer: timer}, :hibernate}
|
||||
def handle_call({:register_stream, stream}, {client_pid, _}, state) do
|
||||
{:reply, :ok, update_stream(state, client_pid, &MapSet.put(&1, stream)), :hibernate}
|
||||
end
|
||||
|
||||
def handle_call({:finish_stream, stream}, {client_pid, _}, state) do
|
||||
{:reply, :ok, update_stream(state, client_pid, &MapSet.delete(&1, stream)), :hibernate}
|
||||
end
|
||||
|
||||
def handle_call({:release_stream, stream}, {client_pid, _}, state) do
|
||||
state = update_stream(state, client_pid, &MapSet.delete(&1, stream))
|
||||
{:reply, :ok, remove_client(state, client_pid, true), :hibernate}
|
||||
end
|
||||
|
||||
def handle_call({:cancel_stream, stream}, {client_pid, _}, %{protocol: :http2} = state) do
|
||||
:ok = Gun.cancel(state.conn_pid, stream)
|
||||
state = update_stream(state, client_pid, &MapSet.delete(&1, stream))
|
||||
{:reply, :ok, remove_client(state, client_pid, true), :hibernate}
|
||||
end
|
||||
|
||||
def handle_call({:cancel_stream, _stream}, _from, state) do
|
||||
{:stop, :normal, :ok, state}
|
||||
end
|
||||
|
||||
def handle_call(:reclaim_info, _from, %{clients: clients} = state)
|
||||
when map_size(clients) == 0 do
|
||||
{:reply, {:idle, state.crf, state.last_reference}, state, :hibernate}
|
||||
end
|
||||
|
||||
def handle_call(:reclaim_info, _from, state) do
|
||||
{:reply, :in_use, state, :hibernate}
|
||||
end
|
||||
|
||||
def handle_call(:reclaim, _from, %{clients: clients} = state) when map_size(clients) == 0 do
|
||||
{:stop, :normal, :reclaimed, state}
|
||||
end
|
||||
|
||||
def handle_call(:reclaim, _from, state) do
|
||||
{:reply, :in_use, state, :hibernate}
|
||||
end
|
||||
|
||||
@impl true
|
||||
def handle_info(:idle_close, state) do
|
||||
# Gun monitors the owner process, and will close the connection automatically
|
||||
# when it's terminated
|
||||
def handle_info({:idle_close, token}, %{timer: {_timer_ref, token}, clients: clients} = state)
|
||||
when map_size(clients) == 0 do
|
||||
# Gun monitors the owner process, and will close the connection automatically.
|
||||
{:stop, :normal, state}
|
||||
end
|
||||
|
||||
def handle_info({:idle_close, _token}, state), do: {:noreply, state, :hibernate}
|
||||
|
||||
@impl true
|
||||
def handle_info({:gun_up, _pid, _protocol}, state) do
|
||||
{:noreply, state, :hibernate}
|
||||
def handle_info({:gun_up, _pid, protocol}, state) do
|
||||
{:noreply, %{state | protocol: protocol}, :hibernate}
|
||||
end
|
||||
|
||||
# Gracefully shutdown if the connection got closed without any streams left
|
||||
@impl true
|
||||
def handle_info({:gun_error, conn_pid, stream, _reason}, %{conn_pid: conn_pid} = state) do
|
||||
{:noreply, delete_stream(state, stream), :hibernate}
|
||||
end
|
||||
|
||||
@impl true
|
||||
def handle_info({:gun_down, _pid, _protocol, _reason, _streams}, %{proxied: true} = state) do
|
||||
{:stop, :normal, state}
|
||||
end
|
||||
|
||||
# Gracefully shutdown if the connection got closed without any streams left.
|
||||
@impl true
|
||||
def handle_info({:gun_down, _pid, _protocol, _reason, []}, state) do
|
||||
{:stop, :normal, state}
|
||||
end
|
||||
|
||||
# Otherwise, wait for retry
|
||||
# Otherwise, wait for retry.
|
||||
@impl true
|
||||
def handle_info({:gun_down, _pid, _protocol, _reason, _killed_streams}, state) do
|
||||
{:noreply, state, :hibernate}
|
||||
end
|
||||
|
||||
@impl true
|
||||
def handle_info({:DOWN, _ref, :process, pid, reason}, state) do
|
||||
def handle_info({:DOWN, ref, :process, pid, reason}, state) do
|
||||
case state.clients[pid] do
|
||||
%{monitor: ^ref} ->
|
||||
:telemetry.execute(
|
||||
[:pleroma, :connection_pool, :client, :dead],
|
||||
%{client_pid: pid, reason: reason},
|
||||
%{key: state.key}
|
||||
)
|
||||
|
||||
handle_cast({:remove_client, pid}, state)
|
||||
handle_dead_client(state, pid)
|
||||
|
||||
_ ->
|
||||
{:noreply, state, :hibernate}
|
||||
end
|
||||
end
|
||||
|
||||
defp add_client(state, client_pid) do
|
||||
update_in(state.clients, fn clients ->
|
||||
case clients[client_pid] do
|
||||
nil ->
|
||||
Map.put(clients, client_pid, %{
|
||||
count: 1,
|
||||
monitor: Process.monitor(client_pid),
|
||||
streams: MapSet.new()
|
||||
})
|
||||
|
||||
client ->
|
||||
Map.put(clients, client_pid, %{client | count: client.count + 1})
|
||||
end
|
||||
end)
|
||||
end
|
||||
|
||||
defp handle_dead_client(state, client_pid) do
|
||||
streams = active_streams(state, client_pid)
|
||||
|
||||
cond do
|
||||
streams == [] ->
|
||||
{:noreply, remove_client(state, client_pid, false, true), :hibernate}
|
||||
|
||||
state.protocol == :http2 ->
|
||||
Enum.each(streams, &(:ok = Gun.cancel(state.conn_pid, &1)))
|
||||
{:noreply, remove_client(state, client_pid, false, true), :hibernate}
|
||||
|
||||
true ->
|
||||
{:stop, :normal, state}
|
||||
end
|
||||
end
|
||||
|
||||
defp active_streams(state, client_pid) do
|
||||
case state.clients[client_pid] do
|
||||
%{streams: streams} -> MapSet.to_list(streams)
|
||||
nil -> []
|
||||
end
|
||||
end
|
||||
|
||||
defp update_stream(state, client_pid, update) do
|
||||
update_in(state.clients, fn clients ->
|
||||
case clients[client_pid] do
|
||||
nil -> clients
|
||||
client -> Map.put(clients, client_pid, %{client | streams: update.(client.streams)})
|
||||
end
|
||||
end)
|
||||
end
|
||||
|
||||
defp delete_stream(state, stream) do
|
||||
update_in(state.clients, fn clients ->
|
||||
Map.new(clients, fn {pid, client} ->
|
||||
{pid, %{client | streams: MapSet.delete(client.streams, stream)}}
|
||||
end)
|
||||
end)
|
||||
end
|
||||
|
||||
defp remove_client(state, client_pid, demonitor, all \\ false) do
|
||||
case state.clients[client_pid] do
|
||||
nil ->
|
||||
state
|
||||
|
||||
%{count: count} = client when count > 1 and not all ->
|
||||
put_in(state.clients[client_pid], %{client | count: count - 1})
|
||||
|
||||
client ->
|
||||
if demonitor, do: Process.demonitor(client.monitor, [:flush])
|
||||
state = update_in(state.clients, &Map.delete(&1, client_pid))
|
||||
|
||||
if map_size(state.clients) == 0, do: schedule_idle_close(state), else: state
|
||||
end
|
||||
end
|
||||
|
||||
defp cancel_idle_timer(%{timer: nil} = state), do: state
|
||||
|
||||
defp cancel_idle_timer(%{timer: {timer_ref, _token}} = state) do
|
||||
Process.cancel_timer(timer_ref)
|
||||
%{state | timer: nil}
|
||||
end
|
||||
|
||||
defp schedule_idle_close(state) do
|
||||
max_idle = Pleroma.Config.get([:connections_pool, :max_idle_time], 30_000)
|
||||
token = make_ref()
|
||||
timer_ref = Process.send_after(self(), {:idle_close, token}, max_idle)
|
||||
%{state | timer: {timer_ref, token}}
|
||||
end
|
||||
|
||||
# LRFU policy: https://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.55.1478
|
||||
|
|
|
|||
|
|
@ -26,7 +26,14 @@ defmodule Pleroma.Gun.ConnectionPool.WorkerSupervisor do
|
|||
def start_worker(opts, true) do
|
||||
case DynamicSupervisor.start_child(__MODULE__, {Worker, opts}) do
|
||||
{:error, :max_children} ->
|
||||
:telemetry.execute([:pleroma, :connection_pool, :provision_failure], %{opts: opts})
|
||||
[key | _] = opts
|
||||
|
||||
:telemetry.execute(
|
||||
[:pleroma, :connection_pool, :provision_failure],
|
||||
%{},
|
||||
%{key: key}
|
||||
)
|
||||
|
||||
{:error, :pool_full}
|
||||
|
||||
res ->
|
||||
|
|
|
|||
253
lib/pleroma/http/adapter/gun.ex
Normal file
253
lib/pleroma/http/adapter/gun.ex
Normal file
|
|
@ -0,0 +1,253 @@
|
|||
# Pleroma: A lightweight social networking server
|
||||
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
|
||||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
defmodule Pleroma.HTTP.Adapter.Gun do
|
||||
@behaviour Tesla.Adapter
|
||||
|
||||
alias Pleroma.Gun.ConnectionPool
|
||||
alias Tesla.Multipart
|
||||
|
||||
@default_timeout 1_000
|
||||
|
||||
@impl Tesla.Adapter
|
||||
def call(env, opts) do
|
||||
adapter_opts =
|
||||
Tesla.Adapter.opts(
|
||||
[close_conn: true, body_as: :plain, send_body: :at_once, receive: true],
|
||||
env,
|
||||
opts
|
||||
)
|
||||
|> Map.new()
|
||||
|
||||
with {:ok, status, headers, body} <- request(env, adapter_opts) do
|
||||
env = %{env | status: status, headers: format_headers(headers), body: body}
|
||||
{:ok, put_stream_ref(env, body)}
|
||||
end
|
||||
end
|
||||
|
||||
defp put_stream_ref(env, %{stream: stream}) do
|
||||
put_in(env.opts[:adapter][:stream], stream)
|
||||
end
|
||||
|
||||
defp put_stream_ref(env, _body), do: env
|
||||
|
||||
defp request(env, %{conn: conn, tunnel: tunnel} = opts) do
|
||||
uri = URI.parse(Tesla.build_url(env))
|
||||
path = request_path(uri, tunnel)
|
||||
method = Tesla.Adapter.Shared.format_method(env.method)
|
||||
{headers, body, send_body} = prepare_body(env.headers, env.body, opts[:send_body])
|
||||
headers = maybe_add_forward_proxy_headers(headers, uri, opts, tunnel)
|
||||
request_opts = request_opts(opts[:reply_to], tunnel)
|
||||
|
||||
stream =
|
||||
try do
|
||||
open_stream(conn, method, path, headers, body, request_opts, send_body)
|
||||
catch
|
||||
kind, reason ->
|
||||
ConnectionPool.release_conn(conn)
|
||||
:erlang.raise(kind, reason, __STACKTRACE__)
|
||||
end
|
||||
|
||||
:ok = ConnectionPool.register_stream(conn, stream)
|
||||
:ok = send_stream_body(conn, stream, body, send_body)
|
||||
response = read_response(conn, stream, opts)
|
||||
|
||||
case response do
|
||||
{:error, reason} ->
|
||||
ConnectionPool.cancel_stream(conn, stream)
|
||||
{:error, {:gun_stream_error, reason}}
|
||||
|
||||
response ->
|
||||
unless opts[:body_as] in [:chunks, :stream] and stream_body?(response) do
|
||||
:ok = ConnectionPool.finish_stream(conn, stream)
|
||||
end
|
||||
|
||||
if opts[:close_conn] and opts[:body_as] not in [:stream, :chunks] do
|
||||
:gun.close(conn)
|
||||
end
|
||||
|
||||
response
|
||||
end
|
||||
end
|
||||
|
||||
defp stream_body?({:ok, _status, _headers, %{stream: _stream}}), do: true
|
||||
defp stream_body?({:ok, _status, _headers, %Stream{}}), do: true
|
||||
defp stream_body?(_response), do: false
|
||||
|
||||
defp request_opts(reply_to, nil), do: %{reply_to: reply_to || self()}
|
||||
|
||||
defp request_opts(reply_to, :forward_proxy), do: %{reply_to: reply_to || self()}
|
||||
|
||||
defp request_opts(reply_to, tunnel) do
|
||||
%{reply_to: reply_to || self(), tunnel: tunnel}
|
||||
end
|
||||
|
||||
defp request_path(uri, :forward_proxy) do
|
||||
path = Tesla.Adapter.Shared.prepare_path(uri.path, uri.query)
|
||||
"#{uri.scheme}://#{authority(uri)}#{path}"
|
||||
end
|
||||
|
||||
defp request_path(uri, _tunnel) do
|
||||
Tesla.Adapter.Shared.prepare_path(uri.path, uri.query)
|
||||
end
|
||||
|
||||
defp maybe_add_forward_proxy_headers(headers, uri, opts, :forward_proxy) do
|
||||
headers
|
||||
|> put_header_new("host", authority(uri))
|
||||
|> maybe_put_proxy_authorization(opts)
|
||||
end
|
||||
|
||||
defp maybe_add_forward_proxy_headers(headers, _uri, _opts, _tunnel), do: headers
|
||||
|
||||
defp maybe_put_proxy_authorization(headers, %{proxy_auth: {username, password}})
|
||||
when is_binary(username) and is_binary(password) do
|
||||
value = "Basic " <> Base.encode64(username <> ":" <> password)
|
||||
List.keystore(headers, "proxy-authorization", 0, {"proxy-authorization", value})
|
||||
end
|
||||
|
||||
defp maybe_put_proxy_authorization(headers, _opts), do: headers
|
||||
|
||||
defp put_header_new(headers, name, value) do
|
||||
if List.keymember?(headers, name, 0), do: headers, else: [{name, value} | headers]
|
||||
end
|
||||
|
||||
defp authority(%URI{scheme: scheme, host: host, port: port}) do
|
||||
host = if String.contains?(host, ":"), do: "[#{host}]", else: host
|
||||
if port == URI.default_port(scheme), do: host, else: "#{host}:#{port}"
|
||||
end
|
||||
|
||||
defp prepare_body(headers, %Multipart{} = multipart, _send_body) do
|
||||
{format_headers(headers ++ Multipart.headers(multipart)), Multipart.body(multipart), :stream}
|
||||
end
|
||||
|
||||
defp prepare_body(headers, body, _send_body)
|
||||
when is_struct(body, Stream) or is_function(body) do
|
||||
{format_headers(headers), body, :stream}
|
||||
end
|
||||
|
||||
defp prepare_body(headers, body, send_body) do
|
||||
{format_headers(headers), body || "", send_body}
|
||||
end
|
||||
|
||||
defp open_stream(conn, method, path, headers, _body, request_opts, :stream) do
|
||||
:gun.headers(conn, method, path, headers, request_opts)
|
||||
end
|
||||
|
||||
defp open_stream(conn, method, path, headers, body, request_opts, :at_once) do
|
||||
:gun.request(conn, method, path, headers, body, request_opts)
|
||||
end
|
||||
|
||||
defp send_stream_body(conn, stream, body, :stream) do
|
||||
try do
|
||||
for data <- body, do: :ok = :gun.data(conn, stream, :nofin, data)
|
||||
:gun.data(conn, stream, :fin, "")
|
||||
catch
|
||||
kind, reason ->
|
||||
ConnectionPool.cancel_stream(conn, stream)
|
||||
:erlang.raise(kind, reason, __STACKTRACE__)
|
||||
end
|
||||
end
|
||||
|
||||
defp send_stream_body(_conn, _stream, _body, :at_once), do: :ok
|
||||
|
||||
defp read_response(conn, stream, opts) do
|
||||
receive? = opts[:receive]
|
||||
|
||||
receive do
|
||||
{:gun_response, ^conn, ^stream, :fin, status, headers} ->
|
||||
{:ok, status, headers, ""}
|
||||
|
||||
{:gun_response, ^conn, ^stream, :nofin, status, headers} ->
|
||||
format_response(conn, stream, opts, status, headers, opts[:body_as])
|
||||
|
||||
{:gun_up, ^conn, _protocol} when receive? ->
|
||||
read_response(conn, stream, opts)
|
||||
|
||||
{:gun_error, ^conn, reason} ->
|
||||
{:error, reason}
|
||||
|
||||
{:gun_error, ^conn, ^stream, reason} ->
|
||||
{:error, reason}
|
||||
|
||||
{:gun_down, ^conn, _protocol, _reason, _killed_streams} when receive? ->
|
||||
read_response(conn, stream, opts)
|
||||
|
||||
{:DOWN, _ref, :process, ^conn, reason} ->
|
||||
{:error, reason}
|
||||
after
|
||||
opts[:timeout] || @default_timeout -> {:error, :recv_response_timeout}
|
||||
end
|
||||
end
|
||||
|
||||
defp format_response(conn, stream, opts, status, headers, :plain) do
|
||||
case read_body(conn, stream, opts) do
|
||||
{:ok, body} ->
|
||||
{:ok, status, headers, body}
|
||||
|
||||
{:error, error} ->
|
||||
:ok = :gun.flush(stream)
|
||||
{:error, error}
|
||||
end
|
||||
end
|
||||
|
||||
defp format_response(conn, stream, opts, status, headers, :stream) do
|
||||
body =
|
||||
Stream.resource(
|
||||
fn -> :reading end,
|
||||
fn
|
||||
:reading ->
|
||||
case Tesla.Adapter.Gun.read_chunk(conn, stream, opts) do
|
||||
{:nofin, part} -> {[part], :reading}
|
||||
{:fin, part} -> {[part], :done}
|
||||
{:error, reason} -> raise "Gun stream failed: #{inspect(reason)}"
|
||||
end
|
||||
|
||||
:done ->
|
||||
{:halt, :done}
|
||||
end,
|
||||
fn _state ->
|
||||
if opts[:close_conn], do: :gun.close(conn)
|
||||
end
|
||||
)
|
||||
|
||||
{:ok, status, headers, body}
|
||||
end
|
||||
|
||||
defp format_response(conn, stream, opts, status, headers, :chunks) do
|
||||
{:ok, status, headers, %{pid: conn, stream: stream, opts: Map.to_list(opts)}}
|
||||
end
|
||||
|
||||
defp read_body(conn, stream, opts, acc \\ "") do
|
||||
receive do
|
||||
{:gun_data, ^conn, ^stream, :fin, body} ->
|
||||
append_body(acc, body, opts[:max_body])
|
||||
|
||||
{:gun_data, ^conn, ^stream, :nofin, body} ->
|
||||
with {:ok, acc} <- append_body(acc, body, opts[:max_body]) do
|
||||
read_body(conn, stream, opts, acc)
|
||||
end
|
||||
|
||||
{:gun_error, ^conn, ^stream, reason} ->
|
||||
{:error, reason}
|
||||
|
||||
{:DOWN, _ref, :process, ^conn, reason} ->
|
||||
{:error, reason}
|
||||
after
|
||||
opts[:timeout] || @default_timeout -> {:error, :recv_body_timeout}
|
||||
end
|
||||
end
|
||||
|
||||
defp append_body(acc, part, nil), do: {:ok, acc <> part}
|
||||
|
||||
defp append_body(acc, part, limit) do
|
||||
body = acc <> part
|
||||
if byte_size(body) <= limit, do: {:ok, body}, else: {:error, :body_too_large}
|
||||
end
|
||||
|
||||
defp format_headers(headers) do
|
||||
Enum.map(headers, fn {key, value} ->
|
||||
{String.downcase(to_string(key)), to_string(value)}
|
||||
end)
|
||||
end
|
||||
end
|
||||
|
|
@ -48,10 +48,10 @@ defmodule Pleroma.HTTP.AdapterHelper.Gun do
|
|||
Keyword.put(opts, :timeout, recv_timeout)
|
||||
end
|
||||
|
||||
# Gun uses [body_as: :stream]
|
||||
# Keep the pool lease until the caller consumes or cancels the body.
|
||||
defp maybe_stream(opts) do
|
||||
case Keyword.pop(opts, :stream, nil) do
|
||||
{true, opts} -> Keyword.put(opts, :body_as, :stream)
|
||||
{true, opts} -> Keyword.put(opts, :body_as, :chunks)
|
||||
{_, opts} -> opts
|
||||
end
|
||||
end
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ defmodule Pleroma.List do
|
|||
import Ecto.Changeset
|
||||
|
||||
alias Pleroma.Activity
|
||||
alias Pleroma.Emoji
|
||||
alias Pleroma.Repo
|
||||
alias Pleroma.User
|
||||
|
||||
|
|
@ -18,14 +19,53 @@ defmodule Pleroma.List do
|
|||
field(:following, {:array, :string}, default: [])
|
||||
field(:ap_id, :string)
|
||||
field(:exclusive, :boolean, default: false)
|
||||
field(:emoji, :string)
|
||||
|
||||
timestamps()
|
||||
end
|
||||
|
||||
def update_changeset(list, attrs \\ %{}) do
|
||||
list
|
||||
|> cast(attrs, [:title, :exclusive])
|
||||
|> cast(attrs, [:title, :exclusive, :emoji])
|
||||
|> validate_required([:title])
|
||||
|> fix_emoji()
|
||||
|> validate_emoji()
|
||||
end
|
||||
|
||||
defp fix_emoji(changeset) do
|
||||
with {:emoji_field, emoji} when is_binary(emoji) <-
|
||||
{:emoji_field, get_field(changeset, :emoji)},
|
||||
{:fixed_emoji, emoji} <-
|
||||
{:fixed_emoji,
|
||||
emoji
|
||||
|> Emoji.fully_qualify_emoji()
|
||||
|> Emoji.maybe_quote()} do
|
||||
put_change(changeset, :emoji, emoji)
|
||||
else
|
||||
{:emoji_field, _} -> changeset
|
||||
end
|
||||
end
|
||||
|
||||
defp validate_emoji(changeset) do
|
||||
validate_change(changeset, :emoji, fn
|
||||
:emoji, nil ->
|
||||
[]
|
||||
|
||||
:emoji, emoji ->
|
||||
if Emoji.unicode?(emoji) or valid_local_custom_emoji?(emoji) do
|
||||
[]
|
||||
else
|
||||
[emoji: "Invalid emoji"]
|
||||
end
|
||||
end)
|
||||
end
|
||||
|
||||
defp valid_local_custom_emoji?(emoji) do
|
||||
with %{file: _path} <- Emoji.get(emoji) do
|
||||
true
|
||||
else
|
||||
_ -> false
|
||||
end
|
||||
end
|
||||
|
||||
def follow_changeset(list, attrs \\ %{}) do
|
||||
|
|
|
|||
|
|
@ -346,6 +346,16 @@ defmodule Pleroma.ModerationLog do
|
|||
"@#{actor_nickname} approved users: #{users_to_nicknames_string(users)}"
|
||||
end
|
||||
|
||||
def get_log_entry_message(%ModerationLog{
|
||||
data: %{
|
||||
"actor" => %{"nickname" => actor_nickname},
|
||||
"action" => "reject",
|
||||
"subject" => users
|
||||
}
|
||||
}) do
|
||||
"@#{actor_nickname} rejected users: #{users_to_nicknames_string(users)}"
|
||||
end
|
||||
|
||||
def get_log_entry_message(%ModerationLog{
|
||||
data: %{
|
||||
"actor" => %{"nickname" => actor_nickname},
|
||||
|
|
|
|||
|
|
@ -35,6 +35,7 @@ defmodule Pleroma.Notification do
|
|||
# remember to add a migration to add it to the `notifications_type` enum
|
||||
# as well.
|
||||
field(:type, :string)
|
||||
field(:group_key, :string)
|
||||
belongs_to(:user, User, type: FlakeId.Ecto.CompatType)
|
||||
belongs_to(:activity, Activity, type: FlakeId.Ecto.CompatType)
|
||||
|
||||
|
|
@ -49,7 +50,7 @@ defmodule Pleroma.Notification do
|
|||
|> type_from_activity()
|
||||
|
||||
notification
|
||||
|> changeset(%{type: type})
|
||||
|> changeset(%{type: type, group_key: grouped_notification_key(type, activity)})
|
||||
|> Repo.update()
|
||||
end
|
||||
end
|
||||
|
|
@ -62,6 +63,99 @@ defmodule Pleroma.Notification do
|
|||
|> Repo.aggregate(:count, :id)
|
||||
end
|
||||
|
||||
@groupable_notification_types ~w{favourite follow reblog}
|
||||
@group_bucket_seconds 12 * 60 * 60
|
||||
|
||||
def groupable_notification_types, do: @groupable_notification_types
|
||||
|
||||
def group_notifications(notifications, grouped_types \\ nil) do
|
||||
grouped_types = normalize_grouped_types(grouped_types)
|
||||
|
||||
{group_keys, grouped_notifications} =
|
||||
Enum.reduce(notifications, {[], %{}}, fn notification,
|
||||
{group_keys, grouped_notifications} ->
|
||||
group_key = group_key(notification, grouped_types)
|
||||
|
||||
if Map.has_key?(grouped_notifications, group_key) do
|
||||
{group_keys, Map.update!(grouped_notifications, group_key, &[notification | &1])}
|
||||
else
|
||||
{[group_key | group_keys], Map.put(grouped_notifications, group_key, [notification])}
|
||||
end
|
||||
end)
|
||||
|
||||
group_keys
|
||||
|> Enum.reverse()
|
||||
|> Enum.map(fn group_key ->
|
||||
grouped_notifications
|
||||
|> Map.fetch!(group_key)
|
||||
|> Enum.reverse()
|
||||
end)
|
||||
end
|
||||
|
||||
def group_key(notification, grouped_types \\ nil)
|
||||
|
||||
def group_key(%Notification{} = notification, nil) do
|
||||
group_key(notification, @groupable_notification_types)
|
||||
end
|
||||
|
||||
def group_key(%Notification{type: type, group_key: group_key} = notification, grouped_types) do
|
||||
grouped_types = normalize_grouped_types(grouped_types)
|
||||
|
||||
if type in @groupable_notification_types and type in grouped_types and is_binary(group_key) do
|
||||
group_key
|
||||
else
|
||||
ungrouped_group_key(notification)
|
||||
end
|
||||
end
|
||||
|
||||
def normalize_grouped_types(nil), do: @groupable_notification_types
|
||||
def normalize_grouped_types(types) when is_list(types), do: types
|
||||
def normalize_grouped_types(type), do: [type]
|
||||
|
||||
defp ungrouped_group_key(%Notification{id: id}), do: "ungrouped-#{id}"
|
||||
|
||||
defp grouped_notification_key(type, activity) when type in @groupable_notification_types do
|
||||
with target_id when is_binary(target_id) <- group_target_id(type, activity) do
|
||||
# Mastodon uses Redis to reuse a recent bucket for rolling 12h windows. Pleroma keeps keys
|
||||
# deterministic and self-contained; clients must treat group_key as opaque either way.
|
||||
"#{type}-#{target_id}-#{group_time_bucket(activity)}"
|
||||
end
|
||||
end
|
||||
|
||||
defp grouped_notification_key(_type, _activity), do: nil
|
||||
|
||||
defp group_time_bucket(%Activity{inserted_at: inserted_at}) when not is_nil(inserted_at) do
|
||||
inserted_at
|
||||
|> NaiveDateTime.to_erl()
|
||||
|> :calendar.datetime_to_gregorian_seconds()
|
||||
|> div(@group_bucket_seconds)
|
||||
end
|
||||
|
||||
defp group_time_bucket(_),
|
||||
do: group_time_bucket(%Activity{inserted_at: NaiveDateTime.utc_now()})
|
||||
|
||||
defp group_target_id(type, activity) when type in ["favourite", "reblog"] do
|
||||
with object_id when is_binary(object_id) <- object_id_for(activity),
|
||||
%Activity{id: id} <- Activity.get_create_by_object_ap_id(object_id) do
|
||||
to_string(id)
|
||||
else
|
||||
_ -> nil
|
||||
end
|
||||
end
|
||||
|
||||
defp group_target_id("follow", %{data: %{"object" => ap_id}}) do
|
||||
case User.get_cached_by_ap_id(ap_id) do
|
||||
%User{id: id} -> to_string(id)
|
||||
_ -> nil
|
||||
end
|
||||
end
|
||||
|
||||
defp group_target_id(_, _), do: nil
|
||||
|
||||
defp object_id_for(%{data: %{"object" => %{"id" => id}}}) when is_binary(id), do: id
|
||||
defp object_id_for(%{data: %{"object" => id}}) when is_binary(id), do: id
|
||||
defp object_id_for(_), do: nil
|
||||
|
||||
@notification_types ~w{
|
||||
favourite
|
||||
follow
|
||||
|
|
@ -79,7 +173,7 @@ defmodule Pleroma.Notification do
|
|||
|
||||
def changeset(%Notification{} = notification, attrs) do
|
||||
notification
|
||||
|> cast(attrs, [:seen, :type])
|
||||
|> cast(attrs, [:seen, :type, :group_key])
|
||||
|> validate_inclusion(:type, @notification_types)
|
||||
end
|
||||
|
||||
|
|
@ -458,7 +552,8 @@ defmodule Pleroma.Notification do
|
|||
user_id: user.id,
|
||||
activity: activity,
|
||||
seen: mark_as_read?(activity, user),
|
||||
type: type
|
||||
type: type,
|
||||
group_key: grouped_notification_key(type, activity)
|
||||
})
|
||||
|> Marker.multi_set_last_read_id(user, "notifications")
|
||||
|> Repo.transaction()
|
||||
|
|
|
|||
|
|
@ -15,12 +15,28 @@ defmodule Pleroma.Object.Containment do
|
|||
actor
|
||||
end
|
||||
|
||||
def get_actor(%{"actor" => nil, "attributedTo" => actor}) when not is_nil(actor) do
|
||||
get_actor(%{"actor" => actor})
|
||||
end
|
||||
|
||||
def get_actor(%{"actor" => nil}), do: nil
|
||||
|
||||
def get_actor(%{"actor" => []}), do: nil
|
||||
|
||||
def get_actor(%{"actor" => actor}) when is_list(actor) do
|
||||
if is_binary(Enum.at(actor, 0)) do
|
||||
Enum.at(actor, 0)
|
||||
else
|
||||
Enum.find(actor, fn %{"type" => type} -> type in ["Person", "Service", "Application"] end)
|
||||
|> Map.get("id")
|
||||
case actor do
|
||||
[first | _] when is_binary(first) ->
|
||||
first
|
||||
|
||||
_ ->
|
||||
Enum.find_value(actor, fn
|
||||
%{"type" => type, "id" => id}
|
||||
when type in ["Person", "Service", "Application"] and is_binary(id) ->
|
||||
id
|
||||
|
||||
_ ->
|
||||
nil
|
||||
end)
|
||||
end
|
||||
end
|
||||
|
||||
|
|
@ -28,10 +44,6 @@ defmodule Pleroma.Object.Containment do
|
|||
id
|
||||
end
|
||||
|
||||
def get_actor(%{"actor" => nil, "attributedTo" => actor}) when not is_nil(actor) do
|
||||
get_actor(%{"actor" => actor})
|
||||
end
|
||||
|
||||
def get_object(%{"object" => id}) when is_binary(id) do
|
||||
id
|
||||
end
|
||||
|
|
@ -67,9 +79,11 @@ defmodule Pleroma.Object.Containment do
|
|||
|
||||
def contain_origin(id, %{"actor" => _actor} = params) do
|
||||
id_uri = URI.parse(id)
|
||||
actor_uri = URI.parse(get_actor(params))
|
||||
|
||||
compare_uris(actor_uri, id_uri)
|
||||
case get_actor(params) do
|
||||
actor when is_binary(actor) -> compare_uris(URI.parse(actor), id_uri)
|
||||
_ -> :error
|
||||
end
|
||||
end
|
||||
|
||||
def contain_origin(id, %{"attributedTo" => actor} = params),
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@ defmodule Pleroma.ReverseProxy do
|
|||
@max_read_duration :timer.seconds(30)
|
||||
@max_body_length :infinity
|
||||
@failed_request_ttl :timer.seconds(60)
|
||||
@sniff_bytes 8 * 1024
|
||||
@methods ~w(GET HEAD)
|
||||
|
||||
@allowed_mime_types Pleroma.Config.get([Pleroma.Upload, :allowed_mime_types], [])
|
||||
|
|
@ -59,7 +60,12 @@ defmodule Pleroma.ReverseProxy do
|
|||
* `inline_content_types`:
|
||||
* `true` will not alter `content-disposition` (up to the upstream),
|
||||
* `false` will add `content-disposition: attachment` to any request,
|
||||
* a list of whitelisted content types
|
||||
* a list of whitelisted content types for which `content-disposition: inline`
|
||||
is always set (overriding any upstream header) so the media can be embedded
|
||||
in pages; the filename is derived from the content type
|
||||
|
||||
* `sniff_content_type` (default `false`): detects an image MIME type from the first
|
||||
response chunk when the upstream type is missing or `application/octet-stream`.
|
||||
|
||||
* `req_headers`, `resp_headers` additional headers.
|
||||
|
||||
|
|
@ -69,11 +75,15 @@ defmodule Pleroma.ReverseProxy do
|
|||
@default_options [pool: :media]
|
||||
|
||||
@inline_content_types [
|
||||
"image/apng",
|
||||
"image/avif",
|
||||
"image/bmp",
|
||||
"image/gif",
|
||||
"image/jpeg",
|
||||
"image/jpg",
|
||||
"image/png",
|
||||
"image/svg+xml",
|
||||
"image/webp",
|
||||
"audio/mpeg",
|
||||
"audio/mp3",
|
||||
"video/webm",
|
||||
|
|
@ -92,6 +102,7 @@ defmodule Pleroma.ReverseProxy do
|
|||
| {:req_headers, [{String.t(), String.t()}]}
|
||||
| {:resp_headers, [{String.t(), String.t()}]}
|
||||
| {:inline_content_types, boolean() | list(String.t())}
|
||||
| {:sniff_content_type, boolean()}
|
||||
| {:redirect_on_failure, boolean()}
|
||||
|
||||
@spec call(Plug.Conn.t(), String.t(), list(option())) :: Plug.Conn.t()
|
||||
|
|
@ -110,12 +121,8 @@ defmodule Pleroma.ReverseProxy do
|
|||
end
|
||||
|
||||
with {:ok, nil} <- @cachex.get(:failed_proxy_url_cache, url),
|
||||
{:ok, code, headers, client} <- request(method, url, req_headers, client_opts),
|
||||
:ok <-
|
||||
header_length_constraint(
|
||||
headers,
|
||||
Keyword.get(opts, :max_body_length, @max_body_length)
|
||||
) do
|
||||
{:ok, code, headers, client} <-
|
||||
request_with_constraints(method, url, req_headers, client_opts, opts) do
|
||||
response(conn, client, url, code, headers, opts)
|
||||
else
|
||||
{:ok, true} ->
|
||||
|
|
@ -170,7 +177,8 @@ defmodule Pleroma.ReverseProxy do
|
|||
{:ok, code, headers} when code in @valid_resp_codes ->
|
||||
{:ok, code, downcase_headers(headers)}
|
||||
|
||||
{:ok, code, _, _} ->
|
||||
{:ok, code, _, client} ->
|
||||
client().close(client)
|
||||
{:error, {:invalid_http_response, code}}
|
||||
|
||||
{:ok, code, _} ->
|
||||
|
|
@ -181,15 +189,37 @@ defmodule Pleroma.ReverseProxy do
|
|||
end
|
||||
end
|
||||
|
||||
defp request_with_constraints(method, url, headers, client_opts, opts) do
|
||||
case request(method, url, headers, client_opts) do
|
||||
{:ok, _code, headers, client} = response ->
|
||||
case header_length_constraint(
|
||||
headers,
|
||||
Keyword.get(opts, :max_body_length, @max_body_length)
|
||||
) do
|
||||
:ok ->
|
||||
response
|
||||
|
||||
error ->
|
||||
client().close(client)
|
||||
error
|
||||
end
|
||||
|
||||
response ->
|
||||
response
|
||||
end
|
||||
end
|
||||
|
||||
defp response(conn, client, url, status, headers, opts) do
|
||||
Logger.debug("#{__MODULE__} #{status} #{url} #{inspect(headers)}")
|
||||
|
||||
{headers, client, prefetched} = maybe_prefetch_for_content_type(headers, client, opts)
|
||||
|
||||
result =
|
||||
conn
|
||||
|> put_resp_headers(build_resp_headers(headers, opts))
|
||||
|> streaming_compat
|
||||
|> send_chunked(status)
|
||||
|> chunk_reply(client, opts)
|
||||
|> chunk_reply(client, opts, prefetched)
|
||||
|
||||
case result do
|
||||
{:ok, conn} ->
|
||||
|
|
@ -209,18 +239,28 @@ defmodule Pleroma.ReverseProxy do
|
|||
end
|
||||
end
|
||||
|
||||
defp chunk_reply(conn, client, opts) do
|
||||
defp chunk_reply(conn, client, opts, :none) do
|
||||
chunk_reply(conn, client, opts, 0, 0)
|
||||
end
|
||||
|
||||
defp chunk_reply(conn, client, opts, sent_so_far, duration) do
|
||||
with {:ok, duration} <-
|
||||
check_read_duration(
|
||||
duration,
|
||||
Keyword.get(opts, :max_read_duration, @max_read_duration)
|
||||
defp chunk_reply(conn, _client, _opts, :done), do: {:ok, conn}
|
||||
defp chunk_reply(conn, _client, _opts, {:error, error}), do: {:error, error, conn}
|
||||
|
||||
defp chunk_reply(conn, client, opts, {:ok, data, duration}) do
|
||||
with :ok <-
|
||||
body_size_constraint(
|
||||
byte_size(data),
|
||||
Keyword.get(opts, :max_body_length, @max_body_length)
|
||||
),
|
||||
{:ok, data, client} <- client().stream_body(client),
|
||||
{:ok, duration} <- increase_read_duration(duration),
|
||||
{:ok, conn} <- chunk(conn, data) do
|
||||
chunk_reply(conn, client, opts, byte_size(data), duration)
|
||||
else
|
||||
{:error, error} -> {:error, error, conn}
|
||||
end
|
||||
end
|
||||
|
||||
defp chunk_reply(conn, client, opts, sent_so_far, duration) do
|
||||
with {:ok, data, client, duration} <- read_chunk(client, duration, opts),
|
||||
sent_so_far = sent_so_far + byte_size(data),
|
||||
:ok <-
|
||||
body_size_constraint(
|
||||
|
|
@ -235,6 +275,66 @@ defmodule Pleroma.ReverseProxy do
|
|||
end
|
||||
end
|
||||
|
||||
defp read_chunk(client, duration, opts) do
|
||||
with {:ok, timer} <-
|
||||
check_read_duration(
|
||||
duration,
|
||||
Keyword.get(opts, :max_read_duration, @max_read_duration)
|
||||
),
|
||||
{:ok, data, client} <- client().stream_body(client),
|
||||
{:ok, duration} <- increase_read_duration(timer) do
|
||||
{:ok, data, client, duration}
|
||||
end
|
||||
end
|
||||
|
||||
defp maybe_prefetch_for_content_type(headers, client, opts) do
|
||||
if Keyword.get(opts, :sniff_content_type, false) and generic_content_type?(headers) do
|
||||
case read_chunk(client, 0, opts) do
|
||||
{:ok, data, client, duration} ->
|
||||
{maybe_put_image_content_type(headers, data), client, {:ok, data, duration}}
|
||||
|
||||
:done ->
|
||||
{headers, client, :done}
|
||||
|
||||
{:error, error} ->
|
||||
{headers, client, {:error, error}}
|
||||
end
|
||||
else
|
||||
{headers, client, :none}
|
||||
end
|
||||
end
|
||||
|
||||
defp generic_content_type?(headers) do
|
||||
headers
|
||||
|> get_content_type()
|
||||
|> String.trim()
|
||||
|> String.downcase()
|
||||
|> then(&(&1 in ["", "application/octet-stream"]))
|
||||
end
|
||||
|
||||
defp maybe_put_image_content_type(headers, data) do
|
||||
with false <- data == "",
|
||||
{:ok, %{mime_type: "image/" <> _ = content_type}} <-
|
||||
Majic.perform({:bytes, binary_part(data, 0, min(byte_size(data), @sniff_bytes))},
|
||||
pool: Pleroma.MajicPool
|
||||
) do
|
||||
[
|
||||
{"content-type", content_type}
|
||||
| Enum.reject(headers, fn {key, _} -> key == "content-type" end)
|
||||
]
|
||||
else
|
||||
_ -> headers
|
||||
end
|
||||
rescue
|
||||
error ->
|
||||
Logger.debug("#{__MODULE__}: content-type sniffing failed: #{Exception.message(error)}")
|
||||
headers
|
||||
catch
|
||||
kind, reason ->
|
||||
Logger.debug("#{__MODULE__}: content-type sniffing failed: #{kind}: #{inspect(reason)}")
|
||||
headers
|
||||
end
|
||||
|
||||
defp head_response(conn, url, code, headers, opts) do
|
||||
Logger.debug("#{__MODULE__} #{code} #{url} #{inspect(headers)}")
|
||||
|
||||
|
|
@ -281,16 +381,33 @@ defmodule Pleroma.ReverseProxy do
|
|||
|> Enum.filter(fn {k, _} -> k in @keep_req_headers end)
|
||||
|> build_req_range_or_encoding_header(opts)
|
||||
|> build_req_user_agent_header(opts)
|
||||
|> Keyword.merge(Keyword.get(opts, :req_headers, []))
|
||||
|> merge_headers(Keyword.get(opts, :req_headers, []))
|
||||
|> maybe_force_identity_encoding(opts)
|
||||
end
|
||||
|
||||
defp merge_headers(headers, extra_headers) do
|
||||
Enum.reduce(extra_headers, headers, fn {key, value}, headers ->
|
||||
List.keystore(headers, String.downcase(key), 0, {String.downcase(key), value})
|
||||
end)
|
||||
end
|
||||
|
||||
defp maybe_force_identity_encoding(headers, opts) do
|
||||
if Keyword.get(opts, :sniff_content_type, false) do
|
||||
List.keystore(headers, "accept-encoding", 0, {"accept-encoding", "identity"})
|
||||
else
|
||||
headers
|
||||
end
|
||||
end
|
||||
|
||||
# Disable content-encoding if any @range_headers are requested (see #1823).
|
||||
defp build_req_range_or_encoding_header(headers, _opts) do
|
||||
range? = Enum.any?(headers, fn {header, _} -> Enum.member?(@range_headers, header) end)
|
||||
|
||||
if range? && List.keymember?(headers, "accept-encoding", 0) do
|
||||
cond do
|
||||
range? && List.keymember?(headers, "accept-encoding", 0) ->
|
||||
List.keydelete(headers, "accept-encoding", 0)
|
||||
else
|
||||
|
||||
true ->
|
||||
headers
|
||||
end
|
||||
end
|
||||
|
|
@ -384,9 +501,33 @@ defmodule Pleroma.ReverseProxy do
|
|||
|
||||
disposition = "attachment; filename=\"#{name}\""
|
||||
|
||||
List.keystore(headers, "content-disposition", 0, {"content-disposition", disposition})
|
||||
replace_header(headers, "content-disposition", disposition)
|
||||
else
|
||||
if opt == true do
|
||||
headers
|
||||
else
|
||||
name = inline_filename(content_type)
|
||||
|
||||
disposition =
|
||||
if name do
|
||||
"inline; filename=\"#{name}\""
|
||||
else
|
||||
"inline"
|
||||
end
|
||||
|
||||
replace_header(headers, "content-disposition", disposition)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
defp replace_header(headers, key, value) do
|
||||
[{key, value} | Enum.reject(headers, fn {header, _} -> header == key end)]
|
||||
end
|
||||
|
||||
defp inline_filename(content_type) do
|
||||
case MIME.extensions(content_type) do
|
||||
[ext | _] when ext != "" -> "inline.#{ext}"
|
||||
_ -> nil
|
||||
end
|
||||
end
|
||||
|
||||
|
|
@ -406,7 +547,7 @@ defmodule Pleroma.ReverseProxy do
|
|||
|
||||
defp header_length_constraint(_, _), do: :ok
|
||||
|
||||
defp body_size_constraint(size, limit) when is_integer(limit) and limit > 0 and size >= limit do
|
||||
defp body_size_constraint(size, limit) when is_integer(limit) and limit > 0 and size > limit do
|
||||
{:error, :body_too_large}
|
||||
end
|
||||
|
||||
|
|
@ -421,7 +562,9 @@ defmodule Pleroma.ReverseProxy do
|
|||
end
|
||||
end
|
||||
|
||||
defp check_read_duration(_, _), do: {:ok, :no_duration_limit, :no_duration_limit}
|
||||
defp check_read_duration(_, _), do: {:ok, :no_duration_limit}
|
||||
|
||||
defp increase_read_duration(:no_duration_limit), do: {:ok, :no_duration_limit}
|
||||
|
||||
defp increase_read_duration({previous_duration, started})
|
||||
when is_integer(previous_duration) and is_integer(started) do
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ defmodule Pleroma.ReverseProxy.Client.Tesla do
|
|||
@behaviour Pleroma.ReverseProxy.Client
|
||||
|
||||
alias Pleroma.Gun.ConnectionPool
|
||||
alias Pleroma.HTTP.AdapterHelper
|
||||
|
||||
@type headers() :: [{String.t(), String.t()}]
|
||||
@type status() :: pos_integer()
|
||||
|
|
@ -20,6 +21,13 @@ defmodule Pleroma.ReverseProxy.Client.Tesla do
|
|||
def request(method, url, headers, body, opts \\ []) do
|
||||
check_adapter()
|
||||
|
||||
# MediaProxyController inserts MediaProxy proxy configuration into the ReverseProxy call.
|
||||
# This config can be unformatted and in String representation, while this works for Hackney,
|
||||
# it does not work for Tesla and the proxy needs to be reformatted.
|
||||
# Gun AdapterHelper doesn't do it, since it does not overwrite proxy config.
|
||||
proxy = AdapterHelper.format_proxy(opts[:proxy])
|
||||
opts = if proxy, do: Keyword.put(opts, :proxy, proxy), else: opts
|
||||
|
||||
opts = Keyword.put(opts, :body_as, :chunks)
|
||||
|
||||
with {:ok, response} <-
|
||||
|
|
@ -45,8 +53,8 @@ defmodule Pleroma.ReverseProxy.Client.Tesla do
|
|||
@impl true
|
||||
@spec stream_body(map()) ::
|
||||
{:ok, binary(), map()} | {:error, atom() | String.t()} | :done | no_return()
|
||||
def stream_body(%{pid: pid, fin: true}) do
|
||||
ConnectionPool.release_conn(pid)
|
||||
def stream_body(%{pid: pid, stream: stream, fin: true}) do
|
||||
ConnectionPool.release_stream(pid, stream)
|
||||
:done
|
||||
end
|
||||
|
||||
|
|
@ -70,6 +78,10 @@ defmodule Pleroma.ReverseProxy.Client.Tesla do
|
|||
|
||||
@impl true
|
||||
@spec close(map) :: :ok | no_return()
|
||||
def close(%{pid: pid, stream: stream}) do
|
||||
ConnectionPool.cancel_stream(pid, stream)
|
||||
end
|
||||
|
||||
def close(%{pid: pid}) do
|
||||
ConnectionPool.release_conn(pid)
|
||||
end
|
||||
|
|
|
|||
|
|
@ -5,13 +5,12 @@ defmodule Pleroma.Search do
|
|||
alias Pleroma.Workers.SearchIndexingWorker
|
||||
|
||||
@spec add_to_index(Activity.t()) :: {:ok, Oban.Job.t() | :noop} | {:error, Oban.Job.changeset()}
|
||||
def add_to_index(%Activity{id: activity_id, object: %Object{} = object} = activity) do
|
||||
with {_, true} <- {:indexable, indexable?(activity)},
|
||||
{_, "public"} <- {:visibility, Visibility.get_visibility(object)} do
|
||||
def add_to_index(%Activity{id: activity_id, object: %Object{}} = activity) do
|
||||
if indexable?(activity) do
|
||||
SearchIndexingWorker.new(%{"op" => "add_to_index", "activity" => activity_id})
|
||||
|> Oban.insert()
|
||||
else
|
||||
_ -> {:ok, :noop}
|
||||
{:ok, :noop}
|
||||
end
|
||||
end
|
||||
|
||||
|
|
@ -41,40 +40,60 @@ defmodule Pleroma.Search do
|
|||
def object_to_search_data(%Object{} = object) do
|
||||
data = object.data
|
||||
|
||||
content_str =
|
||||
case data["content"] do
|
||||
[nil | rest] -> to_string(rest)
|
||||
str -> str
|
||||
end
|
||||
|
||||
content =
|
||||
with {:ok, scrubbed} <-
|
||||
FastSanitize.Sanitizer.scrub(content_str, Pleroma.HTML.Scrubber.SearchIndexing),
|
||||
trimmed <- String.trim(scrubbed) do
|
||||
trimmed
|
||||
end
|
||||
|
||||
# Make sure we have a non-empty string
|
||||
if content != "" do
|
||||
{:ok, published, _} = DateTime.from_iso8601(data["published"])
|
||||
data
|
||||
|> search_texts()
|
||||
|> Enum.map(&sanitize_search_text/1)
|
||||
|> Enum.reject(&(&1 in ["", "."]))
|
||||
|> Enum.join(" ")
|
||||
|
||||
with true <- content != "",
|
||||
published when is_binary(published) <- data["published"],
|
||||
{:ok, published, _} <- DateTime.from_iso8601(published) do
|
||||
%{
|
||||
id: object.id,
|
||||
content: content,
|
||||
ap: data["id"],
|
||||
published: published |> DateTime.to_unix()
|
||||
}
|
||||
else
|
||||
_ -> nil
|
||||
end
|
||||
end
|
||||
|
||||
defp indexable?(%Activity{
|
||||
def indexable?(%Activity{
|
||||
data: %{"type" => "Create"},
|
||||
object: %Object{
|
||||
data: %{"content" => content, "published" => published, "type" => "Note"}
|
||||
}
|
||||
object: %Object{data: %{"published" => published, "type" => "Note"}} = object
|
||||
})
|
||||
when not is_nil(content) and content not in ["", "."] and not is_nil(published),
|
||||
do: true
|
||||
|
||||
defp indexable?(_), do: false
|
||||
when not is_nil(published) do
|
||||
Visibility.get_visibility(object) in ["public", "unlisted"] and
|
||||
not is_nil(object_to_search_data(object))
|
||||
end
|
||||
|
||||
def indexable?(_), do: false
|
||||
|
||||
defp search_texts(data) do
|
||||
[data["content"], data["summary"] | attachment_names(data["attachment"])]
|
||||
end
|
||||
|
||||
defp attachment_names(attachments) when is_list(attachments) do
|
||||
Enum.map(attachments, fn
|
||||
%{"summary" => summary} when is_binary(summary) -> summary
|
||||
%{"name" => name} -> name
|
||||
_ -> nil
|
||||
end)
|
||||
end
|
||||
|
||||
defp attachment_names(_), do: []
|
||||
|
||||
defp sanitize_search_text([nil | rest]), do: sanitize_search_text(to_string(rest))
|
||||
|
||||
defp sanitize_search_text(text) when is_binary(text) do
|
||||
case FastSanitize.Sanitizer.scrub(text, Pleroma.HTML.Scrubber.SearchIndexing) do
|
||||
{:ok, scrubbed} -> String.trim(scrubbed)
|
||||
_ -> ""
|
||||
end
|
||||
end
|
||||
|
||||
defp sanitize_search_text(_), do: ""
|
||||
end
|
||||
|
|
|
|||
137
lib/pleroma/search/backfill.ex
Normal file
137
lib/pleroma/search/backfill.ex
Normal file
|
|
@ -0,0 +1,137 @@
|
|||
# Pleroma: A lightweight social networking server
|
||||
# Copyright © 2017-2026 Pleroma Authors <https://pleroma.social/>
|
||||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
defmodule Pleroma.Search.Backfill do
|
||||
import Ecto.Query
|
||||
|
||||
alias Pleroma.Activity
|
||||
alias Pleroma.Workers.SearchIndexingWorker
|
||||
|
||||
@default_chunk_size 100
|
||||
@default_step_size 1_000
|
||||
@default_limit 100_000
|
||||
|
||||
@doc """
|
||||
Enqueues existing Create activities for search indexing.
|
||||
|
||||
The returned `next_cursor` can be passed as `before` to resume a bounded run.
|
||||
Pass `limit: :infinity` to continue until the oldest activity is reached.
|
||||
"""
|
||||
def run(options \\ []) do
|
||||
options =
|
||||
Keyword.validate!(options,
|
||||
chunk: @default_chunk_size,
|
||||
step: @default_step_size,
|
||||
limit: @default_limit,
|
||||
before: nil,
|
||||
on_page: nil
|
||||
)
|
||||
|
||||
chunk_size = positive_option!(options, :chunk, @default_chunk_size)
|
||||
step_size = positive_option!(options, :step, @default_step_size)
|
||||
limit = limit_option!(options)
|
||||
before = cursor_option!(options[:before])
|
||||
on_page = callback_option!(options[:on_page])
|
||||
|
||||
do_run(before, limit, step_size, chunk_size, on_page, 0)
|
||||
end
|
||||
|
||||
defp do_run(before, remaining, step_size, chunk_size, on_page, enqueued) do
|
||||
page_size = page_size(remaining, step_size)
|
||||
ids = fetch_ids(before, page_size)
|
||||
|
||||
case ids do
|
||||
[] ->
|
||||
result(enqueued, before, true)
|
||||
|
||||
ids ->
|
||||
enqueue(ids, chunk_size)
|
||||
|
||||
page_count = length(ids)
|
||||
next_cursor = List.last(ids)
|
||||
total = enqueued + page_count
|
||||
remaining = decrement(remaining, page_count)
|
||||
|
||||
on_page.(%{count: page_count, enqueued: total, next_cursor: next_cursor})
|
||||
|
||||
cond do
|
||||
remaining == 0 -> result(total, next_cursor, false)
|
||||
page_count < page_size -> result(total, next_cursor, true)
|
||||
true -> do_run(next_cursor, remaining, step_size, chunk_size, on_page, total)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
defp fetch_ids(before, limit) do
|
||||
Activity
|
||||
|> where([a], fragment("?->>'type' = 'Create'", a.data))
|
||||
|> maybe_before(before)
|
||||
|> order_by([a], desc: a.id)
|
||||
|> limit(^limit)
|
||||
|> select([a], a.id)
|
||||
|> Pleroma.Repo.all()
|
||||
end
|
||||
|
||||
defp maybe_before(query, nil), do: query
|
||||
defp maybe_before(query, before), do: where(query, [a], a.id < ^before)
|
||||
|
||||
defp enqueue(ids, chunk_size) do
|
||||
ids
|
||||
|> Enum.chunk_every(chunk_size)
|
||||
|> Enum.each(fn chunk ->
|
||||
chunk
|
||||
|> Enum.map(&SearchIndexingWorker.new(%{"op" => "add_to_index", "activity" => &1}))
|
||||
|> Oban.insert_all()
|
||||
end)
|
||||
end
|
||||
|
||||
defp page_size(:infinity, step_size), do: step_size
|
||||
defp page_size(remaining, step_size), do: min(remaining, step_size)
|
||||
|
||||
defp decrement(:infinity, _count), do: :infinity
|
||||
defp decrement(remaining, count), do: remaining - count
|
||||
|
||||
defp result(enqueued, next_cursor, exhausted) do
|
||||
%{enqueued: enqueued, next_cursor: next_cursor, exhausted: exhausted}
|
||||
end
|
||||
|
||||
defp positive_option!(options, key, default) do
|
||||
case Keyword.get(options, key, default) do
|
||||
value when is_integer(value) and value > 0 -> value
|
||||
value -> raise ArgumentError, "#{key} must be a positive integer, got: #{inspect(value)}"
|
||||
end
|
||||
end
|
||||
|
||||
defp limit_option!(options) do
|
||||
case Keyword.fetch!(options, :limit) do
|
||||
:infinity -> :infinity
|
||||
value when is_integer(value) and value > 0 -> value
|
||||
value -> raise ArgumentError, "limit must be a positive integer, got: #{inspect(value)}"
|
||||
end
|
||||
end
|
||||
|
||||
defp cursor_option!(nil), do: nil
|
||||
|
||||
defp cursor_option!(cursor) do
|
||||
valid? =
|
||||
try do
|
||||
match?({:ok, <<_::binary-size(16)>>}, FlakeId.Ecto.CompatType.dump(cursor))
|
||||
rescue
|
||||
_ -> false
|
||||
end
|
||||
|
||||
if valid? do
|
||||
cursor
|
||||
else
|
||||
raise ArgumentError, "before must be a valid activity id, got: #{inspect(cursor)}"
|
||||
end
|
||||
end
|
||||
|
||||
defp callback_option!(nil), do: fn _ -> :ok end
|
||||
defp callback_option!(callback) when is_function(callback, 1), do: callback
|
||||
|
||||
defp callback_option!(callback) do
|
||||
raise ArgumentError, "on_page must be a function of arity 1, got: #{inspect(callback)}"
|
||||
end
|
||||
end
|
||||
287
lib/pleroma/search/parade_db.ex
Normal file
287
lib/pleroma/search/parade_db.ex
Normal file
|
|
@ -0,0 +1,287 @@
|
|||
# Pleroma: A lightweight social networking server
|
||||
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
|
||||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
defmodule Pleroma.Search.ParadeDB do
|
||||
@behaviour Pleroma.Search.SearchBackend
|
||||
|
||||
require Logger
|
||||
require Pleroma.Constants
|
||||
|
||||
import Ecto.Query
|
||||
|
||||
import Pleroma.Search.DatabaseSearch,
|
||||
only: [
|
||||
maybe_fetch: 3,
|
||||
maybe_restrict_author: 2,
|
||||
maybe_restrict_blocked: 2,
|
||||
maybe_restrict_local: 2
|
||||
]
|
||||
|
||||
alias Pleroma.Activity
|
||||
alias Pleroma.Config.Getting, as: Config
|
||||
alias Pleroma.Search
|
||||
|
||||
@max_limit 40
|
||||
@default_table "pleroma_search_documents"
|
||||
|
||||
defp client_impl do
|
||||
Config.get([Pleroma.Search.ParadeDB, :client_impl]) ||
|
||||
Pleroma.Search.ParadeDB.Client.Postgres
|
||||
end
|
||||
|
||||
defp table do
|
||||
Config.get([Pleroma.Search.ParadeDB, :table], @default_table)
|
||||
end
|
||||
|
||||
defp fuzzy_distance do
|
||||
case Config.get([Pleroma.Search.ParadeDB, :fuzzy_distance], 0) do
|
||||
value when is_integer(value) and value in 0..2 ->
|
||||
value
|
||||
|
||||
value when is_binary(value) ->
|
||||
case Integer.parse(value) do
|
||||
{parsed, ""} when parsed in 0..2 -> parsed
|
||||
_ -> 0
|
||||
end
|
||||
|
||||
_ ->
|
||||
0
|
||||
end
|
||||
end
|
||||
|
||||
@impl true
|
||||
def create_index do
|
||||
with {:ok, _} <- client_impl().query("CREATE EXTENSION IF NOT EXISTS pg_search", []),
|
||||
{:ok, _} <- client_impl().query(create_table_sql(table()), []),
|
||||
{:ok, _} <- client_impl().query(create_object_id_index_sql(table()), []),
|
||||
{:ok, _} <- client_impl().query(create_bm25_index_sql(table()), []) do
|
||||
:ok
|
||||
else
|
||||
{:error, error} ->
|
||||
Logger.error("ParadeDB create_index failed: #{inspect(error)}")
|
||||
{:error, error}
|
||||
end
|
||||
end
|
||||
|
||||
@impl true
|
||||
def drop_index do
|
||||
case client_impl().query("DROP TABLE IF EXISTS #{table()}", []) do
|
||||
{:ok, _} ->
|
||||
:ok
|
||||
|
||||
{:error, error} ->
|
||||
Logger.error("ParadeDB drop_index failed: #{inspect(error)}")
|
||||
{:error, error}
|
||||
end
|
||||
end
|
||||
|
||||
@impl true
|
||||
def healthcheck_endpoints, do: nil
|
||||
|
||||
@impl true
|
||||
def add_to_index(activity) do
|
||||
with true <- Search.indexable?(activity),
|
||||
maybe_search_data when not is_nil(maybe_search_data) <-
|
||||
Search.object_to_search_data(activity.object) do
|
||||
dumped_activity_id = dump_activity_id(activity.id)
|
||||
actor_ap_id = activity.data["actor"]
|
||||
published_at = DateTime.from_unix!(maybe_search_data.published)
|
||||
|
||||
if is_nil(dumped_activity_id) do
|
||||
Logger.error("ParadeDB add_to_index failed: invalid activity id #{inspect(activity.id)}")
|
||||
:ok
|
||||
else
|
||||
sql =
|
||||
"""
|
||||
INSERT INTO #{table()} (id, object_id, object_ap_id, actor_ap_id, content, published_at)
|
||||
VALUES ($1, $2, $3, $4, $5, $6)
|
||||
ON CONFLICT (id) DO UPDATE SET
|
||||
object_id = EXCLUDED.object_id,
|
||||
object_ap_id = EXCLUDED.object_ap_id,
|
||||
actor_ap_id = EXCLUDED.actor_ap_id,
|
||||
content = EXCLUDED.content,
|
||||
published_at = EXCLUDED.published_at
|
||||
"""
|
||||
|
||||
params = [
|
||||
dumped_activity_id,
|
||||
maybe_search_data.id,
|
||||
maybe_search_data.ap,
|
||||
actor_ap_id,
|
||||
maybe_search_data.content,
|
||||
published_at
|
||||
]
|
||||
|
||||
case client_impl().query(sql, params) do
|
||||
{:ok, _} ->
|
||||
:ok
|
||||
|
||||
{:error, error} ->
|
||||
Logger.error("ParadeDB add_to_index failed: #{inspect(error)}")
|
||||
{:error, error}
|
||||
end
|
||||
end
|
||||
else
|
||||
_ -> :ok
|
||||
end
|
||||
end
|
||||
|
||||
@impl true
|
||||
def remove_from_index(object) do
|
||||
sql = "DELETE FROM #{table()} WHERE object_id = $1"
|
||||
|
||||
case client_impl().query(sql, [object.id]) do
|
||||
{:ok, _} ->
|
||||
:ok
|
||||
|
||||
{:error, error} ->
|
||||
Logger.error("ParadeDB remove_from_index failed: #{inspect(error)}")
|
||||
{:error, error}
|
||||
end
|
||||
end
|
||||
|
||||
@impl true
|
||||
def search(user, query, options \\ []) do
|
||||
limit = options |> Keyword.get(:limit, @max_limit) |> min(@max_limit)
|
||||
offset = Keyword.get(options, :offset, 0)
|
||||
author = Keyword.get(options, :author)
|
||||
|
||||
case search_ids(query, limit, offset, author) do
|
||||
{:ok, []} ->
|
||||
maybe_fetch([], user, query)
|
||||
|
||||
{:ok, ids} ->
|
||||
ids
|
||||
|> fetch_activities(user, query, author)
|
||||
|
||||
{:error, error} ->
|
||||
Logger.error("ParadeDB search failed: #{inspect(error)}")
|
||||
maybe_fetch([], user, query)
|
||||
end
|
||||
end
|
||||
|
||||
defp fetch_activities(ids, user, query, author) do
|
||||
dumped_ids =
|
||||
ids
|
||||
|> Enum.map(&dump_activity_id/1)
|
||||
|> Enum.reject(&is_nil/1)
|
||||
|
||||
from(a in Activity, where: a.id in ^dumped_ids)
|
||||
|> Activity.with_preloaded_object()
|
||||
|> Activity.restrict_deactivated_users()
|
||||
|> maybe_restrict_public(user)
|
||||
|> maybe_restrict_local(user)
|
||||
|> maybe_restrict_author(author)
|
||||
|> maybe_restrict_blocked(user)
|
||||
|> order_by(
|
||||
[a],
|
||||
fragment(
|
||||
"array_position(?, ?)",
|
||||
^dumped_ids,
|
||||
a.id
|
||||
)
|
||||
)
|
||||
|> Pleroma.Repo.all()
|
||||
|> maybe_fetch(user, query)
|
||||
end
|
||||
|
||||
defp maybe_restrict_public(query, %Pleroma.User{}) do
|
||||
intended_recipients = [
|
||||
Pleroma.Constants.as_public(),
|
||||
Pleroma.Web.ActivityPub.Utils.as_local_public()
|
||||
]
|
||||
|
||||
from(a in query,
|
||||
where: fragment("?->>'type' = 'Create'", a.data),
|
||||
where: fragment("? && ?", ^intended_recipients, a.recipients)
|
||||
)
|
||||
end
|
||||
|
||||
defp maybe_restrict_public(query, _user) do
|
||||
from(a in query,
|
||||
where: fragment("?->>'type' = 'Create'", a.data),
|
||||
where: ^Pleroma.Constants.as_public() in a.recipients
|
||||
)
|
||||
end
|
||||
|
||||
defp search_ids(query, limit, offset, author) do
|
||||
base_sql =
|
||||
"SELECT id FROM #{table()} WHERE content &&& #{search_query_param_sql(fuzzy_distance())}"
|
||||
|
||||
params = [query]
|
||||
|
||||
{base_sql, params} =
|
||||
case author do
|
||||
%Pleroma.User{ap_id: ap_id} when is_binary(ap_id) ->
|
||||
{base_sql <> " AND actor_ap_id = $2", params ++ [ap_id]}
|
||||
|
||||
_ ->
|
||||
{base_sql, params}
|
||||
end
|
||||
|
||||
limit_arg = length(params) + 1
|
||||
offset_arg = length(params) + 2
|
||||
|
||||
sql =
|
||||
base_sql <>
|
||||
" ORDER BY published_at DESC, id DESC LIMIT $#{limit_arg} OFFSET $#{offset_arg}"
|
||||
|
||||
params = params ++ [limit, offset]
|
||||
|
||||
with {:ok, %{rows: rows}} <- client_impl().query(sql, params) do
|
||||
{:ok, Enum.map(rows, fn [id] -> id end)}
|
||||
end
|
||||
end
|
||||
|
||||
defp search_query_param_sql(0), do: "$1"
|
||||
defp search_query_param_sql(distance), do: "($1::text)::pdb.fuzzy(#{distance})"
|
||||
|
||||
defp create_table_sql(table) do
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS #{table} (
|
||||
id uuid PRIMARY KEY,
|
||||
object_id bigint NOT NULL,
|
||||
object_ap_id text NOT NULL,
|
||||
actor_ap_id text,
|
||||
content text NOT NULL,
|
||||
published_at timestamptz
|
||||
)
|
||||
"""
|
||||
end
|
||||
|
||||
defp create_object_id_index_sql(table) do
|
||||
"CREATE INDEX IF NOT EXISTS #{table}_object_id_idx ON #{table} (object_id)"
|
||||
end
|
||||
|
||||
defp create_bm25_index_sql(table) do
|
||||
"""
|
||||
CREATE INDEX IF NOT EXISTS #{table}_bm25_idx ON #{table}
|
||||
USING bm25 (id, content, (actor_ap_id::pdb.literal), published_at)
|
||||
WITH (key_field = 'id')
|
||||
"""
|
||||
end
|
||||
|
||||
defp dump_activity_id(<<_::binary-size(16)>> = uuid), do: uuid
|
||||
|
||||
defp dump_activity_id(id) when is_binary(id) do
|
||||
case Ecto.UUID.dump(id) do
|
||||
{:ok, <<_::binary-size(16)>> = uuid} ->
|
||||
uuid
|
||||
|
||||
:error ->
|
||||
dump_flake_activity_id(id)
|
||||
end
|
||||
end
|
||||
|
||||
defp dump_activity_id(_), do: nil
|
||||
|
||||
defp dump_flake_activity_id(id) do
|
||||
case FlakeId.Ecto.CompatType.dump(id) do
|
||||
{:ok, <<_::binary-size(16)>> = uuid} -> uuid
|
||||
_ -> nil
|
||||
end
|
||||
rescue
|
||||
_ -> nil
|
||||
end
|
||||
end
|
||||
24
lib/pleroma/search/parade_db/client.ex
Normal file
24
lib/pleroma/search/parade_db/client.ex
Normal file
|
|
@ -0,0 +1,24 @@
|
|||
# Pleroma: A lightweight social networking server
|
||||
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
|
||||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
defmodule Pleroma.Search.ParadeDB.Client do
|
||||
@moduledoc false
|
||||
|
||||
@type sql :: String.t()
|
||||
@type params :: list()
|
||||
@type result :: %{optional(:rows) => list(list())}
|
||||
|
||||
@callback query(sql(), params()) :: {:ok, result()} | {:error, any()}
|
||||
end
|
||||
|
||||
defmodule Pleroma.Search.ParadeDB.Client.Postgres do
|
||||
@moduledoc false
|
||||
|
||||
@behaviour Pleroma.Search.ParadeDB.Client
|
||||
|
||||
@impl true
|
||||
def query(sql, params \\ []) do
|
||||
Ecto.Adapters.SQL.query(Pleroma.Search.ParadeDB.Repo, sql, params)
|
||||
end
|
||||
end
|
||||
24
lib/pleroma/search/parade_db/repo.ex
Normal file
24
lib/pleroma/search/parade_db/repo.ex
Normal file
|
|
@ -0,0 +1,24 @@
|
|||
# Pleroma: A lightweight social networking server
|
||||
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
|
||||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
defmodule Pleroma.Search.ParadeDB.Repo do
|
||||
use Ecto.Repo,
|
||||
otp_app: :pleroma,
|
||||
adapter: Ecto.Adapters.Postgres,
|
||||
migration_timestamps: [type: :naive_datetime_usec]
|
||||
|
||||
@doc """
|
||||
Loads the ParadeDB database URL from `PARADEDB_DATABASE_URL`.
|
||||
|
||||
If the env var is not set, falls back to `config :pleroma, Pleroma.Search.ParadeDB, url: ...`.
|
||||
"""
|
||||
def init(_, opts) do
|
||||
config = Application.get_env(:pleroma, Pleroma.Search.ParadeDB, [])
|
||||
|
||||
url = System.get_env("PARADEDB_DATABASE_URL") || config[:url]
|
||||
url = if is_nil(url), do: nil, else: to_string(url)
|
||||
|
||||
{:ok, Keyword.put(opts, :url, url)}
|
||||
end
|
||||
end
|
||||
|
|
@ -55,8 +55,8 @@ defmodule Pleroma.Telemetry.Logger do
|
|||
|
||||
def handle_event(
|
||||
[:pleroma, :connection_pool, :provision_failure],
|
||||
%{opts: [key | _]},
|
||||
_,
|
||||
%{key: key},
|
||||
_
|
||||
) do
|
||||
Logger.debug(fn ->
|
||||
|
|
|
|||
|
|
@ -12,25 +12,38 @@ defmodule Pleroma.Tesla.Middleware.ConnectionPool do
|
|||
alias Pleroma.Gun.ConnectionPool
|
||||
|
||||
@impl Tesla.Middleware
|
||||
def call(%Tesla.Env{url: url, opts: opts} = env, next, _) do
|
||||
def call(%Tesla.Env{opts: opts} = env, next, middleware_opts) do
|
||||
if opts[:adapter][:body_as] == :stream do
|
||||
{:error, :pooled_streaming_requires_chunks}
|
||||
else
|
||||
do_call(env, next, middleware_opts)
|
||||
end
|
||||
end
|
||||
|
||||
defp do_call(%Tesla.Env{url: url, opts: opts} = env, next, _) do
|
||||
uri = URI.parse(url)
|
||||
|
||||
# Avoid leaking connections when the middleware is called twice
|
||||
# with body_as: :chunks. We assume only the middleware can set
|
||||
# opts[:adapter][:conn]
|
||||
if opts[:adapter][:conn] do
|
||||
ConnectionPool.release_conn(opts[:adapter][:conn])
|
||||
end
|
||||
cleanup_previous_request(env)
|
||||
|
||||
case ConnectionPool.get_conn(uri, opts[:adapter]) do
|
||||
{:ok, conn_pid} ->
|
||||
adapter_opts = Keyword.merge(opts[:adapter], conn: conn_pid, close_conn: false)
|
||||
tunnel = ConnectionPool.tunnel_ref(conn_pid)
|
||||
|
||||
adapter_opts =
|
||||
opts[:adapter]
|
||||
|> Keyword.drop([:conn, :stream, :tunnel])
|
||||
|> Keyword.merge(conn: conn_pid, close_conn: false, tunnel: tunnel)
|
||||
|
||||
opts = Keyword.put(opts, :adapter, adapter_opts)
|
||||
env = %{env | opts: opts}
|
||||
next = use_pool_adapter(next)
|
||||
|
||||
case Tesla.run(env, next) do
|
||||
{:ok, env} ->
|
||||
unless opts[:adapter][:body_as] == :chunks do
|
||||
unless opts[:adapter][:body_as] == :chunks and chunk_client?(env.body) do
|
||||
ConnectionPool.release_conn(conn_pid)
|
||||
{_, res} = pop_in(env.opts[:adapter][:conn])
|
||||
{:ok, res}
|
||||
|
|
@ -38,8 +51,11 @@ defmodule Pleroma.Tesla.Middleware.ConnectionPool do
|
|||
{:ok, env}
|
||||
end
|
||||
|
||||
{:error, {:gun_stream_error, reason}} ->
|
||||
{:error, reason}
|
||||
|
||||
err ->
|
||||
ConnectionPool.release_conn(conn_pid)
|
||||
ConnectionPool.discard_conn(conn_pid)
|
||||
err
|
||||
end
|
||||
|
||||
|
|
@ -47,4 +63,34 @@ defmodule Pleroma.Tesla.Middleware.ConnectionPool do
|
|||
err
|
||||
end
|
||||
end
|
||||
|
||||
defp cleanup_previous_request(%Tesla.Env{
|
||||
body: %{pid: pid, stream: stream},
|
||||
opts: opts
|
||||
}) do
|
||||
if opts[:adapter][:conn], do: ConnectionPool.cancel_stream(pid, stream)
|
||||
end
|
||||
|
||||
defp cleanup_previous_request(%Tesla.Env{opts: opts}) do
|
||||
conn = opts[:adapter][:conn]
|
||||
stream = opts[:adapter][:stream]
|
||||
|
||||
cond do
|
||||
conn && stream -> ConnectionPool.cancel_stream(conn, stream)
|
||||
conn -> ConnectionPool.release_conn(conn)
|
||||
true -> :ok
|
||||
end
|
||||
end
|
||||
|
||||
defp chunk_client?(%{pid: pid, stream: stream}) when is_pid(pid) and not is_nil(stream),
|
||||
do: true
|
||||
|
||||
defp chunk_client?(_body), do: false
|
||||
|
||||
defp use_pool_adapter(next) do
|
||||
List.update_at(next, -1, fn
|
||||
{Tesla.Adapter.Gun, function, args} -> {Pleroma.HTTP.Adapter.Gun, function, args}
|
||||
adapter -> adapter
|
||||
end)
|
||||
end
|
||||
end
|
||||
|
|
|
|||
|
|
@ -273,13 +273,13 @@ defmodule Pleroma.User do
|
|||
end
|
||||
|
||||
def cached_blocked_users_ap_ids(user) do
|
||||
@cachex.fetch!(:user_cache, "blocked_users_ap_ids:#{user.ap_id}", fn _ ->
|
||||
@cachex.fetch!(:user_cache, cache_key(:blocked_users_ap_ids, user), fn _ ->
|
||||
blocked_users_ap_ids(user)
|
||||
end)
|
||||
end
|
||||
|
||||
def cached_muted_users_ap_ids(user) do
|
||||
@cachex.fetch!(:user_cache, "muted_users_ap_ids:#{user.ap_id}", fn _ ->
|
||||
@cachex.fetch!(:user_cache, cache_key(:muted_users_ap_ids, user), fn _ ->
|
||||
muted_users_ap_ids(user)
|
||||
end)
|
||||
end
|
||||
|
|
@ -1255,13 +1255,18 @@ defmodule Pleroma.User do
|
|||
get_cached_by_nickname(nickname)
|
||||
end
|
||||
|
||||
defp cache_key(:id, %User{id: id}), do: cache_key(:id, id)
|
||||
defp cache_key(:nickname, %User{nickname: nickname}), do: cache_key(:nickname, nickname)
|
||||
defp cache_key(namespace, %User{ap_id: ap_id}), do: cache_key(namespace, ap_id)
|
||||
defp cache_key(namespace, value), do: "#{namespace}:#{value}"
|
||||
|
||||
def set_cache({:ok, user}), do: set_cache(user)
|
||||
def set_cache({:error, err}), do: {:error, err}
|
||||
|
||||
def set_cache(%User{} = user) do
|
||||
@cachex.put(:user_cache, "ap_id:#{user.ap_id}", user)
|
||||
@cachex.put(:user_cache, "nickname:#{user.nickname}", user)
|
||||
@cachex.put(:user_cache, "friends_ap_ids:#{user.nickname}", get_user_friends_ap_ids(user))
|
||||
@cachex.put(:user_cache, cache_key(:ap_id, user), user)
|
||||
@cachex.put(:user_cache, cache_key(:nickname, user), user)
|
||||
@cachex.put(:user_cache, cache_key(:friends_ap_ids, user), get_user_friends_ap_ids(user))
|
||||
{:ok, user}
|
||||
end
|
||||
|
||||
|
|
@ -1289,22 +1294,23 @@ defmodule Pleroma.User do
|
|||
|
||||
@spec get_cached_user_friends_ap_ids(User.t()) :: [String.t()]
|
||||
def get_cached_user_friends_ap_ids(user) do
|
||||
@cachex.fetch!(:user_cache, "friends_ap_ids:#{user.ap_id}", fn _ ->
|
||||
@cachex.fetch!(:user_cache, cache_key(:friends_ap_ids, user), fn _ ->
|
||||
get_user_friends_ap_ids(user)
|
||||
end)
|
||||
end
|
||||
|
||||
def invalidate_cache(user) do
|
||||
@cachex.del(:user_cache, "ap_id:#{user.ap_id}")
|
||||
@cachex.del(:user_cache, "nickname:#{user.nickname}")
|
||||
@cachex.del(:user_cache, "friends_ap_ids:#{user.ap_id}")
|
||||
@cachex.del(:user_cache, "blocked_users_ap_ids:#{user.ap_id}")
|
||||
@cachex.del(:user_cache, "muted_users_ap_ids:#{user.ap_id}")
|
||||
@cachex.del(:user_cache, cache_key(:ap_id, user))
|
||||
@cachex.del(:user_cache, cache_key(:nickname, user))
|
||||
@cachex.del(:user_cache, cache_key(:id, user))
|
||||
@cachex.del(:user_cache, cache_key(:friends_ap_ids, user))
|
||||
@cachex.del(:user_cache, cache_key(:blocked_users_ap_ids, user))
|
||||
@cachex.del(:user_cache, cache_key(:muted_users_ap_ids, user))
|
||||
end
|
||||
|
||||
@spec get_cached_by_ap_id(String.t()) :: User.t() | nil
|
||||
def get_cached_by_ap_id(ap_id) do
|
||||
key = "ap_id:#{ap_id}"
|
||||
key = cache_key(:ap_id, ap_id)
|
||||
|
||||
with {:ok, nil} <- @cachex.get(:user_cache, key),
|
||||
user when not is_nil(user) <- get_by_ap_id(ap_id),
|
||||
|
|
@ -1317,14 +1323,14 @@ defmodule Pleroma.User do
|
|||
end
|
||||
|
||||
def get_cached_by_id(id) do
|
||||
key = "id:#{id}"
|
||||
key = cache_key(:id, id)
|
||||
|
||||
ap_id =
|
||||
@cachex.fetch!(:user_cache, key, fn _ ->
|
||||
user = get_by_id(id)
|
||||
|
||||
if user do
|
||||
@cachex.put(:user_cache, "ap_id:#{user.ap_id}", user)
|
||||
@cachex.put(:user_cache, cache_key(:ap_id, user), user)
|
||||
{:commit, user.ap_id}
|
||||
else
|
||||
{:ignore, ""}
|
||||
|
|
@ -1335,7 +1341,7 @@ defmodule Pleroma.User do
|
|||
end
|
||||
|
||||
def get_cached_by_nickname(nickname) do
|
||||
key = "nickname:#{nickname}"
|
||||
key = cache_key(:nickname, nickname)
|
||||
|
||||
@cachex.fetch!(:user_cache, key, fn _ ->
|
||||
case get_or_fetch_by_nickname(nickname) do
|
||||
|
|
@ -1653,7 +1659,7 @@ defmodule Pleroma.User do
|
|||
|> Oban.insert()
|
||||
end
|
||||
|
||||
@cachex.del(:user_cache, "muted_users_ap_ids:#{muter.ap_id}")
|
||||
@cachex.del(:user_cache, cache_key(:muted_users_ap_ids, muter))
|
||||
|
||||
{:ok, Enum.filter([user_mute, user_notification_mute], & &1)}
|
||||
end
|
||||
|
|
@ -1663,7 +1669,7 @@ defmodule Pleroma.User do
|
|||
with {:ok, user_mute} <- UserRelationship.delete_mute(muter, mutee),
|
||||
{:ok, user_notification_mute} <-
|
||||
UserRelationship.delete_notification_mute(muter, mutee) do
|
||||
@cachex.del(:user_cache, "muted_users_ap_ids:#{muter.ap_id}")
|
||||
@cachex.del(:user_cache, cache_key(:muted_users_ap_ids, muter))
|
||||
{:ok, [user_mute, user_notification_mute]}
|
||||
end
|
||||
end
|
||||
|
|
@ -1966,6 +1972,12 @@ defmodule Pleroma.User do
|
|||
|
||||
def approve(%User{} = user), do: {:ok, user}
|
||||
|
||||
def reject(%User{is_approved: false} = user) do
|
||||
delete(user)
|
||||
end
|
||||
|
||||
def reject(%User{} = _user), do: {:error, "User is approved"}
|
||||
|
||||
def confirm(users) when is_list(users) do
|
||||
Repo.transaction(fn ->
|
||||
Enum.map(users, fn user ->
|
||||
|
|
@ -2817,7 +2829,7 @@ defmodule Pleroma.User do
|
|||
{:ok, UserRelationship.t()} | {:error, Ecto.Changeset.t()}
|
||||
defp add_to_block(%User{} = user, %User{} = blocked, expires_at) do
|
||||
with {:ok, relationship} <- UserRelationship.create_block(user, blocked, expires_at) do
|
||||
@cachex.del(:user_cache, "blocked_users_ap_ids:#{user.ap_id}")
|
||||
@cachex.del(:user_cache, cache_key(:blocked_users_ap_ids, user))
|
||||
{:ok, relationship}
|
||||
end
|
||||
end
|
||||
|
|
@ -2826,7 +2838,7 @@ defmodule Pleroma.User do
|
|||
{:ok, UserRelationship.t()} | {:ok, nil} | {:error, Ecto.Changeset.t()}
|
||||
defp remove_from_block(%User{} = user, %User{} = blocked) do
|
||||
with {:ok, relationship} <- UserRelationship.delete_block(user, blocked) do
|
||||
@cachex.del(:user_cache, "blocked_users_ap_ids:#{user.ap_id}")
|
||||
@cachex.del(:user_cache, cache_key(:blocked_users_ap_ids, user))
|
||||
{:ok, relationship}
|
||||
end
|
||||
end
|
||||
|
|
|
|||
|
|
@ -63,7 +63,8 @@ defmodule Pleroma.User.Query do
|
|||
limit: pos_integer(),
|
||||
actor_types: [String.t()],
|
||||
birthday_day: pos_integer(),
|
||||
birthday_month: pos_integer()
|
||||
birthday_month: pos_integer(),
|
||||
staff: boolean()
|
||||
}
|
||||
| map()
|
||||
|
||||
|
|
@ -179,6 +180,10 @@ defmodule Pleroma.User.Query do
|
|||
|
||||
defp compose_query({:external, _}, query), do: location_query(query, false)
|
||||
|
||||
defp compose_query({:active, false}, query) do
|
||||
where(query, [u], u.is_active == false or u.is_approved == false or u.is_confirmed == false)
|
||||
end
|
||||
|
||||
defp compose_query({:active, _}, query) do
|
||||
where(query, [u], u.is_active == true)
|
||||
|> where([u], u.is_approved == true)
|
||||
|
|
@ -202,6 +207,10 @@ defmodule Pleroma.User.Query do
|
|||
where(query, [u], u.is_confirmed != ^bool)
|
||||
end
|
||||
|
||||
defp compose_query({:need_approval, false}, query) do
|
||||
where(query, [u], u.is_approved == true)
|
||||
end
|
||||
|
||||
defp compose_query({:need_approval, _}, query) do
|
||||
where(query, [u], u.is_approved == false)
|
||||
end
|
||||
|
|
@ -285,6 +294,19 @@ defmodule Pleroma.User.Query do
|
|||
|> where([u], fragment("date_part('month', ?)", u.birthday) == ^month)
|
||||
end
|
||||
|
||||
defp compose_query({:staff, false}, query) do
|
||||
where(query, [u], u.is_admin == false and u.is_moderator == false)
|
||||
end
|
||||
|
||||
defp compose_query({:staff, _}, query) do
|
||||
where(query, [u], u.is_admin == true or u.is_moderator == true)
|
||||
end
|
||||
|
||||
defp compose_query({:domain, domain}, query) do
|
||||
query
|
||||
|> where([u], like(u.nickname, ^"%@#{domain}"))
|
||||
end
|
||||
|
||||
defp compose_query(_unsupported_param, query), do: query
|
||||
|
||||
defp location_query(query, local) do
|
||||
|
|
|
|||
|
|
@ -540,9 +540,12 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
|
|||
# and extra sorting on "activities.id DESC NULLS LAST" would worse the query plan
|
||||
opts = Map.put(opts, :skip_extra_order, true)
|
||||
|
||||
Pagination.fetch_paginated(query, opts, pagination)
|
||||
Pagination.fetch_paginated(query, opts, pagination, pagination_binding(opts))
|
||||
end
|
||||
|
||||
defp pagination_binding(%{favorited_by: _}), do: :favorited_activity
|
||||
defp pagination_binding(_), do: nil
|
||||
|
||||
def fetch_activities(recipients, opts \\ %{}, pagination \\ :keyset) do
|
||||
list_memberships = Pleroma.List.memberships(opts[:user])
|
||||
|
||||
|
|
@ -789,6 +792,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
|
|||
|
||||
defp restrict_since(query, %{since_id: ""}), do: query
|
||||
|
||||
defp restrict_since(query, %{favorited_by: _, since_id: _}), do: query
|
||||
|
||||
defp restrict_since(query, %{since_id: since_id}) do
|
||||
from(activity in query, where: activity.id > ^since_id)
|
||||
end
|
||||
|
|
@ -896,14 +901,22 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
|
|||
|> Repo.all()
|
||||
|
||||
# Note: NO extra ordering should be done on "activities.id desc nulls last" for optimal plan
|
||||
query =
|
||||
from(
|
||||
[_activity, object] in query,
|
||||
join: hto in "hashtags_objects",
|
||||
on: hto.object_id == object.id,
|
||||
where: hto.hashtag_id in ^hashtag_ids,
|
||||
where: hto.hashtag_id in ^hashtag_ids
|
||||
)
|
||||
|
||||
if is_nil(query.distinct) do
|
||||
from([_activity, object] in query,
|
||||
distinct: [desc: object.id],
|
||||
order_by: [desc: object.id]
|
||||
)
|
||||
else
|
||||
query
|
||||
end
|
||||
end
|
||||
|
||||
defp restrict_hashtag_any(query, %{tag: tag}) when is_binary(tag) do
|
||||
|
|
@ -1006,6 +1019,14 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
|
|||
|
||||
defp restrict_state(query, _), do: query
|
||||
|
||||
defp restrict_report_target(query, %{report_target_id: actor_id}) do
|
||||
from(activity in query,
|
||||
where: fragment("?->'object'->>0 = ?", activity.data, ^actor_id)
|
||||
)
|
||||
end
|
||||
|
||||
defp restrict_report_target(query, _), do: query
|
||||
|
||||
defp restrict_assigned_account(query, %{assigned_account: assigned_account}) do
|
||||
from(activity in query,
|
||||
where: fragment("?->>'assigned_account' = ?", activity.data, ^assigned_account)
|
||||
|
|
@ -1015,9 +1036,31 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
|
|||
defp restrict_assigned_account(query, _), do: query
|
||||
|
||||
defp restrict_favorited_by(query, %{favorited_by: ap_id}) do
|
||||
newer_favorite =
|
||||
from(newer_favorite in Activity,
|
||||
where: newer_favorite.actor == ^ap_id,
|
||||
where: newer_favorite.id > parent_as(:favorited_activity).id,
|
||||
where: fragment("?->>'type' = ?", newer_favorite.data, "Like"),
|
||||
where:
|
||||
fragment(
|
||||
"associated_object_id(?) = associated_object_id(?)",
|
||||
newer_favorite.data,
|
||||
parent_as(:favorited_activity).data
|
||||
),
|
||||
select: 1
|
||||
)
|
||||
|
||||
from(
|
||||
[_activity, object] in query,
|
||||
where: fragment("(?)->'likes' \\? (?)", object.data, ^ap_id)
|
||||
[activity, object: object] in query,
|
||||
join: favorited_activity in Activity,
|
||||
as: :favorited_activity,
|
||||
on:
|
||||
favorited_activity.actor == ^ap_id and
|
||||
fragment("?->>'type' = ?", favorited_activity.data, "Like") and
|
||||
fragment("associated_object_id(?) = (?)->>'id'", favorited_activity.data, object.data),
|
||||
where: fragment("(?)->'likes' \\? (?)", object.data, ^ap_id),
|
||||
where: not exists(subquery(newer_favorite)),
|
||||
select: %Activity{activity | pagination_id: favorited_activity.id}
|
||||
)
|
||||
end
|
||||
|
||||
|
|
@ -1482,6 +1525,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
|
|||
|> restrict_actor(opts)
|
||||
|> restrict_type(opts)
|
||||
|> restrict_state(opts)
|
||||
|> restrict_report_target(opts)
|
||||
|> restrict_assigned_account(opts)
|
||||
|> restrict_favorited_by(opts)
|
||||
|> restrict_blocked(restrict_blocked_opts)
|
||||
|
|
|
|||
|
|
@ -309,7 +309,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
|
|||
} = conn,
|
||||
%{"nickname" => nickname} = params
|
||||
) do
|
||||
with {:recipient_exists, %User{} = recipient} <-
|
||||
with false <- unsupported_http_message_signature?(conn),
|
||||
{:recipient_exists, %User{} = recipient} <-
|
||||
{:recipient_exists, User.get_cached_by_nickname(nickname)},
|
||||
{:sender_exists, {:ok, %User{} = actor}} <-
|
||||
{:sender_exists, User.get_or_fetch_by_ap_id(params["actor"])},
|
||||
|
|
@ -320,6 +321,9 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
|
|||
Federator.incoming_ap_doc(params)
|
||||
json(conn, "ok")
|
||||
else
|
||||
true ->
|
||||
reject_unsupported_http_message_signature(conn)
|
||||
|
||||
{:recipient_exists, _} ->
|
||||
conn
|
||||
|> put_status(:not_found)
|
||||
|
|
@ -348,11 +352,18 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
|
|||
end
|
||||
|
||||
def inbox(%{assigns: %{valid_signature: true, valid_host_header: true}} = conn, params) do
|
||||
if unsupported_http_message_signature?(conn) do
|
||||
reject_unsupported_http_message_signature(conn)
|
||||
else
|
||||
Federator.incoming_ap_doc(params)
|
||||
json(conn, "ok")
|
||||
end
|
||||
end
|
||||
|
||||
def inbox(%{assigns: %{valid_signature: false}} = conn, params) do
|
||||
if unsupported_http_message_signature?(conn) do
|
||||
reject_unsupported_http_message_signature(conn)
|
||||
else
|
||||
Federator.incoming_failed_signature_ap_doc(%{
|
||||
method: conn.method,
|
||||
req_headers: conn.req_headers,
|
||||
|
|
@ -363,6 +374,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
|
|||
|
||||
json(conn, "ok")
|
||||
end
|
||||
end
|
||||
|
||||
# POST /relay/inbox -or- POST /internal/fetch/inbox
|
||||
def inbox(conn, %{"type" => "Create"} = params) do
|
||||
|
|
@ -381,6 +393,27 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
|
|||
|> json("error, missing HTTP Signature")
|
||||
end
|
||||
|
||||
defp reject_unsupported_http_message_signature(conn) do
|
||||
conn
|
||||
|> put_status(:unauthorized)
|
||||
|> json("error, unsupported HTTP Message Signature")
|
||||
end
|
||||
|
||||
defp unsupported_http_message_signature?(conn) do
|
||||
signature_headers = Plug.Conn.get_req_header(conn, "signature")
|
||||
|
||||
multiple_signature_headers?(signature_headers) or
|
||||
(Plug.Conn.get_req_header(conn, "signature-input") != [] and
|
||||
not cavage_signature?(signature_headers))
|
||||
end
|
||||
|
||||
defp multiple_signature_headers?([_, _ | _]), do: true
|
||||
defp multiple_signature_headers?(_), do: false
|
||||
|
||||
defp cavage_signature?(signature_headers) do
|
||||
Enum.any?(signature_headers, &Regex.match?(~r/(^|,)\s*keyId\s*=/, &1))
|
||||
end
|
||||
|
||||
defp post_inbox_relayed_create(conn, params) do
|
||||
Logger.debug(
|
||||
"Signature missing or not from author, relayed Create message, fetching object from source"
|
||||
|
|
|
|||
|
|
@ -6,6 +6,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.ActivityExpirationPolicy do
|
|||
@moduledoc "Adds expiration to all local Create activities"
|
||||
@behaviour Pleroma.Web.ActivityPub.MRF.Policy
|
||||
|
||||
use Pleroma.Web.ActivityPub.MRF.Policy
|
||||
|
||||
@impl true
|
||||
def filter(activity) do
|
||||
activity =
|
||||
|
|
|
|||
|
|
@ -3,11 +3,11 @@
|
|||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
defmodule Pleroma.Web.ActivityPub.MRF.AntiFollowbotPolicy do
|
||||
@moduledoc "Prevent followbots from following with a bit of heuristic"
|
||||
@behaviour Pleroma.Web.ActivityPub.MRF.Policy
|
||||
alias Pleroma.User
|
||||
|
||||
@moduledoc "Prevent followbots from following with a bit of heuristic"
|
||||
|
||||
@behaviour Pleroma.Web.ActivityPub.MRF.Policy
|
||||
use Pleroma.Web.ActivityPub.MRF.Policy
|
||||
|
||||
# XXX: this should become User.normalize_by_ap_id() or similar, really.
|
||||
defp normalize_by_ap_id(%{"id" => id}), do: User.get_cached_by_ap_id(id)
|
||||
|
|
|
|||
|
|
@ -3,9 +3,11 @@
|
|||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
defmodule Pleroma.Web.ActivityPub.MRF.AntiLinkSpamPolicy do
|
||||
@behaviour Pleroma.Web.ActivityPub.MRF.Policy
|
||||
|
||||
alias Pleroma.User
|
||||
|
||||
@behaviour Pleroma.Web.ActivityPub.MRF.Policy
|
||||
use Pleroma.Web.ActivityPub.MRF.Policy
|
||||
|
||||
require Logger
|
||||
|
||||
|
|
|
|||
|
|
@ -3,11 +3,13 @@
|
|||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
defmodule Pleroma.Web.ActivityPub.MRF.AntiMentionSpamPolicy do
|
||||
@behaviour Pleroma.Web.ActivityPub.MRF.Policy
|
||||
|
||||
alias Pleroma.Config
|
||||
alias Pleroma.User
|
||||
require Pleroma.Constants
|
||||
|
||||
@behaviour Pleroma.Web.ActivityPub.MRF.Policy
|
||||
use Pleroma.Web.ActivityPub.MRF.Policy
|
||||
|
||||
defp user_has_posted?(%User{} = u), do: u.note_count > 0
|
||||
|
||||
|
|
|
|||
|
|
@ -3,10 +3,12 @@
|
|||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
defmodule Pleroma.Web.ActivityPub.MRF.DropPolicy do
|
||||
require Logger
|
||||
@moduledoc "Drop and log everything received"
|
||||
@behaviour Pleroma.Web.ActivityPub.MRF.Policy
|
||||
|
||||
require Logger
|
||||
use Pleroma.Web.ActivityPub.MRF.Policy
|
||||
|
||||
@impl true
|
||||
def filter(activity) do
|
||||
Logger.debug("REJECTING #{inspect(activity)}")
|
||||
|
|
|
|||
|
|
@ -3,6 +3,8 @@
|
|||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
defmodule Pleroma.Web.ActivityPub.MRF.EmojiPolicy do
|
||||
@behaviour Pleroma.Web.ActivityPub.MRF.Policy
|
||||
|
||||
require Pleroma.Constants
|
||||
|
||||
alias Pleroma.Object.Updater
|
||||
|
|
@ -10,7 +12,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.EmojiPolicy do
|
|||
|
||||
@moduledoc "Reject or force-unlisted emojis with certain URLs or names"
|
||||
|
||||
@behaviour Pleroma.Web.ActivityPub.MRF.Policy
|
||||
use Pleroma.Web.ActivityPub.MRF.Policy
|
||||
|
||||
defp config_remove_url do
|
||||
Pleroma.Config.get([:mrf_emoji, :remove_url], [])
|
||||
|
|
|
|||
|
|
@ -3,11 +3,13 @@
|
|||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
defmodule Pleroma.Web.ActivityPub.MRF.EnsureRePrepended do
|
||||
alias Pleroma.Object
|
||||
|
||||
@moduledoc "Ensure a re: is prepended on replies to a post with a Subject"
|
||||
@behaviour Pleroma.Web.ActivityPub.MRF.Policy
|
||||
|
||||
alias Pleroma.Object
|
||||
|
||||
use Pleroma.Web.ActivityPub.MRF.Policy
|
||||
|
||||
@reply_prefix Regex.compile!("^re:[[:space:]]*", [:caseless])
|
||||
|
||||
def history_awareness, do: :auto
|
||||
|
|
|
|||
|
|
@ -4,12 +4,15 @@
|
|||
|
||||
defmodule Pleroma.Web.ActivityPub.MRF.FollowBotPolicy do
|
||||
@behaviour Pleroma.Web.ActivityPub.MRF.Policy
|
||||
|
||||
alias Pleroma.Config
|
||||
alias Pleroma.User
|
||||
alias Pleroma.Web.CommonAPI
|
||||
|
||||
require Logger
|
||||
|
||||
use Pleroma.Web.ActivityPub.MRF.Policy
|
||||
|
||||
@impl true
|
||||
def filter(activity) do
|
||||
with follower_nickname <- Config.get([:mrf_follow_bot, :follower_nickname]),
|
||||
|
|
|
|||
|
|
@ -3,9 +3,12 @@
|
|||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
defmodule Pleroma.Web.ActivityPub.MRF.ForceBotUnlistedPolicy do
|
||||
alias Pleroma.User
|
||||
@behaviour Pleroma.Web.ActivityPub.MRF.Policy
|
||||
@moduledoc "Remove bot posts from federated timeline"
|
||||
@behaviour Pleroma.Web.ActivityPub.MRF.Policy
|
||||
|
||||
alias Pleroma.User
|
||||
|
||||
use Pleroma.Web.ActivityPub.MRF.Policy
|
||||
|
||||
require Pleroma.Constants
|
||||
|
||||
|
|
|
|||
|
|
@ -3,13 +3,14 @@
|
|||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
defmodule Pleroma.Web.ActivityPub.MRF.ForceMention do
|
||||
@behaviour Pleroma.Web.ActivityPub.MRF.Policy
|
||||
require Pleroma.Constants
|
||||
|
||||
alias Pleroma.Config
|
||||
alias Pleroma.Object
|
||||
alias Pleroma.User
|
||||
|
||||
@behaviour Pleroma.Web.ActivityPub.MRF.Policy
|
||||
use Pleroma.Web.ActivityPub.MRF.Policy
|
||||
|
||||
defp get_author(url) do
|
||||
with %Object{data: %{"actor" => actor}} <- Object.normalize(url, fetch: false),
|
||||
|
|
|
|||
|
|
@ -3,13 +3,15 @@
|
|||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
defmodule Pleroma.Web.ActivityPub.MRF.ForceMentionsInContent do
|
||||
@behaviour Pleroma.Web.ActivityPub.MRF.Policy
|
||||
|
||||
require Pleroma.Constants
|
||||
|
||||
alias Pleroma.Formatter
|
||||
alias Pleroma.Object
|
||||
alias Pleroma.User
|
||||
|
||||
@behaviour Pleroma.Web.ActivityPub.MRF.Policy
|
||||
use Pleroma.Web.ActivityPub.MRF.Policy
|
||||
|
||||
@impl true
|
||||
def history_awareness, do: :auto
|
||||
|
|
|
|||
|
|
@ -3,6 +3,8 @@
|
|||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
defmodule Pleroma.Web.ActivityPub.MRF.HashtagPolicy do
|
||||
@behaviour Pleroma.Web.ActivityPub.MRF.Policy
|
||||
|
||||
require Pleroma.Constants
|
||||
|
||||
alias Pleroma.Config
|
||||
|
|
@ -14,7 +16,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.HashtagPolicy do
|
|||
Note: This MRF Policy is always enabled, if you want to disable it you have to set empty lists.
|
||||
"""
|
||||
|
||||
@behaviour Pleroma.Web.ActivityPub.MRF.Policy
|
||||
use Pleroma.Web.ActivityPub.MRF.Policy
|
||||
|
||||
@impl true
|
||||
def history_awareness, do: :manual
|
||||
|
|
|
|||
|
|
@ -3,13 +3,14 @@
|
|||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
defmodule Pleroma.Web.ActivityPub.MRF.HellthreadPolicy do
|
||||
@moduledoc "Block activities with too much mentions (configurable)"
|
||||
@behaviour Pleroma.Web.ActivityPub.MRF.Policy
|
||||
|
||||
alias Pleroma.User
|
||||
|
||||
require Pleroma.Constants
|
||||
|
||||
@moduledoc "Block activities with too much mentions (configurable)"
|
||||
|
||||
@behaviour Pleroma.Web.ActivityPub.MRF.Policy
|
||||
use Pleroma.Web.ActivityPub.MRF.Policy
|
||||
|
||||
defp delist_activity(activity, threshold) when threshold > 0 do
|
||||
follower_collection = User.get_cached_by_ap_id(activity["actor"]).follower_address
|
||||
|
|
|
|||
|
|
@ -6,6 +6,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.InlineQuotePolicy do
|
|||
@moduledoc "Force a quote line into the message content."
|
||||
@behaviour Pleroma.Web.ActivityPub.MRF.Policy
|
||||
|
||||
use Pleroma.Web.ActivityPub.MRF.Policy
|
||||
|
||||
defp build_inline_quote(template, url) do
|
||||
quote_line = String.replace(template, "{url}", "<a href=\"#{url}\">#{url}</a>")
|
||||
|
||||
|
|
|
|||
|
|
@ -3,13 +3,14 @@
|
|||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
defmodule Pleroma.Web.ActivityPub.MRF.KeywordPolicy do
|
||||
@moduledoc "Reject or Word-Replace activities with a keyword or regex"
|
||||
@behaviour Pleroma.Web.ActivityPub.MRF.Policy
|
||||
|
||||
require Pleroma.Constants
|
||||
|
||||
alias Pleroma.Web.ActivityPub.MRF.Utils
|
||||
|
||||
@moduledoc "Reject or Word-Replace activities with a keyword or regex"
|
||||
|
||||
@behaviour Pleroma.Web.ActivityPub.MRF.Policy
|
||||
use Pleroma.Web.ActivityPub.MRF.Policy
|
||||
|
||||
defp string_matches?(string, pattern) when is_binary(pattern) do
|
||||
String.contains?(string, pattern)
|
||||
|
|
|
|||
|
|
@ -6,6 +6,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do
|
|||
@moduledoc "Preloads any attachments in the MediaProxy cache by prefetching them"
|
||||
@behaviour Pleroma.Web.ActivityPub.MRF.Policy
|
||||
|
||||
use Pleroma.Web.ActivityPub.MRF.Policy
|
||||
|
||||
alias Pleroma.HTTP
|
||||
alias Pleroma.Web.MediaProxy
|
||||
|
||||
|
|
|
|||
|
|
@ -4,9 +4,10 @@
|
|||
|
||||
defmodule Pleroma.Web.ActivityPub.MRF.MentionPolicy do
|
||||
@moduledoc "Block activities which mention a user"
|
||||
|
||||
@behaviour Pleroma.Web.ActivityPub.MRF.Policy
|
||||
|
||||
use Pleroma.Web.ActivityPub.MRF.Policy
|
||||
|
||||
@impl true
|
||||
def filter(%{"type" => "Create"} = activity) do
|
||||
reject_actors = Pleroma.Config.get([:mrf_mention, :actors], [])
|
||||
|
|
|
|||
|
|
@ -6,6 +6,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.NoEmptyPolicy do
|
|||
@moduledoc "Filter local activities which have no content"
|
||||
@behaviour Pleroma.Web.ActivityPub.MRF.Policy
|
||||
|
||||
use Pleroma.Web.ActivityPub.MRF.Policy
|
||||
|
||||
alias Pleroma.Web.Endpoint
|
||||
|
||||
@impl true
|
||||
|
|
|
|||
|
|
@ -6,6 +6,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.NoOpPolicy do
|
|||
@moduledoc "Does nothing (lets the messages go through unmodified)"
|
||||
@behaviour Pleroma.Web.ActivityPub.MRF.Policy
|
||||
|
||||
use Pleroma.Web.ActivityPub.MRF.Policy
|
||||
|
||||
@impl true
|
||||
def filter(activity) do
|
||||
{:ok, activity}
|
||||
|
|
|
|||
|
|
@ -6,6 +6,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.NoPlaceholderTextPolicy do
|
|||
@moduledoc "Ensure no content placeholder is present (such as the dot from mastodon)"
|
||||
@behaviour Pleroma.Web.ActivityPub.MRF.Policy
|
||||
|
||||
use Pleroma.Web.ActivityPub.MRF.Policy
|
||||
|
||||
@impl true
|
||||
def history_awareness, do: :auto
|
||||
|
||||
|
|
|
|||
|
|
@ -4,9 +4,11 @@
|
|||
|
||||
defmodule Pleroma.Web.ActivityPub.MRF.NormalizeMarkup do
|
||||
@moduledoc "Scrub configured hypertext markup"
|
||||
@behaviour Pleroma.Web.ActivityPub.MRF.Policy
|
||||
|
||||
alias Pleroma.HTML
|
||||
|
||||
@behaviour Pleroma.Web.ActivityPub.MRF.Policy
|
||||
use Pleroma.Web.ActivityPub.MRF.Policy
|
||||
|
||||
@impl true
|
||||
def history_awareness, do: :auto
|
||||
|
|
|
|||
|
|
@ -38,6 +38,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.NsfwApiPolicy do
|
|||
- `unlist`: Unlist all detected NSFW content? Default: `false`
|
||||
- `reject`: Reject all detected NSFW content (takes precedence)? Default: `false`
|
||||
"""
|
||||
@behaviour Pleroma.Web.ActivityPub.MRF.Policy
|
||||
|
||||
alias Pleroma.Config
|
||||
alias Pleroma.Constants
|
||||
alias Pleroma.HTTP
|
||||
|
|
@ -46,7 +48,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.NsfwApiPolicy do
|
|||
require Logger
|
||||
require Pleroma.Constants
|
||||
|
||||
@behaviour Pleroma.Web.ActivityPub.MRF.Policy
|
||||
use Pleroma.Web.ActivityPub.MRF.Policy
|
||||
|
||||
@policy :mrf_nsfw_api
|
||||
|
||||
def build_request_url(url) do
|
||||
|
|
|
|||
|
|
@ -3,13 +3,15 @@
|
|||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
defmodule Pleroma.Web.ActivityPub.MRF.ObjectAgePolicy do
|
||||
@moduledoc "Filter activities depending on their age"
|
||||
@behaviour Pleroma.Web.ActivityPub.MRF.Policy
|
||||
|
||||
alias Pleroma.Config
|
||||
alias Pleroma.User
|
||||
|
||||
require Pleroma.Constants
|
||||
|
||||
@moduledoc "Filter activities depending on their age"
|
||||
@behaviour Pleroma.Web.ActivityPub.MRF.Policy
|
||||
use Pleroma.Web.ActivityPub.MRF.Policy
|
||||
|
||||
defp check_date(%{"object" => %{"published" => published}} = activity) do
|
||||
with %DateTime{} = now <- DateTime.utc_now(),
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Add a link
Reference in a new issue