diff --git a/.woodpecker/docker-armv7.yaml b/.woodpecker/docker-armv7.yaml deleted file mode 100644 index c921a56f9..000000000 --- a/.woodpecker/docker-armv7.yaml +++ /dev/null @@ -1,85 +0,0 @@ -when: - # Temporary PR validation while arm/v7 Docker builds are being brought up. - - event: pull_request - path: [ "**/*.ex", "**/*.eex", "**/*.exs", "mix.lock", ".woodpecker/**", "Dockerfile" ] - - event: push - branch: ${CI_REPO_DEFAULT_BRANCH} - path: [ "**/*.ex", "**/*.eex", "**/*.exs", "mix.lock", ".woodpecker/**", "Dockerfile" ] - - event: tag - - event: manual - branch: ${CI_REPO_DEFAULT_BRANCH} - - event: manual - branch: stable - -# Target the CAX21 arm64 host with native arm32 userspace support, not generic arm64 runners. -labels: - platform: linux/arm - armv7: native - -variables: - docker_variables: &docker_variables - repo: pleroma/pleroma - registry: git.pleroma.social - armv7_build_settings: &armv7_build_settings - <<: *docker_variables - build_args: - - ELIXIR_IMG=arm32v7/elixir - - ELIXIR_TAG=1.17.3-otp-26-alpine - - ALPINE_IMG=arm32v7/alpine - - ALPINE_VER=3.20 - extra_opts: --custom-platform=linux/arm/v7 - armv7_push_settings: &armv7_push_settings - <<: *armv7_build_settings - username: - from_secret: pleroma-ci-user - password: - from_secret: pleroma-ci-password - -steps: - docker-armv7-pr: - image: woodpeckerci/plugin-kaniko:2.3.1 - when: - - event: pull_request - settings: - <<: *armv7_build_settings - dry_run: true - tags: - - armv7-pr-check - - docker-develop-armv7: - image: woodpeckerci/plugin-kaniko:2.3.1 - when: - - event: push - branch: ${CI_REPO_DEFAULT_BRANCH} - - event: manual - branch: ${CI_REPO_DEFAULT_BRANCH} - settings: - <<: *armv7_push_settings - tags: - - latest-armv7 - - develop-armv7 - - ${CI_COMMIT_SHA:0:8}-armv7 - - docker-stable-armv7: - image: woodpeckerci/plugin-kaniko:2.3.1 - when: - - evaluate: 'CI_PIPELINE_EVENT == "manual" && CI_COMMIT_BRANCH == "stable" && CI_COMMIT_TAG == ""' - settings: - <<: *armv7_push_settings - tags: - - latest-armv7 - - stable-armv7 - - ${CI_COMMIT_SHA:0:8}-armv7 - - docker-stable-tag-armv7: - image: woodpeckerci/plugin-kaniko:2.3.1 - when: - - event: tag - - evaluate: 'CI_PIPELINE_EVENT == "manual" && CI_COMMIT_BRANCH == "stable" && CI_COMMIT_TAG != ""' - settings: - <<: *armv7_push_settings - tags: - - latest-armv7 - - stable-armv7 - - ${CI_COMMIT_SHA:0:8}-armv7 - - ${CI_COMMIT_TAG}-armv7 diff --git a/.woodpecker/docker-combine.yaml b/.woodpecker/docker-combine.yaml index 6c91d26e9..9b9f89fdb 100644 --- a/.woodpecker/docker-combine.yaml +++ b/.woodpecker/docker-combine.yaml @@ -27,7 +27,7 @@ steps: settings: &docker_settings registry: "git.pleroma.social" image: "pleroma/pleroma" - architectures: [amd64, arm64] + architectures: [amd64, arm64, armv7] tags: - latest - develop diff --git a/.woodpecker/docker.yaml b/.woodpecker/docker.yaml index 9a5be78e2..1bcf1d2ed 100644 --- a/.woodpecker/docker.yaml +++ b/.woodpecker/docker.yaml @@ -12,6 +12,7 @@ matrix: platform: - linux/amd64 - linux/arm64 + - linux/arm # This is needed for the when clauses below. labels: @@ -26,11 +27,19 @@ variables: from_secret: pleroma-ci-user password: from_secret: pleroma-ci-password - kaniko_image: &kaniko_image woodpeckerci/plugin-kaniko:2.3.1 + armv7_docker_variables: &armv7_docker_variables + <<: *docker_variables + build_args: + - ELIXIR_IMG=docker.io/arm32v7/elixir + - ELIXIR_TAG=1.17.3-otp-26-alpine + - ALPINE_IMG=docker.io/arm32v7/alpine + - ALPINE_VER=3.20 + extra_opts: --custom-platform=linux/arm/v7 + kaniko_image: &kaniko_image docker.io/woodpeckerci/plugin-kaniko:2.3.2 steps: docker-develop-amd64: - image: woodpeckerci/plugin-kaniko:2.3.1 + image: *kaniko_image when: - evaluate: 'platform == "linux/amd64" && CI_COMMIT_BRANCH == "${CI_REPO_DEFAULT_BRANCH}"' settings: @@ -41,7 +50,7 @@ steps: - ${CI_COMMIT_SHA:0:8}-amd64 docker-develop-arm64: - image: woodpeckerci/plugin-kaniko:2.3.1 + image: *kaniko_image when: - evaluate: 'platform == "linux/arm64" && CI_COMMIT_BRANCH == "${CI_REPO_DEFAULT_BRANCH}"' settings: @@ -51,6 +60,17 @@ steps: - develop-arm64 - ${CI_COMMIT_SHA:0:8}-arm64 + docker-develop-armv7: + image: *kaniko_image + when: + - evaluate: 'platform == "linux/arm" && CI_COMMIT_BRANCH == "${CI_REPO_DEFAULT_BRANCH}"' + settings: + <<: *armv7_docker_variables + tags: + - latest-armv7 + - develop-armv7 + - ${CI_COMMIT_SHA:0:8}-armv7 + docker-stable-amd64: image: *kaniko_image when: @@ -106,3 +126,25 @@ steps: tags: - <<: *arm64_tags - ${CI_COMMIT_TAG}-arm64 + + docker-stable-armv7: + image: *kaniko_image + when: + - evaluate: 'platform == "linux/arm" && CI_PIPELINE_EVENT == "manual" && CI_COMMIT_BRANCH == "stable" && CI_COMMIT_TAG == ""' + settings: + <<: *armv7_docker_variables + tags: &armv7_tags + - latest-armv7 + - stable-armv7 + - ${CI_COMMIT_SHA:0:8}-armv7 + + docker-stable-tag-armv7: + image: *kaniko_image + when: + - evaluate: 'platform == "linux/arm" && CI_PIPELINE_EVENT == "tag"' + - evaluate: 'platform == "linux/arm" && CI_PIPELINE_EVENT == "manual" && CI_COMMIT_BRANCH == "stable" && CI_COMMIT_TAG != ""' + settings: + <<: *armv7_docker_variables + tags: + - <<: *armv7_tags + - ${CI_COMMIT_TAG}-armv7 diff --git a/.woodpecker/unit-testing-elixir-1.15.yaml b/.woodpecker/unit-testing-elixir-1.15.yaml index a5bcc0ef3..0f8eac566 100644 --- a/.woodpecker/unit-testing-elixir-1.15.yaml +++ b/.woodpecker/unit-testing-elixir-1.15.yaml @@ -24,10 +24,11 @@ steps: entrypoint: *script_file_entrypoint environment: MIX_ENV: test + VIX_COMPILATION_MODE: PLATFORM_PROVIDED_LIBVIPS DB_HOST: postgres DB_PORT: 5432 commands: - - apk add --no-cache build-base cmake exiftool ffmpeg file-dev git openssl + - apk add --no-cache build-base cmake exiftool ffmpeg vips-dev file-dev git openssl - adduser -D -h /home/testuser testuser - mkdir -p /home/testuser/.mix /home/testuser/.hex - chown -R testuser:testuser . /home/testuser diff --git a/.woodpecker/unit-testing-elixir-1.18.yaml b/.woodpecker/unit-testing-elixir-1.19.yaml similarity index 85% rename from .woodpecker/unit-testing-elixir-1.18.yaml rename to .woodpecker/unit-testing-elixir-1.19.yaml index 6e576fad9..60b0826a7 100644 --- a/.woodpecker/unit-testing-elixir-1.18.yaml +++ b/.woodpecker/unit-testing-elixir-1.19.yaml @@ -19,15 +19,16 @@ variables: - 'printf "%s" "$CI_SCRIPT" | base64 -d > /tmp/ci-script.sh && /bin/sh -xe /tmp/ci-script.sh' steps: - unit-testing-elixir-1.18: - image: elixir:1.18-otp-27-alpine + unit-testing-elixir-1.19: + image: elixir:1.19-otp-28-alpine entrypoint: *script_file_entrypoint environment: MIX_ENV: test + VIX_COMPILATION_MODE: PLATFORM_PROVIDED_LIBVIPS DB_HOST: postgres DB_PORT: 5432 commands: - - apk add --no-cache build-base cmake exiftool ffmpeg file-dev git openssl + - apk add --no-cache build-base cmake exiftool ffmpeg vips-dev file-dev git openssl - adduser -D -h /home/testuser testuser - mkdir -p /home/testuser/.mix /home/testuser/.hex - chown -R testuser:testuser . /home/testuser diff --git a/Dockerfile b/Dockerfile index 5fd1cc0a1..87c698f93 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,8 +1,8 @@ # https://hub.docker.com/r/hexpm/elixir/tags ARG ELIXIR_IMG=hexpm/elixir ARG ELIXIR_VER=1.17.3 -ARG ERLANG_VER=26.2.5.6 -ARG ALPINE_VER=3.17.9 +ARG ERLANG_VER=27.3.4.14 +ARG ALPINE_VER=3.23.5 ARG ELIXIR_TAG=${ELIXIR_VER}-erlang-${ERLANG_VER}-alpine-${ALPINE_VER} ARG ALPINE_IMG=alpine diff --git a/changelog.d/account-favourites-order.fix b/changelog.d/account-favourites-order.fix new file mode 100644 index 000000000..94e27e499 --- /dev/null +++ b/changelog.d/account-favourites-order.fix @@ -0,0 +1 @@ +Fixed `/api/v1/pleroma/accounts/:id/favourites` ordering and pagination to use the favourite date. diff --git a/changelog.d/disabling-chats.change b/changelog.d/disabling-chats.change new file mode 100644 index 000000000..579679a03 --- /dev/null +++ b/changelog.d/disabling-chats.change @@ -0,0 +1 @@ +Allow disabling Chats feature diff --git a/changelog.d/docker-arm32v7-pr-forbid-build.skip b/changelog.d/docker-arm32v7-pr-forbid-build.skip new file mode 100644 index 000000000..e69de29bb diff --git a/changelog.d/docker-baseimage-ssl.fix b/changelog.d/docker-baseimage-ssl.fix new file mode 100644 index 000000000..441423f49 --- /dev/null +++ b/changelog.d/docker-baseimage-ssl.fix @@ -0,0 +1 @@ +Dockerfile: update base images to fix problems with SSL handshakes \ No newline at end of file diff --git a/changelog.d/elixir-1-19.add b/changelog.d/elixir-1-19.add new file mode 100644 index 000000000..b010b2dfa --- /dev/null +++ b/changelog.d/elixir-1-19.add @@ -0,0 +1 @@ +Implemented ConfigDB configuration blacklist and accompanying Mix task diff --git a/changelog.d/elixir-1-19.fix b/changelog.d/elixir-1-19.fix new file mode 100644 index 000000000..53079b0f7 --- /dev/null +++ b/changelog.d/elixir-1-19.fix @@ -0,0 +1 @@ +Fixed Elixir 1.19 warnings diff --git a/changelog.d/fix-preferred-frontend.fix b/changelog.d/fix-preferred-frontend.fix new file mode 100644 index 000000000..07bf4d0c6 --- /dev/null +++ b/changelog.d/fix-preferred-frontend.fix @@ -0,0 +1 @@ +Fix preferred frontend not working due to InstanceStatic plug prematurely returning HTTP 404 diff --git a/changelog.d/grouped-notifications-api.add b/changelog.d/grouped-notifications-api.add new file mode 100644 index 000000000..8c2cc5397 --- /dev/null +++ b/changelog.d/grouped-notifications-api.add @@ -0,0 +1 @@ +Add Mastodon-compatible grouped notifications API endpoints under `/api/v2/notifications` for newly-created notifications diff --git a/changelog.d/grouped_notifications_flag.skip b/changelog.d/grouped_notifications_flag.skip new file mode 100644 index 000000000..e69de29bb diff --git a/changelog.d/gun-pool-rework.fix b/changelog.d/gun-pool-rework.fix new file mode 100644 index 000000000..d555b5bce --- /dev/null +++ b/changelog.d/gun-pool-rework.fix @@ -0,0 +1 @@ +Fixed Gun HTTP/1 concurrency, forwarding and authenticated proxy handling, tunnel requests, and streamed connection cleanup. Updated Gun and Cowlib for HTTP client security fixes. diff --git a/changelog.d/hashtag-timeline.fix b/changelog.d/hashtag-timeline.fix new file mode 100644 index 000000000..c5814d9c9 --- /dev/null +++ b/changelog.d/hashtag-timeline.fix @@ -0,0 +1 @@ +Fix hashtag timeline query for additional tag handling diff --git a/changelog.d/list-emoji.add b/changelog.d/list-emoji.add new file mode 100644 index 000000000..dce4874d3 --- /dev/null +++ b/changelog.d/list-emoji.add @@ -0,0 +1 @@ +Allow attaching emojis to user lists diff --git a/changelog.d/logger-backends.change b/changelog.d/logger-backends.change new file mode 100644 index 000000000..5f044c2c7 --- /dev/null +++ b/changelog.d/logger-backends.change @@ -0,0 +1 @@ +Changed how Logger is configured due to changes in Elixir and added deprecation warnings for the old configuration format diff --git a/changelog.d/mediaproxy-content-type.fix b/changelog.d/mediaproxy-content-type.fix new file mode 100644 index 000000000..f36dd6283 --- /dev/null +++ b/changelog.d/mediaproxy-content-type.fix @@ -0,0 +1 @@ +Detect image content types for extensionless MediaProxy responses. diff --git a/changelog.d/oauth-user-preload.change b/changelog.d/oauth-user-preload.change new file mode 100644 index 000000000..43f5ea06b --- /dev/null +++ b/changelog.d/oauth-user-preload.change @@ -0,0 +1 @@ +Avoid a duplicate user query during OAuth authentication. diff --git a/changelog.d/publisher-worker-atoms.fix b/changelog.d/publisher-worker-atoms.fix new file mode 100644 index 000000000..2df6608ba --- /dev/null +++ b/changelog.d/publisher-worker-atoms.fix @@ -0,0 +1 @@ +Avoid creating atoms from serialized PublisherWorker parameters. diff --git a/changelog.d/rss.fix b/changelog.d/rss.fix new file mode 100644 index 000000000..a3455698a --- /dev/null +++ b/changelog.d/rss.fix @@ -0,0 +1 @@ +Fix RSS feeds to be W3C compliant and render better in RSS readers diff --git a/changelog.d/stream-domain-block-following.fix b/changelog.d/stream-domain-block-following.fix new file mode 100644 index 000000000..84890c680 --- /dev/null +++ b/changelog.d/stream-domain-block-following.fix @@ -0,0 +1 @@ +Keep followed users visible in streams when their domain is blocked. diff --git a/changelog.d/unit-testing-build-vix.skip b/changelog.d/unit-testing-build-vix.skip new file mode 100644 index 000000000..e69de29bb diff --git a/changelog.d/user-cache-keys.fix b/changelog.d/user-cache-keys.fix new file mode 100644 index 000000000..4af21dd1f --- /dev/null +++ b/changelog.d/user-cache-keys.fix @@ -0,0 +1 @@ +Fix inconsistent User cache keys and invalidate numeric ID aliases. diff --git a/changelog.d/woodpecker-armv7-fixups.fix b/changelog.d/woodpecker-armv7-fixups.fix new file mode 100644 index 000000000..7bbe7ab3e --- /dev/null +++ b/changelog.d/woodpecker-armv7-fixups.fix @@ -0,0 +1 @@ +Include arm32v7 in multi-arch Docker images diff --git a/config/config.exs b/config/config.exs index 3f880af53..f13c12fb9 100644 --- a/config/config.exs +++ b/config/config.exs @@ -133,13 +133,17 @@ config :pleroma, Pleroma.Web.Endpoint, ] # Configures Elixir's Logger -config :logger, backends: [:console] +# Primary config +config :logger, level: :debug -config :logger, :console, - level: :debug, +# Console config +config :logger, :default_handler, level: :debug + +config :logger, :default_formatter, format: "\n$time $metadata[$level] $message\n", metadata: [:actor, :path, :type, :user] +# Syslog config config :logger, :ex_syslogger, level: :debug, ident: "pleroma", @@ -962,6 +966,10 @@ config :pleroma, Pleroma.Search.QdrantSearch, vectors: %{size: 384, distance: "Cosine"} } +config :pleroma, :database_config_blacklist, [ + {:pleroma, :logger} +] + config :pleroma, :database_config_whitelist, [ {:pleroma}, {:cors_plug}, @@ -971,6 +979,8 @@ config :pleroma, :database_config_whitelist, [ {:web_push_encryption, :vapid_details} ] +config :pleroma, Pleroma.Chat, enabled: true + # Import environment specific config. This must remain at the bottom # of this file so it overrides the configuration defined above. import_config "#{Mix.env()}.exs" diff --git a/config/description.exs b/config/description.exs index 7c377588c..6ba3c32d2 100644 --- a/config/description.exs +++ b/config/description.exs @@ -2726,6 +2726,19 @@ config :pleroma, :config_description, [ } ] }, + %{ + group: :pleroma, + key: Pleroma.Chat, + type: :group, + description: "Pleroma Chat settings", + children: [ + %{ + key: :enabled, + type: :boolean, + description: "Enables the chats API." + } + ] + }, %{ group: :pleroma, key: :http, @@ -3085,9 +3098,16 @@ config :pleroma, :config_description, [ %{ key: :max_connections, type: :integer, - description: "Maximum number of connections in the pool. Default: 250 connections.", + description: + "Maximum total number of connections in the pool. HTTP/1 origins may use multiple connections within this limit, while HTTP/2 connections are multiplexed. Default: 250 connections.", suggestions: [250] }, + %{ + key: :max_idle_time, + type: :integer, + description: "Time before an unused connection is closed. Default: 30000ms.", + suggestions: [30_000] + }, %{ key: :connect_timeout, type: :integer, diff --git a/config/dev.exs b/config/dev.exs index 14cf4a6dc..e86bdb7fd 100644 --- a/config/dev.exs +++ b/config/dev.exs @@ -36,7 +36,7 @@ config :pleroma, Pleroma.Emails.Mailer, adapter: Swoosh.Adapters.Local # different ports. # Do not include timestamps in development logs -config :logger, Logger.Backends.Console, format: "$metadata[$level] $message\n" +config :logger, :default_formatter, format: "$metadata[$level] $message\n" # Set a higher stacktrace during development. Avoid configuring such # in production as building large stacktraces may be expensive. diff --git a/config/prod.exs b/config/prod.exs index 2d252bf02..1dfbbe937 100644 --- a/config/prod.exs +++ b/config/prod.exs @@ -20,8 +20,7 @@ config :pleroma, Pleroma.Web.Endpoint, config :phoenix, serve_endpoints: true # Do not print debug messages in production -config :logger, Logger.Backends.Console, level: :info -config :logger, :console, level: :info +config :logger, :default_handler, level: :info config :logger, :ex_syslogger, level: :info # ## SSL Support diff --git a/config/test.exs b/config/test.exs index 9652b7d4b..ed5283a26 100644 --- a/config/test.exs +++ b/config/test.exs @@ -15,9 +15,11 @@ config :pleroma, Pleroma.Captcha, method: Pleroma.Captcha.Mock # Print only warnings and errors during test -config :logger, :console, - level: :warning, - format: "\n[$level] $message\n" +config :logger, :default_handler, level: :warning + +config :logger, :default_formatter, + format: "\n[$level] $message\n", + colors: [enabled: false] config :pleroma, :auth, oauth_consumer_strategies: [] diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md index 9efa1c8b3..0275d2be0 100644 --- a/docs/configuration/cheatsheet.md +++ b/docs/configuration/cheatsheet.md @@ -573,7 +573,8 @@ Settings for HTTP connection pool. * `:connection_acquisition_wait` - Timeout to acquire a connection from pool.The total max time is this value multiplied by the number of retries. * `connection_acquisition_retries` - Number of attempts to acquire the connection from the pool if it is overloaded. Each attempt is timed `:connection_acquisition_wait` apart. -* `:max_connections` - Maximum number of connections in the pool. +* `:max_connections` - Maximum total number of connections in the pool. HTTP/1 origins may use multiple connections within this limit, while HTTP/2 connections are multiplexed. +* `:max_idle_time` - Time before an unused connection is closed. * `:connect_timeout` - Timeout to connect to the host. * `:reclaim_multiplier` - Multiplied by `:max_connections` this will be the maximum number of idle connections that will be reclaimed in case the pool is overloaded. @@ -858,21 +859,23 @@ Web Push Notifications configuration. You can use the mix task `mix web_push.gen * ``private_key``: VAPID private key ## :logger -* `backends`: `:console` is used to send logs to stdout, `{ExSyslogger, :ex_syslogger}` to log to syslog +* Logging to console/stdout is done by default, use `{ExSyslogger, :ex_syslogger}` to log to syslog An example to enable ONLY ExSyslogger (f/ex in ``prod.secret.exs``) with info and debug suppressed: ```elixir -config :logger, +config :pleroma, :logger, backends: [{ExSyslogger, :ex_syslogger}] +config :logger, default_handler: false + config :logger, :ex_syslogger, level: :warning ``` Another example, keeping console output and adding the pid to syslog output: ```elixir -config :logger, - backends: [:console, {ExSyslogger, :ex_syslogger}] +config :pleroma, :logger, + backends: [{ExSyslogger, :ex_syslogger}] config :logger, :ex_syslogger, level: :warning, @@ -883,23 +886,22 @@ See: [loggerโ€™s documentation](https://hexdocs.pm/logger/Logger.html) and [ex_s An example of logging info to local syslog, but debug to console: ```elixir -config :logger, - backends: [ {ExSyslogger, :ex_syslogger}, :console ], - level: :info +config :pleroma, :logger, + backends: [{ExSyslogger, :ex_syslogger}] config :logger, :ex_syslogger, level: :info, ident: "pleroma", format: "$metadata[$level] $message" -config :logger, :console, - level: :debug, +config :logger, :default_handler, + level: :debug + +config :logger, :default_formatter, format: "\n$time $metadata[$level] $message\n", metadata: [:request_id] ``` - - ## Database options ### RUM indexing for full text search diff --git a/docs/configuration/howto_proxy.md b/docs/configuration/howto_proxy.md index 10a635266..d805ff965 100644 --- a/docs/configuration/howto_proxy.md +++ b/docs/configuration/howto_proxy.md @@ -10,3 +10,6 @@ The other way to do it, for example, with Tor you would most likely add somethin ``` config :pleroma, :http, proxy_url: {:socks5, :localhost, 9050} ``` + +The Gun HTTP adapter supports SOCKS5. SOCKS4 proxies require another HTTP adapter. +For regular HTTP proxies, Gun forwards plain HTTP requests directly and uses CONNECT tunnels for HTTPS requests. diff --git a/docs/installation/generic_dependencies.include b/docs/installation/generic_dependencies.include index 588a6c019..382f07a6c 100644 --- a/docs/installation/generic_dependencies.include +++ b/docs/installation/generic_dependencies.include @@ -1,8 +1,8 @@ ## Required dependencies * PostgreSQL >=11.0 -* Elixir >=1.15.0 <1.19 -* Erlang OTP >=23.0.0 (supported: <28) +* Elixir >=1.15.0, <1.20 +* Erlang OTP >=24.0.0 (supported: <29) * git * file / libmagic * gcc or clang diff --git a/lib/mix/tasks/pleroma/config.ex b/lib/mix/tasks/pleroma/config.ex index c5f2e9b0a..a10506236 100644 --- a/lib/mix/tasks/pleroma/config.ex +++ b/lib/mix/tasks/pleroma/config.ex @@ -252,40 +252,43 @@ defmodule Mix.Tasks.Pleroma.Config do if whitelisted_configs in [nil, false] do shell_error("No unwanted settings in ConfigDB. No changes made.") else - whitelisted_groups = - whitelisted_configs - |> Enum.filter(fn - {_group} -> true - _ -> false - end) - |> Enum.map(fn {group} -> group end) - - whitelisted_keys = - whitelisted_configs - |> Enum.filter(fn - {_group, _key} -> true - _ -> false - end) + {whitelisted_groups, whitelisted_keys} = get_filters(whitelisted_configs) filtered = from(c in ConfigDB) |> Repo.all() |> Enum.filter(¬_whitelisted?(&1, whitelisted_groups, whitelisted_keys)) - if not Enum.empty?(filtered) do - shell_info("The following settings will be removed from ConfigDB:\n") - Enum.each(filtered, &dump(&1)) + do_config_filter(filtered, force) + end + end - if force or shell_prompt("Are you sure you want to continue?", "n") in ~w(Yn Y y) do - filtered_ids = Enum.map(filtered, fn %{id: id} -> id end) + # Removes blacklisted configuration sections + def run(["filter_blacklisted" | rest]) do + {options, [], []} = + OptionParser.parse( + rest, + strict: [force: :boolean], + aliases: [f: :force] + ) - Repo.delete_all(from(c in ConfigDB, where: c.id in ^filtered_ids)) - else - shell_error("No changes made.") - end - else - shell_error("No unwanted settings in ConfigDB. No changes made.") - end + force = Keyword.get(options, :force, false) + + start_pleroma() + + blacklisted_configs = Pleroma.Config.get(:database_config_blacklist) + + if blacklisted_configs in [nil, false] do + shell_error("No unwanted settings in ConfigDB. No changes made.") + else + {blacklisted_groups, blacklisted_keys} = get_filters(blacklisted_configs) + + filtered = + from(c in ConfigDB) + |> Repo.all() + |> Enum.filter(&blacklisted?(&1, blacklisted_groups, blacklisted_keys)) + + do_config_filter(filtered, force) end end @@ -490,8 +493,49 @@ defmodule Mix.Tasks.Pleroma.Config do Ecto.Adapters.SQL.query!(Repo, "ALTER SEQUENCE config_id_seq RESTART;") end + defp get_filters(config) do + groups = + config + |> Enum.filter(fn + {_group} -> true + _ -> false + end) + |> Enum.map(fn {group} -> group end) + + keys = + config + |> Enum.filter(fn + {_group, _key} -> true + _ -> false + end) + + {groups, keys} + end + defp not_whitelisted?(%{group: group, key: key}, whitelisted_groups, whitelisted_keys) do not Enum.member?(whitelisted_groups, group) and not Enum.member?(whitelisted_keys, {group, key}) end + + defp blacklisted?(%{group: group, key: key}, blacklisted_groups, blacklisted_keys) do + Enum.member?(blacklisted_groups, group) or + Enum.member?(blacklisted_keys, {group, key}) + end + + defp do_config_filter(filtered, _) when length(filtered) == 0 do + shell_error("No unwanted settings in ConfigDB. No changes made.") + end + + defp do_config_filter(filtered, force) do + shell_info("The following settings will be removed from ConfigDB:\n") + Enum.each(filtered, &dump(&1)) + + if force or shell_prompt("Are you sure you want to continue?", "n") in ~w(Yn Y y) do + filtered_ids = Enum.map(filtered, fn %{id: id} -> id end) + + Repo.delete_all(from(c in ConfigDB, where: c.id in ^filtered_ids)) + else + shell_error("No changes made.") + end + end end diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 8e1c5de0d..78073b430 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -43,6 +43,7 @@ defmodule Pleroma.Application do # every time the application is restarted, so we disable module # conflicts at runtime Code.compiler_options(ignore_module_conflict: true) + configure_logger() Pleroma.Telemetry.Logger.attach() Config.Holder.save_default() Pleroma.HTML.compile_scrubbers() @@ -113,6 +114,44 @@ defmodule Pleroma.Application do Supervisor.start_link(children, opts) end + def configure_logger do + Config.get([:logger, :backends], []) + |> Enum.each(fn backend -> + case backend_to_logger(backend) do + {:ok, logger} -> + add_logger(logger) + + {:error, :console_backend} -> + Logger.warning(":console is no longer considered a backend and is enabled by default") + end + end) + end + + defp add_logger(backend) do + case LoggerBackends.add(backend) do + {:ok, _} -> + Logger.debug("Successfully added logger backend: #{inspect(backend)}") + + {:error, :already_present} -> + :ok + + {:error, reason} -> + Logger.error("Failed to add logger backend #{inspect(backend)}: #{inspect(reason)}") + end + end + + defp backend_to_logger(:console), do: {:error, :console_backend} + + defp backend_to_logger({:ex_syslogger = backend, name}) do + Logger.warning( + "Configuration {:#{backend}, :#{name}} is incorrect. Use {ExSyslogger, :#{name}} instead!" + ) + + {:ok, {ExSyslogger, name}} + end + + defp backend_to_logger(backend), do: {:ok, backend} + def load_custom_modules do dir = Config.get([:modules, :runtime_dir]) diff --git a/lib/pleroma/chat.ex b/lib/pleroma/chat.ex index 926176bfa..6ac1b24b8 100644 --- a/lib/pleroma/chat.ex +++ b/lib/pleroma/chat.ex @@ -108,4 +108,6 @@ defmodule Pleroma.Chat do |> cast(%{pinned: false}, [:pinned]) |> Repo.update() end + + def enabled?, do: Pleroma.Config.get([__MODULE__, :enabled], true) end diff --git a/lib/pleroma/config/deprecation_warnings.ex b/lib/pleroma/config/deprecation_warnings.ex index 58d164dc7..934a4a02e 100644 --- a/lib/pleroma/config/deprecation_warnings.ex +++ b/lib/pleroma/config/deprecation_warnings.ex @@ -20,6 +20,8 @@ defmodule Pleroma.Config.DeprecationWarnings do "\n* `config :pleroma, :instance, mrf_transparency_exclusions` is now `config :pleroma, :mrf, transparency_exclusions`"} ] + @logger_formatter_config_knobs [:colors, :format, :metadata, :truncate, :utc_log] + def check_exiftool_filter do filters = Config.get([Pleroma.Upload]) |> Keyword.get(:filters, []) @@ -218,7 +220,8 @@ defmodule Pleroma.Config.DeprecationWarnings do check_quarantined_instances_tuples(), check_transparency_exclusions_tuples(), check_simple_policy_tuples(), - check_exiftool_filter() + check_exiftool_filter(), + check_deprecated_logger_config() ] |> Enum.reduce(:ok, fn :ok, :ok -> :ok @@ -415,4 +418,72 @@ defmodule Pleroma.Config.DeprecationWarnings do :ok end end + + defp merge_deprecated_logger_config(config) do + formatter_config = Enum.filter(config, fn {k, _} -> k in @logger_formatter_config_knobs end) + formatter = Logger.default_formatter(formatter_config) + log_level = Keyword.get(config, :level, nil) + + Logger.debug(""" + Reconfiguring console Logger with deprecated configuration syntax. + Handler configuration: + log_level: #{inspect(log_level)} + + Formatter: + #{inspect(formatter)} + """) + + if Config.get(:env) != :test do + if log_level, do: :logger.update_handler_config(:default, :level, log_level) + :logger.set_handler_config(:default, :formatter, formatter) + end + end + + @spec check_deprecated_logger_config() :: :ok | :error + def check_deprecated_logger_config do + backends_config = Application.get_env(:logger, :backends) + console_config = Application.get_env(:logger, :console) + + # NOTE: No need to merge the old backends config since it still works. + # And new configuration will just add new Logger backends. + backend = + if backends_config do + Logger.warning(""" + !!!DEPRECATION WARNING!!! + Your configuration is using deprecated syntax for configuring backends of Elixir's logger. + `config :logger, backends: [...]` is deprecated syntax due to changes in Elixir. + Please update your configuration at your earliest convenience to use: + `config :pleroma, :logger, backends: [...]` + + Note: `:console` is no longer considered a backend and is used by default, you can disable it using: + `config :logger, :default_handler: false` + """) + + true + else + false + end + + console = + if console_config do + Logger.warning(""" + !!!DEPRECATION WARNING!!! + Your configuration is using deprecated syntax for configuring logging to console. + `config :logger, :console` is deprecated syntax due to changes in Elixir. + Please update your configuration at your earliest convenience to use + `config :logger, default_handler` and `config :logger, :default_formatter`. + + Note: `:default_handler` is used only for the `level` setting. All other configurations go under + `:default_formatter`. For more info visit: https://hexdocs.pm/logger/Logger.html#module-backends-and-backwards-compatibility + """) + + merge_deprecated_logger_config(console_config) + + true + else + false + end + + if backend or console, do: :error, else: :ok + end end diff --git a/lib/pleroma/config/transfer_task.ex b/lib/pleroma/config/transfer_task.ex index 140dd7711..3e59dc4e0 100644 --- a/lib/pleroma/config/transfer_task.ex +++ b/lib/pleroma/config/transfer_task.ex @@ -51,7 +51,7 @@ defmodule Pleroma.Config.TransferTask do started_applications = Application.started_applications() - reject = [nil, :postgrex] + reject = [nil, :postgrex, :logger] reject = if restart_pleroma? do diff --git a/lib/pleroma/docs/generator.ex b/lib/pleroma/docs/generator.ex index 93b19484f..70413837a 100644 --- a/lib/pleroma/docs/generator.ex +++ b/lib/pleroma/docs/generator.ex @@ -12,13 +12,27 @@ defmodule Pleroma.Docs.Generator do @spec list_behaviour_implementations(behaviour :: module()) :: [module()] def list_behaviour_implementations(behaviour) do - :code.all_loaded() - |> Enum.filter(fn {module, _} -> + # :code.all_loaded() no longer works on Elixir 1.19 and newer since modules are lazily loaded. + # We can get a complete list of modules by interating over Application.loaded_applications/0, + # but that includes every module which would get loaded below, which is very slow. + # Hardcode the Pleroma application instead, which is good enough for now. + application_modules = + case :application.get_key(:pleroma, :modules) do + {:ok, modules} -> modules + :undefined -> [] + end + + # At compile time or in tests some modules will be outside of the Pleroma application. + loaded_modules = Enum.map(:code.all_loaded(), fn {module, _} -> module end) + + (application_modules ++ loaded_modules) + |> Enum.uniq() + |> Enum.filter(fn module -> + Code.ensure_loaded(module) + # This shouldn't be needed as all modules are expected to have module_info/1, # but in test environments some transient modules `:elixir_compiler_XX` # are loaded for some reason (where XX is a random integer). - Code.ensure_loaded(module) - if function_exported?(module, :module_info, 1) do module.module_info(:attributes) |> Keyword.get_values(:behaviour) @@ -26,7 +40,6 @@ defmodule Pleroma.Docs.Generator do |> Enum.member?(behaviour) end end) - |> Enum.map(fn {module, _} -> module end) end @doc """ diff --git a/lib/pleroma/gun.ex b/lib/pleroma/gun.ex index c2d6b4bf2..764eb4b72 100644 --- a/lib/pleroma/gun.ex +++ b/lib/pleroma/gun.ex @@ -6,9 +6,12 @@ defmodule Pleroma.Gun do @callback open(charlist(), pos_integer(), map()) :: {:ok, pid()} @callback info(pid()) :: map() @callback close(pid()) :: :ok + @callback cancel(pid(), reference() | [reference()]) :: :ok @callback await_up(pid, pos_integer()) :: {:ok, atom()} | {:error, atom()} @callback connect(pid(), map()) :: reference() - @callback await(pid(), reference()) :: {:response, :fin, 200, []} + @callback await(pid(), reference()) :: tuple() + @callback await_tunnel_up(pid(), reference() | :undefined, timeout()) :: + {:ok, atom()} | {:error, term()} @callback set_owner(pid(), pid()) :: :ok defp api, do: Pleroma.Config.get([Pleroma.Gun], Pleroma.Gun.API) @@ -19,11 +22,15 @@ defmodule Pleroma.Gun do def close(pid), do: api().close(pid) + def cancel(pid, stream), do: api().cancel(pid, stream) + def await_up(pid, timeout \\ 5_000), do: api().await_up(pid, timeout) def connect(pid, opts), do: api().connect(pid, opts) def await(pid, ref), do: api().await(pid, ref) + def await_tunnel_up(pid, ref, timeout), do: api().await_tunnel_up(pid, ref, timeout) + def set_owner(pid, owner), do: api().set_owner(pid, owner) end diff --git a/lib/pleroma/gun/api.ex b/lib/pleroma/gun/api.ex index ff2100623..8f6387069 100644 --- a/lib/pleroma/gun/api.ex +++ b/lib/pleroma/gun/api.ex @@ -32,6 +32,9 @@ defmodule Pleroma.Gun.API do @impl Gun defdelegate close(pid), to: :gun + @impl Gun + defdelegate cancel(pid, stream), to: :gun + @impl Gun defdelegate await_up(pid, timeout \\ 5_000), to: :gun @@ -41,6 +44,28 @@ defmodule Pleroma.Gun.API do @impl Gun defdelegate await(pid, ref), to: :gun + @impl Gun + def await_tunnel_up(pid, stream_ref, timeout) do + monitor_ref = Process.monitor(pid) + + result = + receive do + {:gun_tunnel_up, ^pid, ^stream_ref, protocol} -> + {:ok, protocol} + + {:gun_down, ^pid, _protocol, reason, _killed_streams} -> + {:error, {:down, reason}} + + {:DOWN, ^monitor_ref, :process, ^pid, reason} -> + {:error, {:down, reason}} + after + timeout -> {:error, :timeout} + end + + Process.demonitor(monitor_ref, [:flush]) + result + end + @impl Gun defdelegate set_owner(pid, owner), to: :gun end diff --git a/lib/pleroma/gun/conn.ex b/lib/pleroma/gun/conn.ex index 804cd11c7..85a2fb0e8 100644 --- a/lib/pleroma/gun/conn.ex +++ b/lib/pleroma/gun/conn.ex @@ -42,18 +42,34 @@ defmodule Pleroma.Gun.Conn do Map.put(opts, :tls_opts, tls_opts) end + defp do_open(%URI{scheme: "http"} = uri, %{proxy: {proxy_host, proxy_port}} = opts) do + with open_opts <- opts |> proxy_open_opts() |> Map.put(:protocols, [:http]), + {:ok, conn} <- Gun.open(proxy_host, proxy_port, open_opts), + {:ok, protocol} <- await_up(conn, opts[:connect_timeout]) do + {:ok, conn, protocol, :forward_proxy} + else + error -> + Logger.warning( + "Opening forward proxy connection to #{compose_uri_log(uri)} failed with error #{inspect(error)}" + ) + + error + end + end + defp do_open(uri, %{proxy: {proxy_host, proxy_port}} = opts) do connect_opts = uri |> destination_opts() |> add_http2_opts(uri.scheme, Map.get(opts, :tls_opts, [])) + |> add_proxy_auth(opts) - with open_opts <- Map.delete(opts, :tls_opts), + with open_opts <- proxy_open_opts(opts), {:ok, conn} <- Gun.open(proxy_host, proxy_port, open_opts), - {:ok, protocol} <- Gun.await_up(conn, opts[:connect_timeout]), - stream <- Gun.connect(conn, connect_opts), - {:response, :fin, 200, _} <- Gun.await(conn, stream) do - {:ok, conn, protocol} + {:ok, _proxy_protocol} <- await_up(conn, opts[:connect_timeout]) do + conn + |> Gun.connect(connect_opts) + |> await_proxy_connect(conn, opts[:connect_timeout]) else error -> Logger.warning( @@ -64,30 +80,29 @@ defmodule Pleroma.Gun.Conn do end end - defp do_open(uri, %{proxy: {proxy_type, proxy_host, proxy_port}} = opts) do - version = - proxy_type - |> to_string() - |> String.last() - |> case do - "4" -> 4 - _ -> 5 - end + defp do_open(_uri, %{proxy: {:socks4, _proxy_host, _proxy_port}}) do + {:error, :socks4_unsupported} + end + defp do_open(uri, %{proxy: {proxy_type, proxy_host, proxy_port}} = opts) + when proxy_type in [:socks, :socks5] do socks_opts = uri |> destination_opts() |> add_http2_opts(uri.scheme, Map.get(opts, :tls_opts, [])) - |> Map.put(:version, version) + |> Map.put(:version, 5) + |> add_socks_proxy_auth(opts) - opts = + open_opts = opts + |> proxy_open_opts() |> Map.put(:protocols, [:socks]) |> Map.put(:socks_opts, socks_opts) - with {:ok, conn} <- Gun.open(proxy_host, proxy_port, opts), - {:ok, protocol} <- Gun.await_up(conn, opts[:connect_timeout]) do - {:ok, conn, protocol} + with {:ok, conn} <- Gun.open(proxy_host, proxy_port, open_opts), + {:ok, :socks} <- await_up(conn, opts[:connect_timeout]), + {:ok, protocol} <- await_tunnel_up(conn, :undefined, opts[:connect_timeout]) do + {:ok, conn, protocol, nil} else error -> Logger.warning( @@ -100,10 +115,11 @@ defmodule Pleroma.Gun.Conn do defp do_open(%URI{host: host, port: port} = uri, opts) do host = Pleroma.HTTP.AdapterHelper.parse_host(host) + opts = Map.put(opts, :transport, transport(uri.scheme)) with {:ok, conn} <- Gun.open(host, port, opts), - {:ok, protocol} <- Gun.await_up(conn, opts[:connect_timeout]) do - {:ok, conn, protocol} + {:ok, protocol} <- await_up(conn, opts[:connect_timeout]) do + {:ok, conn, protocol, nil} else error -> Logger.warning( @@ -120,11 +136,90 @@ defmodule Pleroma.Gun.Conn do end defp add_http2_opts(opts, "https", tls_opts) do - Map.merge(opts, %{protocols: [:http2], transport: :tls, tls_opts: tls_opts}) + Map.merge(opts, %{protocols: [:http2, :http], transport: :tls, tls_opts: tls_opts}) end defp add_http2_opts(opts, _, _), do: opts + defp add_proxy_auth(connect_opts, %{proxy_auth: {username, password}}) + when is_binary(username) and is_binary(password) do + Map.merge(connect_opts, %{username: username, password: password}) + end + + defp add_proxy_auth(connect_opts, _), do: connect_opts + + defp add_socks_proxy_auth(socks_opts, %{proxy_auth: {username, password}}) + when is_binary(username) and is_binary(password) do + Map.put(socks_opts, :auth, [{:username_password, username, password}]) + end + + defp add_socks_proxy_auth(socks_opts, _), do: socks_opts + + defp proxy_open_opts(opts) do + proxy_tls_opts = Map.get(opts, :proxy_tls_opts, []) + + opts = + opts + |> Map.drop([:proxy, :proxy_auth, :proxy_tls_opts, :tls_opts]) + |> Map.put_new(:transport, :tcp) + + if opts.transport == :tls do + Map.put(opts, :tls_opts, proxy_tls_opts) + else + opts + end + end + + defp await_up(conn, timeout) do + case Gun.await_up(conn, timeout) do + {:ok, protocol} -> + {:ok, protocol} + + error -> + Gun.close(conn) + error + end + end + + defp await_proxy_connect(stream, conn, timeout) do + case Gun.await(conn, stream) do + {:response, :fin, 200, _headers} -> + case await_tunnel_up(conn, stream, timeout) do + {:ok, protocol} -> {:ok, conn, protocol, stream} + error -> error + end + + {:response, _fin, 407, _headers} -> + close_with_error(conn, :proxy_auth_failed) + + {:response, _fin, status, _headers} when is_integer(status) -> + close_with_error(conn, {:proxy_connect_failed, status}) + + error -> + Gun.close(conn) + error + end + end + + defp close_with_error(conn, reason) do + Gun.close(conn) + {:error, reason} + end + + defp transport("https"), do: :tls + defp transport(_), do: :tcp + + defp await_tunnel_up(conn, stream_ref, timeout) do + case Gun.await_tunnel_up(conn, stream_ref, timeout) do + {:ok, protocol} -> + {:ok, protocol} + + error -> + Gun.close(conn) + error + end + end + def compose_uri_log(%URI{scheme: scheme, host: host, path: path}) do "#{scheme}://#{host}#{path}" end diff --git a/lib/pleroma/gun/connection_pool.ex b/lib/pleroma/gun/connection_pool.ex index 2e851de19..5e4fc5c3f 100644 --- a/lib/pleroma/gun/connection_pool.ex +++ b/lib/pleroma/gun/connection_pool.ex @@ -4,53 +4,132 @@ defmodule Pleroma.Gun.ConnectionPool do @registry __MODULE__ + @locks Module.concat(__MODULE__, Locks) alias Pleroma.Gun.ConnectionPool.WorkerSupervisor + @connection_option_keys [ + :http_opts, + :http2_opts, + :protocols, + :proxy, + :proxy_auth, + :proxy_tls_opts, + :retry, + :retry_timeout, + :socks_opts, + :tcp_opts, + :tls_opts, + :transport, + :ws_opts + ] + def children do [ - {Registry, keys: :unique, name: @registry}, + {Registry, keys: :duplicate, name: @registry}, + {Registry, keys: :unique, name: @locks}, Pleroma.Gun.ConnectionPool.WorkerSupervisor ] end @spec get_conn(URI.t(), keyword()) :: {:ok, pid()} | {:error, term()} def get_conn(uri, opts) do - key = "#{uri.scheme}:#{uri.host}:#{uri.port}" + key = connection_key(uri, opts) - case Registry.lookup(@registry, key) do - # The key has already been registered, but connection is not up yet - [{worker_pid, nil}] -> - get_gun_pid_from_worker(worker_pid, true) - - [{worker_pid, {gun_pid, _used_by, _crf, _last_reference}}] -> - GenServer.call(worker_pid, :add_client) + case checkout_existing(Registry.lookup(@registry, key)) do + {:ok, gun_pid} -> {:ok, gun_pid} - [] -> - # :gun.set_owner fails in :connected state for whatevever reason, - # so we open the connection in the process directly and send it's pid back - # We trust gun to handle timeouts by itself - case WorkerSupervisor.start_worker([key, uri, opts, self()]) do - {:ok, worker_pid} -> - get_gun_pid_from_worker(worker_pid, false) + :not_available -> + provision_or_wait(key, uri, opts) + end + end - {:error, {:already_started, worker_pid}} -> - get_gun_pid_from_worker(worker_pid, true) + defp checkout_existing([]), do: :not_available - err -> - err + defp checkout_existing([{_worker_pid, nil} | workers]), do: checkout_existing(workers) + + defp checkout_existing([{worker_pid, _value} | workers]) do + case checkout_worker(worker_pid) do + {:ok, gun_pid} -> {:ok, gun_pid} + _ -> checkout_existing(workers) + end + end + + defp provision_or_wait(key, uri, opts) do + result = with_route_lock(key, fn -> provision(key, uri, opts) end) + + case result do + {:ok, gun_pid} -> + {:ok, gun_pid} + + {:new, worker_pid} -> + get_gun_pid_from_worker(worker_pid) + + {:wait, worker_pid} -> + case checkout_worker(worker_pid, :infinity) do + {:ok, gun_pid} -> {:ok, gun_pid} + :busy -> get_conn(uri, opts) + error -> error + end + + error -> + error + end + end + + defp provision(key, uri, opts) do + workers = Registry.lookup(@registry, key) + + case checkout_existing(workers) do + {:ok, gun_pid} -> + {:ok, gun_pid} + + :not_available -> + case Enum.find(workers, fn {_worker_pid, value} -> is_nil(value) end) do + {worker_pid, nil} -> {:wait, worker_pid} + nil -> start_worker(key, uri, opts) end end end - defp get_gun_pid_from_worker(worker_pid, register) do + defp with_route_lock(key, function) do + case Registry.register(@locks, key, nil) do + {:ok, _} -> + try do + function.() + after + Registry.unregister(@locks, key) + end + + {:error, {:already_registered, _owner}} -> + Process.sleep(1) + with_route_lock(key, function) + end + end + + defp start_worker(key, uri, opts) do + # :gun.set_owner fails in :connected state, so the worker opens the connection itself. + case WorkerSupervisor.start_worker([key, uri, opts, self()]) do + {:ok, worker_pid} -> {:new, worker_pid} + error -> error + end + end + + defp checkout_worker(worker_pid, timeout \\ 5_000) do + try do + GenServer.call(worker_pid, :checkout, timeout) + catch + :exit, reason -> {:error, {:connection_worker_exit, reason}} + end + end + + defp get_gun_pid_from_worker(worker_pid) do # GenServer.call will block the process for timeout length if # the server crashes on startup (which will happen if gun fails to connect) # so instead we use cast + monitor ref = Process.monitor(worker_pid) - if register, do: GenServer.cast(worker_pid, {:add_client, self()}) receive do {:conn_pid, pid} -> @@ -68,19 +147,115 @@ defmodule Pleroma.Gun.ConnectionPool do @spec release_conn(pid()) :: :ok def release_conn(conn_pid) do - # :ets.fun2ms(fn {_, {worker_pid, {gun_pid, _, _, _}}} when gun_pid == conn_pid -> + # :ets.fun2ms(fn {_, {worker_pid, {gun_pid, _tunnel}}} when gun_pid == conn_pid -> # worker_pid end) query_result = Registry.select(@registry, [ - {{:_, :"$1", {:"$2", :_, :_, :_}}, [{:==, :"$2", conn_pid}], [:"$1"]} + {{:_, :"$1", {:"$2", :_}}, [{:==, :"$2", conn_pid}], [:"$1"]} ]) case query_result do [worker_pid] -> - GenServer.call(worker_pid, :remove_client) + try do + GenServer.call(worker_pid, :remove_client) + catch + :exit, _ -> :ok + end [] -> :ok end end + + @spec discard_conn(pid()) :: :ok + def discard_conn(conn_pid) do + case worker_for_conn(conn_pid) do + nil -> + :ok + + worker_pid -> + DynamicSupervisor.terminate_child(WorkerSupervisor, worker_pid) + end + end + + @spec cancel_stream(pid(), reference() | [reference()]) :: :ok + def cancel_stream(conn_pid, stream) do + case worker_for_conn(conn_pid) do + nil -> + Pleroma.Gun.cancel(conn_pid, stream) + + worker_pid -> + try do + GenServer.call(worker_pid, {:cancel_stream, stream}) + catch + :exit, _ -> :ok + end + end + end + + @spec register_stream(pid(), reference() | [reference()]) :: :ok + def register_stream(conn_pid, stream) do + call_worker(conn_pid, {:register_stream, stream}) + end + + @spec finish_stream(pid(), reference() | [reference()]) :: :ok + def finish_stream(conn_pid, stream) do + call_worker(conn_pid, {:finish_stream, stream}) + end + + @spec release_stream(pid(), reference() | [reference()]) :: :ok + def release_stream(conn_pid, stream) do + call_worker(conn_pid, {:release_stream, stream}) + end + + @spec tunnel_ref(pid()) :: reference() | :forward_proxy | nil + def tunnel_ref(conn_pid) do + case Registry.select(@registry, [ + {{:_, :_, {:"$1", :"$2"}}, [{:==, :"$1", conn_pid}], [:"$2"]} + ]) do + [tunnel] -> tunnel + [] -> nil + end + end + + defp worker_for_conn(conn_pid) do + case Registry.select(@registry, [ + {{:_, :"$1", {:"$2", :_}}, [{:==, :"$2", conn_pid}], [:"$1"]} + ]) do + [worker_pid] -> worker_pid + [] -> nil + end + end + + defp call_worker(conn_pid, message) do + case worker_for_conn(conn_pid) do + nil -> + :ok + + worker_pid -> + try do + GenServer.call(worker_pid, message) + catch + :exit, _ -> :ok + end + end + end + + defp connection_key(uri, opts) do + origin = { + String.downcase(uri.scheme), + String.downcase(uri.host), + uri.port + } + + profile = + opts + |> Map.new() + |> Map.take(@connection_option_keys) + |> :erlang.term_to_binary() + |> then(&:crypto.hash(:sha256, &1)) + + fingerprint = Base.encode16(profile, case: :lower) + Enum.join(Tuple.to_list(origin) ++ [fingerprint], ":") + end end diff --git a/lib/pleroma/gun/connection_pool/reclaimer.ex b/lib/pleroma/gun/connection_pool/reclaimer.ex index 3580d38f5..bd7de7e31 100644 --- a/lib/pleroma/gun/connection_pool/reclaimer.ex +++ b/lib/pleroma/gun/connection_pool/reclaimer.ex @@ -9,11 +9,11 @@ defmodule Pleroma.Gun.ConnectionPool.Reclaimer do def start_monitor do pid = - case GenServer.start(__MODULE__, [], name: {:via, Registry, {registry(), "reclaimer"}}) do + case GenServer.start(__MODULE__, [], name: __MODULE__) do {:ok, pid} -> pid - {:error, {:already_registered, pid}} -> + {:error, {:already_started, pid}} -> pid end @@ -41,17 +41,16 @@ defmodule Pleroma.Gun.ConnectionPool.Reclaimer do reclaim_max: reclaim_max }) - # :ets.fun2ms( - # fn {_, {worker_pid, {_, used_by, crf, last_reference}}} when used_by == [] -> - # {worker_pid, crf, last_reference} end) - unused_conns = + workers = Registry.select( registry(), [ - {{:_, :"$1", {:_, :"$2", :"$3", :"$4"}}, [{:==, :"$2", []}], [{{:"$1", :"$3", :"$4"}}]} + {{:_, :"$1", {:_, :_}}, [], [:"$1"]} ] ) + unused_conns = Enum.flat_map(workers, &reclaim_info/1) + case unused_conns do [] -> :telemetry.execute( @@ -65,25 +64,43 @@ defmodule Pleroma.Gun.ConnectionPool.Reclaimer do {:stop, :no_unused_conns, nil} unused_conns -> - reclaimed = + reclaimed_count = unused_conns - |> Enum.sort(fn {_pid1, crf1, last_reference1}, {_pid2, crf2, last_reference2} -> - crf1 <= crf2 and last_reference1 <= last_reference2 - end) - |> Enum.take(reclaim_max) + |> Enum.sort_by(fn {_pid, crf, last_reference} -> {crf, last_reference} end) + |> Enum.reduce_while(0, fn + _worker, count when count == reclaim_max -> + {:halt, count} - reclaimed - |> Enum.each(fn {pid, _, _} -> - DynamicSupervisor.terminate_child(Pleroma.Gun.ConnectionPool.WorkerSupervisor, pid) - end) + {worker_pid, _crf, _last_reference}, count -> + if reclaim(worker_pid), do: {:cont, count + 1}, else: {:cont, count} + end) :telemetry.execute( [:pleroma, :connection_pool, :reclaim, :stop], - %{reclaimed_count: Enum.count(reclaimed)}, + %{reclaimed_count: reclaimed_count}, %{max_connections: max_connections} ) {:stop, :normal, nil} end end + + defp reclaim_info(worker_pid) do + try do + case GenServer.call(worker_pid, :reclaim_info) do + {:idle, crf, last_reference} -> [{worker_pid, crf, last_reference}] + :in_use -> [] + end + catch + :exit, _ -> [] + end + end + + defp reclaim(worker_pid) do + try do + GenServer.call(worker_pid, :reclaim) == :reclaimed + catch + :exit, _ -> false + end + end end diff --git a/lib/pleroma/gun/connection_pool/worker.ex b/lib/pleroma/gun/connection_pool/worker.ex index 38527ec1d..24e067e87 100644 --- a/lib/pleroma/gun/connection_pool/worker.ex +++ b/lib/pleroma/gun/connection_pool/worker.ex @@ -8,25 +8,22 @@ defmodule Pleroma.Gun.ConnectionPool.Worker do defp registry, do: Pleroma.Gun.ConnectionPool - def start_link([key | _] = opts) do - GenServer.start_link(__MODULE__, opts, name: {:via, Registry, {registry(), key}}) - end + def start_link(opts), do: GenServer.start_link(__MODULE__, opts) @impl true - def init([_key, _uri, _opts, _client_pid] = opts) do + def init([key, _uri, _opts, _client_pid] = opts) do + {:ok, _} = Registry.register(registry(), key, nil) {:ok, nil, {:continue, {:connect, opts}}} end @impl true def handle_continue({:connect, [key, uri, opts, client_pid]}, _) do - with {:ok, conn_pid, protocol} <- Gun.Conn.open(uri, opts), + with {:ok, conn_pid, protocol, tunnel} <- Gun.Conn.open(uri, opts), Process.link(conn_pid) do time = :erlang.monotonic_time(:millisecond) - {_, _} = - Registry.update_value(registry(), key, fn _ -> - {conn_pid, [client_pid], 1, time} - end) + Registry.unregister(registry(), key) + {:ok, _} = Registry.register(registry(), key, {conn_pid, tunnel}) send(client_pid, {:conn_pid, conn_pid}) @@ -34,8 +31,14 @@ defmodule Pleroma.Gun.ConnectionPool.Worker do %{ key: key, timer: nil, - client_monitors: %{client_pid => Process.monitor(client_pid)}, - protocol: protocol + clients: %{ + client_pid => %{count: 1, monitor: Process.monitor(client_pid), streams: MapSet.new()} + }, + conn_pid: conn_pid, + protocol: protocol, + proxied: Keyword.has_key?(opts, :proxy), + crf: 1, + last_reference: time }, :hibernate} else err -> @@ -44,106 +47,225 @@ defmodule Pleroma.Gun.ConnectionPool.Worker do end @impl true - def handle_cast({:add_client, client_pid}, state) do - case handle_call(:add_client, {client_pid, nil}, state) do - {:reply, conn_pid, state, :hibernate} -> - send(client_pid, {:conn_pid, conn_pid}) - {:noreply, state, :hibernate} - end + def handle_call(:checkout, _from, %{protocol: protocol, clients: clients} = state) + when protocol != :http2 and map_size(clients) > 0 do + {:reply, :busy, state, :hibernate} end - @impl true - def handle_cast({:remove_client, client_pid}, state) do - case handle_call(:remove_client, {client_pid, nil}, state) do - {:reply, _, state, :hibernate} -> - {:noreply, state, :hibernate} - end - end - - @impl true - def handle_call(:add_client, {client_pid, _}, %{key: key, protocol: protocol} = state) do + def handle_call(:checkout, {client_pid, _}, %{key: key, protocol: protocol} = state) do time = :erlang.monotonic_time(:millisecond) - {{conn_pid, used_by, _, _}, _} = - Registry.update_value(registry(), key, fn {conn_pid, used_by, crf, last_reference} -> - {conn_pid, [client_pid | used_by], crf(time - last_reference, crf), time} - end) + [{_, {conn_pid, _tunnel}}] = + Registry.lookup(registry(), key) |> Enum.filter(&(elem(&1, 0) == self())) + + state = + state + |> cancel_idle_timer() + |> add_client(client_pid) + |> Map.put(:crf, crf(time - state.last_reference, state.crf)) + |> Map.put(:last_reference, time) :telemetry.execute( [:pleroma, :connection_pool, :client, :add], - %{client_pid: client_pid, clients: used_by}, + %{client_pid: client_pid, clients: Map.keys(state.clients)}, %{key: state.key, protocol: protocol} ) - state = - if state.timer != nil do - Process.cancel_timer(state[:timer]) - %{state | timer: nil} - else - state - end - - ref = Process.monitor(client_pid) - - state = put_in(state.client_monitors[client_pid], ref) - {:reply, conn_pid, state, :hibernate} + {:reply, {:ok, conn_pid}, state, :hibernate} end @impl true - def handle_call(:remove_client, {client_pid, _}, %{key: key} = state) do - {{_conn_pid, used_by, _crf, _last_reference}, _} = - Registry.update_value(registry(), key, fn {conn_pid, used_by, crf, last_reference} -> - {conn_pid, List.delete(used_by, client_pid), crf, last_reference} - end) + def handle_call(:remove_client, {client_pid, _}, state) do + case active_streams(state, client_pid) do + [] -> + {:reply, :ok, remove_client(state, client_pid, true), :hibernate} - {ref, state} = pop_in(state.client_monitors[client_pid]) + streams when state.protocol == :http2 -> + Enum.each(streams, &(:ok = Gun.cancel(state.conn_pid, &1))) + {:reply, :ok, remove_client(state, client_pid, true), :hibernate} - Process.demonitor(ref, [:flush]) + _streams -> + {:stop, :normal, :ok, state} + end + end - timer = - if used_by == [] do - max_idle = Pleroma.Config.get([:connections_pool, :max_idle_time], 30_000) - Process.send_after(self(), :idle_close, max_idle) - else - nil - end + def handle_call({:register_stream, stream}, {client_pid, _}, state) do + {:reply, :ok, update_stream(state, client_pid, &MapSet.put(&1, stream)), :hibernate} + end - {:reply, :ok, %{state | timer: timer}, :hibernate} + def handle_call({:finish_stream, stream}, {client_pid, _}, state) do + {:reply, :ok, update_stream(state, client_pid, &MapSet.delete(&1, stream)), :hibernate} + end + + def handle_call({:release_stream, stream}, {client_pid, _}, state) do + state = update_stream(state, client_pid, &MapSet.delete(&1, stream)) + {:reply, :ok, remove_client(state, client_pid, true), :hibernate} + end + + def handle_call({:cancel_stream, stream}, {client_pid, _}, %{protocol: :http2} = state) do + :ok = Gun.cancel(state.conn_pid, stream) + state = update_stream(state, client_pid, &MapSet.delete(&1, stream)) + {:reply, :ok, remove_client(state, client_pid, true), :hibernate} + end + + def handle_call({:cancel_stream, _stream}, _from, state) do + {:stop, :normal, :ok, state} + end + + def handle_call(:reclaim_info, _from, %{clients: clients} = state) + when map_size(clients) == 0 do + {:reply, {:idle, state.crf, state.last_reference}, state, :hibernate} + end + + def handle_call(:reclaim_info, _from, state) do + {:reply, :in_use, state, :hibernate} + end + + def handle_call(:reclaim, _from, %{clients: clients} = state) when map_size(clients) == 0 do + {:stop, :normal, :reclaimed, state} + end + + def handle_call(:reclaim, _from, state) do + {:reply, :in_use, state, :hibernate} end @impl true - def handle_info(:idle_close, state) do - # Gun monitors the owner process, and will close the connection automatically - # when it's terminated + def handle_info({:idle_close, token}, %{timer: {_timer_ref, token}, clients: clients} = state) + when map_size(clients) == 0 do + # Gun monitors the owner process, and will close the connection automatically. {:stop, :normal, state} end + def handle_info({:idle_close, _token}, state), do: {:noreply, state, :hibernate} + @impl true - def handle_info({:gun_up, _pid, _protocol}, state) do - {:noreply, state, :hibernate} + def handle_info({:gun_up, _pid, protocol}, state) do + {:noreply, %{state | protocol: protocol}, :hibernate} end - # Gracefully shutdown if the connection got closed without any streams left + @impl true + def handle_info({:gun_error, conn_pid, stream, _reason}, %{conn_pid: conn_pid} = state) do + {:noreply, delete_stream(state, stream), :hibernate} + end + + @impl true + def handle_info({:gun_down, _pid, _protocol, _reason, _streams}, %{proxied: true} = state) do + {:stop, :normal, state} + end + + # Gracefully shutdown if the connection got closed without any streams left. @impl true def handle_info({:gun_down, _pid, _protocol, _reason, []}, state) do {:stop, :normal, state} end - # Otherwise, wait for retry + # Otherwise, wait for retry. @impl true def handle_info({:gun_down, _pid, _protocol, _reason, _killed_streams}, state) do {:noreply, state, :hibernate} end @impl true - def handle_info({:DOWN, _ref, :process, pid, reason}, state) do - :telemetry.execute( - [:pleroma, :connection_pool, :client, :dead], - %{client_pid: pid, reason: reason}, - %{key: state.key} - ) + def handle_info({:DOWN, ref, :process, pid, reason}, state) do + case state.clients[pid] do + %{monitor: ^ref} -> + :telemetry.execute( + [:pleroma, :connection_pool, :client, :dead], + %{client_pid: pid, reason: reason}, + %{key: state.key} + ) - handle_cast({:remove_client, pid}, state) + handle_dead_client(state, pid) + + _ -> + {:noreply, state, :hibernate} + end + end + + defp add_client(state, client_pid) do + update_in(state.clients, fn clients -> + case clients[client_pid] do + nil -> + Map.put(clients, client_pid, %{ + count: 1, + monitor: Process.monitor(client_pid), + streams: MapSet.new() + }) + + client -> + Map.put(clients, client_pid, %{client | count: client.count + 1}) + end + end) + end + + defp handle_dead_client(state, client_pid) do + streams = active_streams(state, client_pid) + + cond do + streams == [] -> + {:noreply, remove_client(state, client_pid, false, true), :hibernate} + + state.protocol == :http2 -> + Enum.each(streams, &(:ok = Gun.cancel(state.conn_pid, &1))) + {:noreply, remove_client(state, client_pid, false, true), :hibernate} + + true -> + {:stop, :normal, state} + end + end + + defp active_streams(state, client_pid) do + case state.clients[client_pid] do + %{streams: streams} -> MapSet.to_list(streams) + nil -> [] + end + end + + defp update_stream(state, client_pid, update) do + update_in(state.clients, fn clients -> + case clients[client_pid] do + nil -> clients + client -> Map.put(clients, client_pid, %{client | streams: update.(client.streams)}) + end + end) + end + + defp delete_stream(state, stream) do + update_in(state.clients, fn clients -> + Map.new(clients, fn {pid, client} -> + {pid, %{client | streams: MapSet.delete(client.streams, stream)}} + end) + end) + end + + defp remove_client(state, client_pid, demonitor, all \\ false) do + case state.clients[client_pid] do + nil -> + state + + %{count: count} = client when count > 1 and not all -> + put_in(state.clients[client_pid], %{client | count: count - 1}) + + client -> + if demonitor, do: Process.demonitor(client.monitor, [:flush]) + state = update_in(state.clients, &Map.delete(&1, client_pid)) + + if map_size(state.clients) == 0, do: schedule_idle_close(state), else: state + end + end + + defp cancel_idle_timer(%{timer: nil} = state), do: state + + defp cancel_idle_timer(%{timer: {timer_ref, _token}} = state) do + Process.cancel_timer(timer_ref) + %{state | timer: nil} + end + + defp schedule_idle_close(state) do + max_idle = Pleroma.Config.get([:connections_pool, :max_idle_time], 30_000) + token = make_ref() + timer_ref = Process.send_after(self(), {:idle_close, token}, max_idle) + %{state | timer: {timer_ref, token}} end # LRFU policy: https://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.55.1478 diff --git a/lib/pleroma/gun/connection_pool/worker_supervisor.ex b/lib/pleroma/gun/connection_pool/worker_supervisor.ex index b9dedf61e..2c60bea8f 100644 --- a/lib/pleroma/gun/connection_pool/worker_supervisor.ex +++ b/lib/pleroma/gun/connection_pool/worker_supervisor.ex @@ -26,7 +26,14 @@ defmodule Pleroma.Gun.ConnectionPool.WorkerSupervisor do def start_worker(opts, true) do case DynamicSupervisor.start_child(__MODULE__, {Worker, opts}) do {:error, :max_children} -> - :telemetry.execute([:pleroma, :connection_pool, :provision_failure], %{opts: opts}) + [key | _] = opts + + :telemetry.execute( + [:pleroma, :connection_pool, :provision_failure], + %{}, + %{key: key} + ) + {:error, :pool_full} res -> diff --git a/lib/pleroma/http/adapter/gun.ex b/lib/pleroma/http/adapter/gun.ex new file mode 100644 index 000000000..dea1798a1 --- /dev/null +++ b/lib/pleroma/http/adapter/gun.ex @@ -0,0 +1,253 @@ +# Pleroma: A lightweight social networking server +# Copyright ยฉ 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.HTTP.Adapter.Gun do + @behaviour Tesla.Adapter + + alias Pleroma.Gun.ConnectionPool + alias Tesla.Multipart + + @default_timeout 1_000 + + @impl Tesla.Adapter + def call(env, opts) do + adapter_opts = + Tesla.Adapter.opts( + [close_conn: true, body_as: :plain, send_body: :at_once, receive: true], + env, + opts + ) + |> Map.new() + + with {:ok, status, headers, body} <- request(env, adapter_opts) do + env = %{env | status: status, headers: format_headers(headers), body: body} + {:ok, put_stream_ref(env, body)} + end + end + + defp put_stream_ref(env, %{stream: stream}) do + put_in(env.opts[:adapter][:stream], stream) + end + + defp put_stream_ref(env, _body), do: env + + defp request(env, %{conn: conn, tunnel: tunnel} = opts) do + uri = URI.parse(Tesla.build_url(env)) + path = request_path(uri, tunnel) + method = Tesla.Adapter.Shared.format_method(env.method) + {headers, body, send_body} = prepare_body(env.headers, env.body, opts[:send_body]) + headers = maybe_add_forward_proxy_headers(headers, uri, opts, tunnel) + request_opts = request_opts(opts[:reply_to], tunnel) + + stream = + try do + open_stream(conn, method, path, headers, body, request_opts, send_body) + catch + kind, reason -> + ConnectionPool.release_conn(conn) + :erlang.raise(kind, reason, __STACKTRACE__) + end + + :ok = ConnectionPool.register_stream(conn, stream) + :ok = send_stream_body(conn, stream, body, send_body) + response = read_response(conn, stream, opts) + + case response do + {:error, reason} -> + ConnectionPool.cancel_stream(conn, stream) + {:error, {:gun_stream_error, reason}} + + response -> + unless opts[:body_as] in [:chunks, :stream] and stream_body?(response) do + :ok = ConnectionPool.finish_stream(conn, stream) + end + + if opts[:close_conn] and opts[:body_as] not in [:stream, :chunks] do + :gun.close(conn) + end + + response + end + end + + defp stream_body?({:ok, _status, _headers, %{stream: _stream}}), do: true + defp stream_body?({:ok, _status, _headers, %Stream{}}), do: true + defp stream_body?(_response), do: false + + defp request_opts(reply_to, nil), do: %{reply_to: reply_to || self()} + + defp request_opts(reply_to, :forward_proxy), do: %{reply_to: reply_to || self()} + + defp request_opts(reply_to, tunnel) do + %{reply_to: reply_to || self(), tunnel: tunnel} + end + + defp request_path(uri, :forward_proxy) do + path = Tesla.Adapter.Shared.prepare_path(uri.path, uri.query) + "#{uri.scheme}://#{authority(uri)}#{path}" + end + + defp request_path(uri, _tunnel) do + Tesla.Adapter.Shared.prepare_path(uri.path, uri.query) + end + + defp maybe_add_forward_proxy_headers(headers, uri, opts, :forward_proxy) do + headers + |> put_header_new("host", authority(uri)) + |> maybe_put_proxy_authorization(opts) + end + + defp maybe_add_forward_proxy_headers(headers, _uri, _opts, _tunnel), do: headers + + defp maybe_put_proxy_authorization(headers, %{proxy_auth: {username, password}}) + when is_binary(username) and is_binary(password) do + value = "Basic " <> Base.encode64(username <> ":" <> password) + List.keystore(headers, "proxy-authorization", 0, {"proxy-authorization", value}) + end + + defp maybe_put_proxy_authorization(headers, _opts), do: headers + + defp put_header_new(headers, name, value) do + if List.keymember?(headers, name, 0), do: headers, else: [{name, value} | headers] + end + + defp authority(%URI{scheme: scheme, host: host, port: port}) do + host = if String.contains?(host, ":"), do: "[#{host}]", else: host + if port == URI.default_port(scheme), do: host, else: "#{host}:#{port}" + end + + defp prepare_body(headers, %Multipart{} = multipart, _send_body) do + {format_headers(headers ++ Multipart.headers(multipart)), Multipart.body(multipart), :stream} + end + + defp prepare_body(headers, body, _send_body) + when is_struct(body, Stream) or is_function(body) do + {format_headers(headers), body, :stream} + end + + defp prepare_body(headers, body, send_body) do + {format_headers(headers), body || "", send_body} + end + + defp open_stream(conn, method, path, headers, _body, request_opts, :stream) do + :gun.headers(conn, method, path, headers, request_opts) + end + + defp open_stream(conn, method, path, headers, body, request_opts, :at_once) do + :gun.request(conn, method, path, headers, body, request_opts) + end + + defp send_stream_body(conn, stream, body, :stream) do + try do + for data <- body, do: :ok = :gun.data(conn, stream, :nofin, data) + :gun.data(conn, stream, :fin, "") + catch + kind, reason -> + ConnectionPool.cancel_stream(conn, stream) + :erlang.raise(kind, reason, __STACKTRACE__) + end + end + + defp send_stream_body(_conn, _stream, _body, :at_once), do: :ok + + defp read_response(conn, stream, opts) do + receive? = opts[:receive] + + receive do + {:gun_response, ^conn, ^stream, :fin, status, headers} -> + {:ok, status, headers, ""} + + {:gun_response, ^conn, ^stream, :nofin, status, headers} -> + format_response(conn, stream, opts, status, headers, opts[:body_as]) + + {:gun_up, ^conn, _protocol} when receive? -> + read_response(conn, stream, opts) + + {:gun_error, ^conn, reason} -> + {:error, reason} + + {:gun_error, ^conn, ^stream, reason} -> + {:error, reason} + + {:gun_down, ^conn, _protocol, _reason, _killed_streams} when receive? -> + read_response(conn, stream, opts) + + {:DOWN, _ref, :process, ^conn, reason} -> + {:error, reason} + after + opts[:timeout] || @default_timeout -> {:error, :recv_response_timeout} + end + end + + defp format_response(conn, stream, opts, status, headers, :plain) do + case read_body(conn, stream, opts) do + {:ok, body} -> + {:ok, status, headers, body} + + {:error, error} -> + :ok = :gun.flush(stream) + {:error, error} + end + end + + defp format_response(conn, stream, opts, status, headers, :stream) do + body = + Stream.resource( + fn -> :reading end, + fn + :reading -> + case Tesla.Adapter.Gun.read_chunk(conn, stream, opts) do + {:nofin, part} -> {[part], :reading} + {:fin, part} -> {[part], :done} + {:error, reason} -> raise "Gun stream failed: #{inspect(reason)}" + end + + :done -> + {:halt, :done} + end, + fn _state -> + if opts[:close_conn], do: :gun.close(conn) + end + ) + + {:ok, status, headers, body} + end + + defp format_response(conn, stream, opts, status, headers, :chunks) do + {:ok, status, headers, %{pid: conn, stream: stream, opts: Map.to_list(opts)}} + end + + defp read_body(conn, stream, opts, acc \\ "") do + receive do + {:gun_data, ^conn, ^stream, :fin, body} -> + append_body(acc, body, opts[:max_body]) + + {:gun_data, ^conn, ^stream, :nofin, body} -> + with {:ok, acc} <- append_body(acc, body, opts[:max_body]) do + read_body(conn, stream, opts, acc) + end + + {:gun_error, ^conn, ^stream, reason} -> + {:error, reason} + + {:DOWN, _ref, :process, ^conn, reason} -> + {:error, reason} + after + opts[:timeout] || @default_timeout -> {:error, :recv_body_timeout} + end + end + + defp append_body(acc, part, nil), do: {:ok, acc <> part} + + defp append_body(acc, part, limit) do + body = acc <> part + if byte_size(body) <= limit, do: {:ok, body}, else: {:error, :body_too_large} + end + + defp format_headers(headers) do + Enum.map(headers, fn {key, value} -> + {String.downcase(to_string(key)), to_string(value)} + end) + end +end diff --git a/lib/pleroma/http/adapter_helper/gun.ex b/lib/pleroma/http/adapter_helper/gun.ex index 30ba26765..9ac87adb5 100644 --- a/lib/pleroma/http/adapter_helper/gun.ex +++ b/lib/pleroma/http/adapter_helper/gun.ex @@ -48,10 +48,10 @@ defmodule Pleroma.HTTP.AdapterHelper.Gun do Keyword.put(opts, :timeout, recv_timeout) end - # Gun uses [body_as: :stream] + # Keep the pool lease until the caller consumes or cancels the body. defp maybe_stream(opts) do case Keyword.pop(opts, :stream, nil) do - {true, opts} -> Keyword.put(opts, :body_as, :stream) + {true, opts} -> Keyword.put(opts, :body_as, :chunks) {_, opts} -> opts end end diff --git a/lib/pleroma/list.ex b/lib/pleroma/list.ex index cd23bd0a9..3e8a83342 100644 --- a/lib/pleroma/list.ex +++ b/lib/pleroma/list.ex @@ -9,6 +9,7 @@ defmodule Pleroma.List do import Ecto.Changeset alias Pleroma.Activity + alias Pleroma.Emoji alias Pleroma.Repo alias Pleroma.User @@ -18,14 +19,53 @@ defmodule Pleroma.List do field(:following, {:array, :string}, default: []) field(:ap_id, :string) field(:exclusive, :boolean, default: false) + field(:emoji, :string) timestamps() end def update_changeset(list, attrs \\ %{}) do list - |> cast(attrs, [:title, :exclusive]) + |> cast(attrs, [:title, :exclusive, :emoji]) |> validate_required([:title]) + |> fix_emoji() + |> validate_emoji() + end + + defp fix_emoji(changeset) do + with {:emoji_field, emoji} when is_binary(emoji) <- + {:emoji_field, get_field(changeset, :emoji)}, + {:fixed_emoji, emoji} <- + {:fixed_emoji, + emoji + |> Emoji.fully_qualify_emoji() + |> Emoji.maybe_quote()} do + put_change(changeset, :emoji, emoji) + else + {:emoji_field, _} -> changeset + end + end + + defp validate_emoji(changeset) do + validate_change(changeset, :emoji, fn + :emoji, nil -> + [] + + :emoji, emoji -> + if Emoji.unicode?(emoji) or valid_local_custom_emoji?(emoji) do + [] + else + [emoji: "Invalid emoji"] + end + end) + end + + defp valid_local_custom_emoji?(emoji) do + with %{file: _path} <- Emoji.get(emoji) do + true + else + _ -> false + end end def follow_changeset(list, attrs \\ %{}) do diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex index b8693c3a8..afd750823 100644 --- a/lib/pleroma/notification.ex +++ b/lib/pleroma/notification.ex @@ -35,6 +35,7 @@ defmodule Pleroma.Notification do # remember to add a migration to add it to the `notifications_type` enum # as well. field(:type, :string) + field(:group_key, :string) belongs_to(:user, User, type: FlakeId.Ecto.CompatType) belongs_to(:activity, Activity, type: FlakeId.Ecto.CompatType) @@ -49,7 +50,7 @@ defmodule Pleroma.Notification do |> type_from_activity() notification - |> changeset(%{type: type}) + |> changeset(%{type: type, group_key: grouped_notification_key(type, activity)}) |> Repo.update() end end @@ -62,6 +63,99 @@ defmodule Pleroma.Notification do |> Repo.aggregate(:count, :id) end + @groupable_notification_types ~w{favourite follow reblog} + @group_bucket_seconds 12 * 60 * 60 + + def groupable_notification_types, do: @groupable_notification_types + + def group_notifications(notifications, grouped_types \\ nil) do + grouped_types = normalize_grouped_types(grouped_types) + + {group_keys, grouped_notifications} = + Enum.reduce(notifications, {[], %{}}, fn notification, + {group_keys, grouped_notifications} -> + group_key = group_key(notification, grouped_types) + + if Map.has_key?(grouped_notifications, group_key) do + {group_keys, Map.update!(grouped_notifications, group_key, &[notification | &1])} + else + {[group_key | group_keys], Map.put(grouped_notifications, group_key, [notification])} + end + end) + + group_keys + |> Enum.reverse() + |> Enum.map(fn group_key -> + grouped_notifications + |> Map.fetch!(group_key) + |> Enum.reverse() + end) + end + + def group_key(notification, grouped_types \\ nil) + + def group_key(%Notification{} = notification, nil) do + group_key(notification, @groupable_notification_types) + end + + def group_key(%Notification{type: type, group_key: group_key} = notification, grouped_types) do + grouped_types = normalize_grouped_types(grouped_types) + + if type in @groupable_notification_types and type in grouped_types and is_binary(group_key) do + group_key + else + ungrouped_group_key(notification) + end + end + + def normalize_grouped_types(nil), do: @groupable_notification_types + def normalize_grouped_types(types) when is_list(types), do: types + def normalize_grouped_types(type), do: [type] + + defp ungrouped_group_key(%Notification{id: id}), do: "ungrouped-#{id}" + + defp grouped_notification_key(type, activity) when type in @groupable_notification_types do + with target_id when is_binary(target_id) <- group_target_id(type, activity) do + # Mastodon uses Redis to reuse a recent bucket for rolling 12h windows. Pleroma keeps keys + # deterministic and self-contained; clients must treat group_key as opaque either way. + "#{type}-#{target_id}-#{group_time_bucket(activity)}" + end + end + + defp grouped_notification_key(_type, _activity), do: nil + + defp group_time_bucket(%Activity{inserted_at: inserted_at}) when not is_nil(inserted_at) do + inserted_at + |> NaiveDateTime.to_erl() + |> :calendar.datetime_to_gregorian_seconds() + |> div(@group_bucket_seconds) + end + + defp group_time_bucket(_), + do: group_time_bucket(%Activity{inserted_at: NaiveDateTime.utc_now()}) + + defp group_target_id(type, activity) when type in ["favourite", "reblog"] do + with object_id when is_binary(object_id) <- object_id_for(activity), + %Activity{id: id} <- Activity.get_create_by_object_ap_id(object_id) do + to_string(id) + else + _ -> nil + end + end + + defp group_target_id("follow", %{data: %{"object" => ap_id}}) do + case User.get_cached_by_ap_id(ap_id) do + %User{id: id} -> to_string(id) + _ -> nil + end + end + + defp group_target_id(_, _), do: nil + + defp object_id_for(%{data: %{"object" => %{"id" => id}}}) when is_binary(id), do: id + defp object_id_for(%{data: %{"object" => id}}) when is_binary(id), do: id + defp object_id_for(_), do: nil + @notification_types ~w{ favourite follow @@ -79,7 +173,7 @@ defmodule Pleroma.Notification do def changeset(%Notification{} = notification, attrs) do notification - |> cast(attrs, [:seen, :type]) + |> cast(attrs, [:seen, :type, :group_key]) |> validate_inclusion(:type, @notification_types) end @@ -458,7 +552,8 @@ defmodule Pleroma.Notification do user_id: user.id, activity: activity, seen: mark_as_read?(activity, user), - type: type + type: type, + group_key: grouped_notification_key(type, activity) }) |> Marker.multi_set_last_read_id(user, "notifications") |> Repo.transaction() diff --git a/lib/pleroma/reverse_proxy.ex b/lib/pleroma/reverse_proxy.ex index c7ee47c6e..e87c3f625 100644 --- a/lib/pleroma/reverse_proxy.ex +++ b/lib/pleroma/reverse_proxy.ex @@ -17,6 +17,7 @@ defmodule Pleroma.ReverseProxy do @max_read_duration :timer.seconds(30) @max_body_length :infinity @failed_request_ttl :timer.seconds(60) + @sniff_bytes 8 * 1024 @methods ~w(GET HEAD) @allowed_mime_types Pleroma.Config.get([Pleroma.Upload, :allowed_mime_types], []) @@ -59,7 +60,12 @@ defmodule Pleroma.ReverseProxy do * `inline_content_types`: * `true` will not alter `content-disposition` (up to the upstream), * `false` will add `content-disposition: attachment` to any request, - * a list of whitelisted content types + * a list of whitelisted content types for which `content-disposition: inline` + is always set (overriding any upstream header) so the media can be embedded + in pages; the filename is derived from the content type + + * `sniff_content_type` (default `false`): detects an image MIME type from the first + response chunk when the upstream type is missing or `application/octet-stream`. * `req_headers`, `resp_headers` additional headers. @@ -69,11 +75,15 @@ defmodule Pleroma.ReverseProxy do @default_options [pool: :media] @inline_content_types [ + "image/apng", + "image/avif", + "image/bmp", "image/gif", "image/jpeg", "image/jpg", "image/png", "image/svg+xml", + "image/webp", "audio/mpeg", "audio/mp3", "video/webm", @@ -92,6 +102,7 @@ defmodule Pleroma.ReverseProxy do | {:req_headers, [{String.t(), String.t()}]} | {:resp_headers, [{String.t(), String.t()}]} | {:inline_content_types, boolean() | list(String.t())} + | {:sniff_content_type, boolean()} | {:redirect_on_failure, boolean()} @spec call(Plug.Conn.t(), String.t(), list(option())) :: Plug.Conn.t() @@ -110,12 +121,8 @@ defmodule Pleroma.ReverseProxy do end with {:ok, nil} <- @cachex.get(:failed_proxy_url_cache, url), - {:ok, code, headers, client} <- request(method, url, req_headers, client_opts), - :ok <- - header_length_constraint( - headers, - Keyword.get(opts, :max_body_length, @max_body_length) - ) do + {:ok, code, headers, client} <- + request_with_constraints(method, url, req_headers, client_opts, opts) do response(conn, client, url, code, headers, opts) else {:ok, true} -> @@ -170,7 +177,8 @@ defmodule Pleroma.ReverseProxy do {:ok, code, headers} when code in @valid_resp_codes -> {:ok, code, downcase_headers(headers)} - {:ok, code, _, _} -> + {:ok, code, _, client} -> + client().close(client) {:error, {:invalid_http_response, code}} {:ok, code, _} -> @@ -181,15 +189,37 @@ defmodule Pleroma.ReverseProxy do end end + defp request_with_constraints(method, url, headers, client_opts, opts) do + case request(method, url, headers, client_opts) do + {:ok, _code, headers, client} = response -> + case header_length_constraint( + headers, + Keyword.get(opts, :max_body_length, @max_body_length) + ) do + :ok -> + response + + error -> + client().close(client) + error + end + + response -> + response + end + end + defp response(conn, client, url, status, headers, opts) do Logger.debug("#{__MODULE__} #{status} #{url} #{inspect(headers)}") + {headers, client, prefetched} = maybe_prefetch_for_content_type(headers, client, opts) + result = conn |> put_resp_headers(build_resp_headers(headers, opts)) |> streaming_compat |> send_chunked(status) - |> chunk_reply(client, opts) + |> chunk_reply(client, opts, prefetched) case result do {:ok, conn} -> @@ -209,18 +239,28 @@ defmodule Pleroma.ReverseProxy do end end - defp chunk_reply(conn, client, opts) do + defp chunk_reply(conn, client, opts, :none) do chunk_reply(conn, client, opts, 0, 0) end - defp chunk_reply(conn, client, opts, sent_so_far, duration) do - with {:ok, duration} <- - check_read_duration( - duration, - Keyword.get(opts, :max_read_duration, @max_read_duration) + defp chunk_reply(conn, _client, _opts, :done), do: {:ok, conn} + defp chunk_reply(conn, _client, _opts, {:error, error}), do: {:error, error, conn} + + defp chunk_reply(conn, client, opts, {:ok, data, duration}) do + with :ok <- + body_size_constraint( + byte_size(data), + Keyword.get(opts, :max_body_length, @max_body_length) ), - {:ok, data, client} <- client().stream_body(client), - {:ok, duration} <- increase_read_duration(duration), + {:ok, conn} <- chunk(conn, data) do + chunk_reply(conn, client, opts, byte_size(data), duration) + else + {:error, error} -> {:error, error, conn} + end + end + + defp chunk_reply(conn, client, opts, sent_so_far, duration) do + with {:ok, data, client, duration} <- read_chunk(client, duration, opts), sent_so_far = sent_so_far + byte_size(data), :ok <- body_size_constraint( @@ -235,6 +275,66 @@ defmodule Pleroma.ReverseProxy do end end + defp read_chunk(client, duration, opts) do + with {:ok, timer} <- + check_read_duration( + duration, + Keyword.get(opts, :max_read_duration, @max_read_duration) + ), + {:ok, data, client} <- client().stream_body(client), + {:ok, duration} <- increase_read_duration(timer) do + {:ok, data, client, duration} + end + end + + defp maybe_prefetch_for_content_type(headers, client, opts) do + if Keyword.get(opts, :sniff_content_type, false) and generic_content_type?(headers) do + case read_chunk(client, 0, opts) do + {:ok, data, client, duration} -> + {maybe_put_image_content_type(headers, data), client, {:ok, data, duration}} + + :done -> + {headers, client, :done} + + {:error, error} -> + {headers, client, {:error, error}} + end + else + {headers, client, :none} + end + end + + defp generic_content_type?(headers) do + headers + |> get_content_type() + |> String.trim() + |> String.downcase() + |> then(&(&1 in ["", "application/octet-stream"])) + end + + defp maybe_put_image_content_type(headers, data) do + with false <- data == "", + {:ok, %{mime_type: "image/" <> _ = content_type}} <- + Majic.perform({:bytes, binary_part(data, 0, min(byte_size(data), @sniff_bytes))}, + pool: Pleroma.MajicPool + ) do + [ + {"content-type", content_type} + | Enum.reject(headers, fn {key, _} -> key == "content-type" end) + ] + else + _ -> headers + end + rescue + error -> + Logger.debug("#{__MODULE__}: content-type sniffing failed: #{Exception.message(error)}") + headers + catch + kind, reason -> + Logger.debug("#{__MODULE__}: content-type sniffing failed: #{kind}: #{inspect(reason)}") + headers + end + defp head_response(conn, url, code, headers, opts) do Logger.debug("#{__MODULE__} #{code} #{url} #{inspect(headers)}") @@ -281,17 +381,34 @@ defmodule Pleroma.ReverseProxy do |> Enum.filter(fn {k, _} -> k in @keep_req_headers end) |> build_req_range_or_encoding_header(opts) |> build_req_user_agent_header(opts) - |> Keyword.merge(Keyword.get(opts, :req_headers, [])) + |> merge_headers(Keyword.get(opts, :req_headers, [])) + |> maybe_force_identity_encoding(opts) + end + + defp merge_headers(headers, extra_headers) do + Enum.reduce(extra_headers, headers, fn {key, value}, headers -> + List.keystore(headers, String.downcase(key), 0, {String.downcase(key), value}) + end) + end + + defp maybe_force_identity_encoding(headers, opts) do + if Keyword.get(opts, :sniff_content_type, false) do + List.keystore(headers, "accept-encoding", 0, {"accept-encoding", "identity"}) + else + headers + end end # Disable content-encoding if any @range_headers are requested (see #1823). defp build_req_range_or_encoding_header(headers, _opts) do range? = Enum.any?(headers, fn {header, _} -> Enum.member?(@range_headers, header) end) - if range? && List.keymember?(headers, "accept-encoding", 0) do - List.keydelete(headers, "accept-encoding", 0) - else - headers + cond do + range? && List.keymember?(headers, "accept-encoding", 0) -> + List.keydelete(headers, "accept-encoding", 0) + + true -> + headers end end @@ -384,9 +501,33 @@ defmodule Pleroma.ReverseProxy do disposition = "attachment; filename=\"#{name}\"" - List.keystore(headers, "content-disposition", 0, {"content-disposition", disposition}) + replace_header(headers, "content-disposition", disposition) else - headers + if opt == true do + headers + else + name = inline_filename(content_type) + + disposition = + if name do + "inline; filename=\"#{name}\"" + else + "inline" + end + + replace_header(headers, "content-disposition", disposition) + end + end + end + + defp replace_header(headers, key, value) do + [{key, value} | Enum.reject(headers, fn {header, _} -> header == key end)] + end + + defp inline_filename(content_type) do + case MIME.extensions(content_type) do + [ext | _] when ext != "" -> "inline.#{ext}" + _ -> nil end end @@ -406,7 +547,7 @@ defmodule Pleroma.ReverseProxy do defp header_length_constraint(_, _), do: :ok - defp body_size_constraint(size, limit) when is_integer(limit) and limit > 0 and size >= limit do + defp body_size_constraint(size, limit) when is_integer(limit) and limit > 0 and size > limit do {:error, :body_too_large} end @@ -421,7 +562,9 @@ defmodule Pleroma.ReverseProxy do end end - defp check_read_duration(_, _), do: {:ok, :no_duration_limit, :no_duration_limit} + defp check_read_duration(_, _), do: {:ok, :no_duration_limit} + + defp increase_read_duration(:no_duration_limit), do: {:ok, :no_duration_limit} defp increase_read_duration({previous_duration, started}) when is_integer(previous_duration) and is_integer(started) do diff --git a/lib/pleroma/reverse_proxy/client/tesla.ex b/lib/pleroma/reverse_proxy/client/tesla.ex index 4596d7a7f..941d23ea2 100644 --- a/lib/pleroma/reverse_proxy/client/tesla.ex +++ b/lib/pleroma/reverse_proxy/client/tesla.ex @@ -6,6 +6,7 @@ defmodule Pleroma.ReverseProxy.Client.Tesla do @behaviour Pleroma.ReverseProxy.Client alias Pleroma.Gun.ConnectionPool + alias Pleroma.HTTP.AdapterHelper @type headers() :: [{String.t(), String.t()}] @type status() :: pos_integer() @@ -20,6 +21,13 @@ defmodule Pleroma.ReverseProxy.Client.Tesla do def request(method, url, headers, body, opts \\ []) do check_adapter() + # MediaProxyController inserts MediaProxy proxy configuration into the ReverseProxy call. + # This config can be unformatted and in String representation, while this works for Hackney, + # it does not work for Tesla and the proxy needs to be reformatted. + # Gun AdapterHelper doesn't do it, since it does not overwrite proxy config. + proxy = AdapterHelper.format_proxy(opts[:proxy]) + opts = if proxy, do: Keyword.put(opts, :proxy, proxy), else: opts + opts = Keyword.put(opts, :body_as, :chunks) with {:ok, response} <- @@ -45,8 +53,8 @@ defmodule Pleroma.ReverseProxy.Client.Tesla do @impl true @spec stream_body(map()) :: {:ok, binary(), map()} | {:error, atom() | String.t()} | :done | no_return() - def stream_body(%{pid: pid, fin: true}) do - ConnectionPool.release_conn(pid) + def stream_body(%{pid: pid, stream: stream, fin: true}) do + ConnectionPool.release_stream(pid, stream) :done end @@ -70,6 +78,10 @@ defmodule Pleroma.ReverseProxy.Client.Tesla do @impl true @spec close(map) :: :ok | no_return() + def close(%{pid: pid, stream: stream}) do + ConnectionPool.cancel_stream(pid, stream) + end + def close(%{pid: pid}) do ConnectionPool.release_conn(pid) end diff --git a/lib/pleroma/telemetry/logger.ex b/lib/pleroma/telemetry/logger.ex index 31ce3cc20..e0522e869 100644 --- a/lib/pleroma/telemetry/logger.ex +++ b/lib/pleroma/telemetry/logger.ex @@ -55,8 +55,8 @@ defmodule Pleroma.Telemetry.Logger do def handle_event( [:pleroma, :connection_pool, :provision_failure], - %{opts: [key | _]}, _, + %{key: key}, _ ) do Logger.debug(fn -> diff --git a/lib/pleroma/tesla/middleware/connection_pool.ex b/lib/pleroma/tesla/middleware/connection_pool.ex index de74270f8..c2d8c5bb8 100644 --- a/lib/pleroma/tesla/middleware/connection_pool.ex +++ b/lib/pleroma/tesla/middleware/connection_pool.ex @@ -12,25 +12,38 @@ defmodule Pleroma.Tesla.Middleware.ConnectionPool do alias Pleroma.Gun.ConnectionPool @impl Tesla.Middleware - def call(%Tesla.Env{url: url, opts: opts} = env, next, _) do + def call(%Tesla.Env{opts: opts} = env, next, middleware_opts) do + if opts[:adapter][:body_as] == :stream do + {:error, :pooled_streaming_requires_chunks} + else + do_call(env, next, middleware_opts) + end + end + + defp do_call(%Tesla.Env{url: url, opts: opts} = env, next, _) do uri = URI.parse(url) # Avoid leaking connections when the middleware is called twice # with body_as: :chunks. We assume only the middleware can set # opts[:adapter][:conn] - if opts[:adapter][:conn] do - ConnectionPool.release_conn(opts[:adapter][:conn]) - end + cleanup_previous_request(env) case ConnectionPool.get_conn(uri, opts[:adapter]) do {:ok, conn_pid} -> - adapter_opts = Keyword.merge(opts[:adapter], conn: conn_pid, close_conn: false) + tunnel = ConnectionPool.tunnel_ref(conn_pid) + + adapter_opts = + opts[:adapter] + |> Keyword.drop([:conn, :stream, :tunnel]) + |> Keyword.merge(conn: conn_pid, close_conn: false, tunnel: tunnel) + opts = Keyword.put(opts, :adapter, adapter_opts) env = %{env | opts: opts} + next = use_pool_adapter(next) case Tesla.run(env, next) do {:ok, env} -> - unless opts[:adapter][:body_as] == :chunks do + unless opts[:adapter][:body_as] == :chunks and chunk_client?(env.body) do ConnectionPool.release_conn(conn_pid) {_, res} = pop_in(env.opts[:adapter][:conn]) {:ok, res} @@ -38,8 +51,11 @@ defmodule Pleroma.Tesla.Middleware.ConnectionPool do {:ok, env} end + {:error, {:gun_stream_error, reason}} -> + {:error, reason} + err -> - ConnectionPool.release_conn(conn_pid) + ConnectionPool.discard_conn(conn_pid) err end @@ -47,4 +63,34 @@ defmodule Pleroma.Tesla.Middleware.ConnectionPool do err end end + + defp cleanup_previous_request(%Tesla.Env{ + body: %{pid: pid, stream: stream}, + opts: opts + }) do + if opts[:adapter][:conn], do: ConnectionPool.cancel_stream(pid, stream) + end + + defp cleanup_previous_request(%Tesla.Env{opts: opts}) do + conn = opts[:adapter][:conn] + stream = opts[:adapter][:stream] + + cond do + conn && stream -> ConnectionPool.cancel_stream(conn, stream) + conn -> ConnectionPool.release_conn(conn) + true -> :ok + end + end + + defp chunk_client?(%{pid: pid, stream: stream}) when is_pid(pid) and not is_nil(stream), + do: true + + defp chunk_client?(_body), do: false + + defp use_pool_adapter(next) do + List.update_at(next, -1, fn + {Tesla.Adapter.Gun, function, args} -> {Pleroma.HTTP.Adapter.Gun, function, args} + adapter -> adapter + end) + end end diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 75da41da9..768a6f3c2 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -272,13 +272,13 @@ defmodule Pleroma.User do end def cached_blocked_users_ap_ids(user) do - @cachex.fetch!(:user_cache, "blocked_users_ap_ids:#{user.ap_id}", fn _ -> + @cachex.fetch!(:user_cache, cache_key(:blocked_users_ap_ids, user), fn _ -> blocked_users_ap_ids(user) end) end def cached_muted_users_ap_ids(user) do - @cachex.fetch!(:user_cache, "muted_users_ap_ids:#{user.ap_id}", fn _ -> + @cachex.fetch!(:user_cache, cache_key(:muted_users_ap_ids, user), fn _ -> muted_users_ap_ids(user) end) end @@ -1253,13 +1253,18 @@ defmodule Pleroma.User do get_cached_by_nickname(nickname) end + defp cache_key(:id, %User{id: id}), do: cache_key(:id, id) + defp cache_key(:nickname, %User{nickname: nickname}), do: cache_key(:nickname, nickname) + defp cache_key(namespace, %User{ap_id: ap_id}), do: cache_key(namespace, ap_id) + defp cache_key(namespace, value), do: "#{namespace}:#{value}" + def set_cache({:ok, user}), do: set_cache(user) def set_cache({:error, err}), do: {:error, err} def set_cache(%User{} = user) do - @cachex.put(:user_cache, "ap_id:#{user.ap_id}", user) - @cachex.put(:user_cache, "nickname:#{user.nickname}", user) - @cachex.put(:user_cache, "friends_ap_ids:#{user.nickname}", get_user_friends_ap_ids(user)) + @cachex.put(:user_cache, cache_key(:ap_id, user), user) + @cachex.put(:user_cache, cache_key(:nickname, user), user) + @cachex.put(:user_cache, cache_key(:friends_ap_ids, user), get_user_friends_ap_ids(user)) {:ok, user} end @@ -1287,22 +1292,23 @@ defmodule Pleroma.User do @spec get_cached_user_friends_ap_ids(User.t()) :: [String.t()] def get_cached_user_friends_ap_ids(user) do - @cachex.fetch!(:user_cache, "friends_ap_ids:#{user.ap_id}", fn _ -> + @cachex.fetch!(:user_cache, cache_key(:friends_ap_ids, user), fn _ -> get_user_friends_ap_ids(user) end) end def invalidate_cache(user) do - @cachex.del(:user_cache, "ap_id:#{user.ap_id}") - @cachex.del(:user_cache, "nickname:#{user.nickname}") - @cachex.del(:user_cache, "friends_ap_ids:#{user.ap_id}") - @cachex.del(:user_cache, "blocked_users_ap_ids:#{user.ap_id}") - @cachex.del(:user_cache, "muted_users_ap_ids:#{user.ap_id}") + @cachex.del(:user_cache, cache_key(:ap_id, user)) + @cachex.del(:user_cache, cache_key(:nickname, user)) + @cachex.del(:user_cache, cache_key(:id, user)) + @cachex.del(:user_cache, cache_key(:friends_ap_ids, user)) + @cachex.del(:user_cache, cache_key(:blocked_users_ap_ids, user)) + @cachex.del(:user_cache, cache_key(:muted_users_ap_ids, user)) end @spec get_cached_by_ap_id(String.t()) :: User.t() | nil def get_cached_by_ap_id(ap_id) do - key = "ap_id:#{ap_id}" + key = cache_key(:ap_id, ap_id) with {:ok, nil} <- @cachex.get(:user_cache, key), user when not is_nil(user) <- get_by_ap_id(ap_id), @@ -1315,14 +1321,14 @@ defmodule Pleroma.User do end def get_cached_by_id(id) do - key = "id:#{id}" + key = cache_key(:id, id) ap_id = @cachex.fetch!(:user_cache, key, fn _ -> user = get_by_id(id) if user do - @cachex.put(:user_cache, "ap_id:#{user.ap_id}", user) + @cachex.put(:user_cache, cache_key(:ap_id, user), user) {:commit, user.ap_id} else {:ignore, ""} @@ -1333,7 +1339,7 @@ defmodule Pleroma.User do end def get_cached_by_nickname(nickname) do - key = "nickname:#{nickname}" + key = cache_key(:nickname, nickname) @cachex.fetch!(:user_cache, key, fn _ -> case get_or_fetch_by_nickname(nickname) do @@ -1651,7 +1657,7 @@ defmodule Pleroma.User do |> Oban.insert() end - @cachex.del(:user_cache, "muted_users_ap_ids:#{muter.ap_id}") + @cachex.del(:user_cache, cache_key(:muted_users_ap_ids, muter)) {:ok, Enum.filter([user_mute, user_notification_mute], & &1)} end @@ -1661,7 +1667,7 @@ defmodule Pleroma.User do with {:ok, user_mute} <- UserRelationship.delete_mute(muter, mutee), {:ok, user_notification_mute} <- UserRelationship.delete_notification_mute(muter, mutee) do - @cachex.del(:user_cache, "muted_users_ap_ids:#{muter.ap_id}") + @cachex.del(:user_cache, cache_key(:muted_users_ap_ids, muter)) {:ok, [user_mute, user_notification_mute]} end end @@ -2815,7 +2821,7 @@ defmodule Pleroma.User do {:ok, UserRelationship.t()} | {:error, Ecto.Changeset.t()} defp add_to_block(%User{} = user, %User{} = blocked, expires_at) do with {:ok, relationship} <- UserRelationship.create_block(user, blocked, expires_at) do - @cachex.del(:user_cache, "blocked_users_ap_ids:#{user.ap_id}") + @cachex.del(:user_cache, cache_key(:blocked_users_ap_ids, user)) {:ok, relationship} end end @@ -2824,7 +2830,7 @@ defmodule Pleroma.User do {:ok, UserRelationship.t()} | {:ok, nil} | {:error, Ecto.Changeset.t()} defp remove_from_block(%User{} = user, %User{} = blocked) do with {:ok, relationship} <- UserRelationship.delete_block(user, blocked) do - @cachex.del(:user_cache, "blocked_users_ap_ids:#{user.ap_id}") + @cachex.del(:user_cache, cache_key(:blocked_users_ap_ids, user)) {:ok, relationship} end end diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 1e8bfe3dd..10d3fe063 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -537,9 +537,12 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do # and extra sorting on "activities.id DESC NULLS LAST" would worse the query plan opts = Map.put(opts, :skip_extra_order, true) - Pagination.fetch_paginated(query, opts, pagination) + Pagination.fetch_paginated(query, opts, pagination, pagination_binding(opts)) end + defp pagination_binding(%{favorited_by: _}), do: :favorited_activity + defp pagination_binding(_), do: nil + def fetch_activities(recipients, opts \\ %{}, pagination \\ :keyset) do list_memberships = Pleroma.List.memberships(opts[:user]) @@ -786,6 +789,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do defp restrict_since(query, %{since_id: ""}), do: query + defp restrict_since(query, %{favorited_by: _, since_id: _}), do: query + defp restrict_since(query, %{since_id: since_id}) do from(activity in query, where: activity.id > ^since_id) end @@ -893,14 +898,22 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do |> Repo.all() # Note: NO extra ordering should be done on "activities.id desc nulls last" for optimal plan - from( - [_activity, object] in query, - join: hto in "hashtags_objects", - on: hto.object_id == object.id, - where: hto.hashtag_id in ^hashtag_ids, - distinct: [desc: object.id], - order_by: [desc: object.id] - ) + query = + from( + [_activity, object] in query, + join: hto in "hashtags_objects", + on: hto.object_id == object.id, + where: hto.hashtag_id in ^hashtag_ids + ) + + if is_nil(query.distinct) do + from([_activity, object] in query, + distinct: [desc: object.id], + order_by: [desc: object.id] + ) + else + query + end end defp restrict_hashtag_any(query, %{tag: tag}) when is_binary(tag) do @@ -1012,9 +1025,31 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do defp restrict_assigned_account(query, _), do: query defp restrict_favorited_by(query, %{favorited_by: ap_id}) do + newer_favorite = + from(newer_favorite in Activity, + where: newer_favorite.actor == ^ap_id, + where: newer_favorite.id > parent_as(:favorited_activity).id, + where: fragment("?->>'type' = ?", newer_favorite.data, "Like"), + where: + fragment( + "associated_object_id(?) = associated_object_id(?)", + newer_favorite.data, + parent_as(:favorited_activity).data + ), + select: 1 + ) + from( - [_activity, object] in query, - where: fragment("(?)->'likes' \\? (?)", object.data, ^ap_id) + [activity, object: object] in query, + join: favorited_activity in Activity, + as: :favorited_activity, + on: + favorited_activity.actor == ^ap_id and + fragment("?->>'type' = ?", favorited_activity.data, "Like") and + fragment("associated_object_id(?) = (?)->>'id'", favorited_activity.data, object.data), + where: fragment("(?)->'likes' \\? (?)", object.data, ^ap_id), + where: not exists(subquery(newer_favorite)), + select: %Activity{activity | pagination_id: favorited_activity.id} ) end diff --git a/lib/pleroma/web/activity_pub/mrf/activity_expiration_policy.ex b/lib/pleroma/web/activity_pub/mrf/activity_expiration_policy.ex index 88f6ca028..41abe17c0 100644 --- a/lib/pleroma/web/activity_pub/mrf/activity_expiration_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/activity_expiration_policy.ex @@ -6,6 +6,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.ActivityExpirationPolicy do @moduledoc "Adds expiration to all local Create activities" @behaviour Pleroma.Web.ActivityPub.MRF.Policy + use Pleroma.Web.ActivityPub.MRF.Policy + @impl true def filter(activity) do activity = diff --git a/lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex b/lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex index 8ea61aec2..990ff31c9 100644 --- a/lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex @@ -3,11 +3,11 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.ActivityPub.MRF.AntiFollowbotPolicy do + @moduledoc "Prevent followbots from following with a bit of heuristic" + @behaviour Pleroma.Web.ActivityPub.MRF.Policy alias Pleroma.User - @moduledoc "Prevent followbots from following with a bit of heuristic" - - @behaviour Pleroma.Web.ActivityPub.MRF.Policy + use Pleroma.Web.ActivityPub.MRF.Policy # XXX: this should become User.normalize_by_ap_id() or similar, really. defp normalize_by_ap_id(%{"id" => id}), do: User.get_cached_by_ap_id(id) diff --git a/lib/pleroma/web/activity_pub/mrf/anti_link_spam_policy.ex b/lib/pleroma/web/activity_pub/mrf/anti_link_spam_policy.ex index 2be6d8df4..5c3b66e37 100644 --- a/lib/pleroma/web/activity_pub/mrf/anti_link_spam_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/anti_link_spam_policy.ex @@ -3,9 +3,11 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.ActivityPub.MRF.AntiLinkSpamPolicy do + @behaviour Pleroma.Web.ActivityPub.MRF.Policy + alias Pleroma.User - @behaviour Pleroma.Web.ActivityPub.MRF.Policy + use Pleroma.Web.ActivityPub.MRF.Policy require Logger diff --git a/lib/pleroma/web/activity_pub/mrf/anti_mention_spam_policy.ex b/lib/pleroma/web/activity_pub/mrf/anti_mention_spam_policy.ex index 1d76a307b..c6f5781b3 100644 --- a/lib/pleroma/web/activity_pub/mrf/anti_mention_spam_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/anti_mention_spam_policy.ex @@ -3,11 +3,13 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.ActivityPub.MRF.AntiMentionSpamPolicy do + @behaviour Pleroma.Web.ActivityPub.MRF.Policy + alias Pleroma.Config alias Pleroma.User require Pleroma.Constants - @behaviour Pleroma.Web.ActivityPub.MRF.Policy + use Pleroma.Web.ActivityPub.MRF.Policy defp user_has_posted?(%User{} = u), do: u.note_count > 0 diff --git a/lib/pleroma/web/activity_pub/mrf/drop_policy.ex b/lib/pleroma/web/activity_pub/mrf/drop_policy.ex index cf07db7f3..4aa1968f7 100644 --- a/lib/pleroma/web/activity_pub/mrf/drop_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/drop_policy.ex @@ -3,10 +3,12 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.ActivityPub.MRF.DropPolicy do - require Logger @moduledoc "Drop and log everything received" @behaviour Pleroma.Web.ActivityPub.MRF.Policy + require Logger + use Pleroma.Web.ActivityPub.MRF.Policy + @impl true def filter(activity) do Logger.debug("REJECTING #{inspect(activity)}") diff --git a/lib/pleroma/web/activity_pub/mrf/emoji_policy.ex b/lib/pleroma/web/activity_pub/mrf/emoji_policy.ex index 1de5280d9..cc5fd49bf 100644 --- a/lib/pleroma/web/activity_pub/mrf/emoji_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/emoji_policy.ex @@ -3,6 +3,8 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.ActivityPub.MRF.EmojiPolicy do + @behaviour Pleroma.Web.ActivityPub.MRF.Policy + require Pleroma.Constants alias Pleroma.Object.Updater @@ -10,7 +12,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.EmojiPolicy do @moduledoc "Reject or force-unlisted emojis with certain URLs or names" - @behaviour Pleroma.Web.ActivityPub.MRF.Policy + use Pleroma.Web.ActivityPub.MRF.Policy defp config_remove_url do Pleroma.Config.get([:mrf_emoji, :remove_url], []) diff --git a/lib/pleroma/web/activity_pub/mrf/ensure_re_prepended.ex b/lib/pleroma/web/activity_pub/mrf/ensure_re_prepended.ex index f5983c8a7..6e43e746c 100644 --- a/lib/pleroma/web/activity_pub/mrf/ensure_re_prepended.ex +++ b/lib/pleroma/web/activity_pub/mrf/ensure_re_prepended.ex @@ -3,11 +3,13 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.ActivityPub.MRF.EnsureRePrepended do - alias Pleroma.Object - @moduledoc "Ensure a re: is prepended on replies to a post with a Subject" @behaviour Pleroma.Web.ActivityPub.MRF.Policy + alias Pleroma.Object + + use Pleroma.Web.ActivityPub.MRF.Policy + @reply_prefix Regex.compile!("^re:[[:space:]]*", [:caseless]) def history_awareness, do: :auto diff --git a/lib/pleroma/web/activity_pub/mrf/follow_bot_policy.ex b/lib/pleroma/web/activity_pub/mrf/follow_bot_policy.ex index 480a03ef6..4612b2cc5 100644 --- a/lib/pleroma/web/activity_pub/mrf/follow_bot_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/follow_bot_policy.ex @@ -4,12 +4,15 @@ defmodule Pleroma.Web.ActivityPub.MRF.FollowBotPolicy do @behaviour Pleroma.Web.ActivityPub.MRF.Policy + alias Pleroma.Config alias Pleroma.User alias Pleroma.Web.CommonAPI require Logger + use Pleroma.Web.ActivityPub.MRF.Policy + @impl true def filter(activity) do with follower_nickname <- Config.get([:mrf_follow_bot, :follower_nickname]), diff --git a/lib/pleroma/web/activity_pub/mrf/force_bot_unlisted_policy.ex b/lib/pleroma/web/activity_pub/mrf/force_bot_unlisted_policy.ex index 3b3251dc3..58d02f433 100644 --- a/lib/pleroma/web/activity_pub/mrf/force_bot_unlisted_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/force_bot_unlisted_policy.ex @@ -3,9 +3,12 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.ActivityPub.MRF.ForceBotUnlistedPolicy do - alias Pleroma.User - @behaviour Pleroma.Web.ActivityPub.MRF.Policy @moduledoc "Remove bot posts from federated timeline" + @behaviour Pleroma.Web.ActivityPub.MRF.Policy + + alias Pleroma.User + + use Pleroma.Web.ActivityPub.MRF.Policy require Pleroma.Constants diff --git a/lib/pleroma/web/activity_pub/mrf/force_mention.ex b/lib/pleroma/web/activity_pub/mrf/force_mention.ex index 4ea23540d..62aff36bc 100644 --- a/lib/pleroma/web/activity_pub/mrf/force_mention.ex +++ b/lib/pleroma/web/activity_pub/mrf/force_mention.ex @@ -3,13 +3,14 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.ActivityPub.MRF.ForceMention do + @behaviour Pleroma.Web.ActivityPub.MRF.Policy require Pleroma.Constants alias Pleroma.Config alias Pleroma.Object alias Pleroma.User - @behaviour Pleroma.Web.ActivityPub.MRF.Policy + use Pleroma.Web.ActivityPub.MRF.Policy defp get_author(url) do with %Object{data: %{"actor" => actor}} <- Object.normalize(url, fetch: false), diff --git a/lib/pleroma/web/activity_pub/mrf/force_mentions_in_content.ex b/lib/pleroma/web/activity_pub/mrf/force_mentions_in_content.ex index caae365e5..e7c3f2d88 100644 --- a/lib/pleroma/web/activity_pub/mrf/force_mentions_in_content.ex +++ b/lib/pleroma/web/activity_pub/mrf/force_mentions_in_content.ex @@ -3,13 +3,15 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.ActivityPub.MRF.ForceMentionsInContent do + @behaviour Pleroma.Web.ActivityPub.MRF.Policy + require Pleroma.Constants alias Pleroma.Formatter alias Pleroma.Object alias Pleroma.User - @behaviour Pleroma.Web.ActivityPub.MRF.Policy + use Pleroma.Web.ActivityPub.MRF.Policy @impl true def history_awareness, do: :auto diff --git a/lib/pleroma/web/activity_pub/mrf/hashtag_policy.ex b/lib/pleroma/web/activity_pub/mrf/hashtag_policy.ex index 72f2274ed..a3fae3bae 100644 --- a/lib/pleroma/web/activity_pub/mrf/hashtag_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/hashtag_policy.ex @@ -3,6 +3,8 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.ActivityPub.MRF.HashtagPolicy do + @behaviour Pleroma.Web.ActivityPub.MRF.Policy + require Pleroma.Constants alias Pleroma.Config @@ -14,7 +16,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.HashtagPolicy do Note: This MRF Policy is always enabled, if you want to disable it you have to set empty lists. """ - @behaviour Pleroma.Web.ActivityPub.MRF.Policy + use Pleroma.Web.ActivityPub.MRF.Policy @impl true def history_awareness, do: :manual diff --git a/lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex b/lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex index 3a80d0a69..a84a7b0c8 100644 --- a/lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex @@ -3,13 +3,14 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.ActivityPub.MRF.HellthreadPolicy do + @moduledoc "Block activities with too much mentions (configurable)" + @behaviour Pleroma.Web.ActivityPub.MRF.Policy + alias Pleroma.User require Pleroma.Constants - @moduledoc "Block activities with too much mentions (configurable)" - - @behaviour Pleroma.Web.ActivityPub.MRF.Policy + use Pleroma.Web.ActivityPub.MRF.Policy defp delist_activity(activity, threshold) when threshold > 0 do follower_collection = User.get_cached_by_ap_id(activity["actor"]).follower_address diff --git a/lib/pleroma/web/activity_pub/mrf/inline_quote_policy.ex b/lib/pleroma/web/activity_pub/mrf/inline_quote_policy.ex index 51dcd3918..17879b4bd 100644 --- a/lib/pleroma/web/activity_pub/mrf/inline_quote_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/inline_quote_policy.ex @@ -6,6 +6,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.InlineQuotePolicy do @moduledoc "Force a quote line into the message content." @behaviour Pleroma.Web.ActivityPub.MRF.Policy + use Pleroma.Web.ActivityPub.MRF.Policy + defp build_inline_quote(template, url) do quote_line = String.replace(template, "{url}", "#{url}") diff --git a/lib/pleroma/web/activity_pub/mrf/keyword_policy.ex b/lib/pleroma/web/activity_pub/mrf/keyword_policy.ex index 6ba6fd509..33f32e781 100644 --- a/lib/pleroma/web/activity_pub/mrf/keyword_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/keyword_policy.ex @@ -3,13 +3,14 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.ActivityPub.MRF.KeywordPolicy do + @moduledoc "Reject or Word-Replace activities with a keyword or regex" + @behaviour Pleroma.Web.ActivityPub.MRF.Policy + require Pleroma.Constants alias Pleroma.Web.ActivityPub.MRF.Utils - @moduledoc "Reject or Word-Replace activities with a keyword or regex" - - @behaviour Pleroma.Web.ActivityPub.MRF.Policy + use Pleroma.Web.ActivityPub.MRF.Policy defp string_matches?(string, pattern) when is_binary(pattern) do String.contains?(string, pattern) diff --git a/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex b/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex index 43c2c0449..07046a666 100644 --- a/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex @@ -6,6 +6,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do @moduledoc "Preloads any attachments in the MediaProxy cache by prefetching them" @behaviour Pleroma.Web.ActivityPub.MRF.Policy + use Pleroma.Web.ActivityPub.MRF.Policy + alias Pleroma.HTTP alias Pleroma.Web.MediaProxy diff --git a/lib/pleroma/web/activity_pub/mrf/mention_policy.ex b/lib/pleroma/web/activity_pub/mrf/mention_policy.ex index f7bff121f..8e108a0ae 100644 --- a/lib/pleroma/web/activity_pub/mrf/mention_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/mention_policy.ex @@ -4,9 +4,10 @@ defmodule Pleroma.Web.ActivityPub.MRF.MentionPolicy do @moduledoc "Block activities which mention a user" - @behaviour Pleroma.Web.ActivityPub.MRF.Policy + use Pleroma.Web.ActivityPub.MRF.Policy + @impl true def filter(%{"type" => "Create"} = activity) do reject_actors = Pleroma.Config.get([:mrf_mention, :actors], []) diff --git a/lib/pleroma/web/activity_pub/mrf/no_empty_policy.ex b/lib/pleroma/web/activity_pub/mrf/no_empty_policy.ex index 08dd39878..8283fd504 100644 --- a/lib/pleroma/web/activity_pub/mrf/no_empty_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/no_empty_policy.ex @@ -6,6 +6,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.NoEmptyPolicy do @moduledoc "Filter local activities which have no content" @behaviour Pleroma.Web.ActivityPub.MRF.Policy + use Pleroma.Web.ActivityPub.MRF.Policy + alias Pleroma.Web.Endpoint @impl true diff --git a/lib/pleroma/web/activity_pub/mrf/no_op_policy.ex b/lib/pleroma/web/activity_pub/mrf/no_op_policy.ex index 64a5872bc..3adbb598a 100644 --- a/lib/pleroma/web/activity_pub/mrf/no_op_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/no_op_policy.ex @@ -6,6 +6,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.NoOpPolicy do @moduledoc "Does nothing (lets the messages go through unmodified)" @behaviour Pleroma.Web.ActivityPub.MRF.Policy + use Pleroma.Web.ActivityPub.MRF.Policy + @impl true def filter(activity) do {:ok, activity} diff --git a/lib/pleroma/web/activity_pub/mrf/no_placeholder_text_policy.ex b/lib/pleroma/web/activity_pub/mrf/no_placeholder_text_policy.ex index c6f239a5e..2a10c7e97 100644 --- a/lib/pleroma/web/activity_pub/mrf/no_placeholder_text_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/no_placeholder_text_policy.ex @@ -6,6 +6,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.NoPlaceholderTextPolicy do @moduledoc "Ensure no content placeholder is present (such as the dot from mastodon)" @behaviour Pleroma.Web.ActivityPub.MRF.Policy + use Pleroma.Web.ActivityPub.MRF.Policy + @impl true def history_awareness, do: :auto diff --git a/lib/pleroma/web/activity_pub/mrf/normalize_markup.ex b/lib/pleroma/web/activity_pub/mrf/normalize_markup.ex index 91855ef84..de0f5b619 100644 --- a/lib/pleroma/web/activity_pub/mrf/normalize_markup.ex +++ b/lib/pleroma/web/activity_pub/mrf/normalize_markup.ex @@ -4,9 +4,11 @@ defmodule Pleroma.Web.ActivityPub.MRF.NormalizeMarkup do @moduledoc "Scrub configured hypertext markup" + @behaviour Pleroma.Web.ActivityPub.MRF.Policy + alias Pleroma.HTML - @behaviour Pleroma.Web.ActivityPub.MRF.Policy + use Pleroma.Web.ActivityPub.MRF.Policy @impl true def history_awareness, do: :auto diff --git a/lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex b/lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex index 52aaf05aa..7e3ec3de5 100644 --- a/lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex @@ -38,6 +38,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.NsfwApiPolicy do - `unlist`: Unlist all detected NSFW content? Default: `false` - `reject`: Reject all detected NSFW content (takes precedence)? Default: `false` """ + @behaviour Pleroma.Web.ActivityPub.MRF.Policy + alias Pleroma.Config alias Pleroma.Constants alias Pleroma.HTTP @@ -46,7 +48,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.NsfwApiPolicy do require Logger require Pleroma.Constants - @behaviour Pleroma.Web.ActivityPub.MRF.Policy + use Pleroma.Web.ActivityPub.MRF.Policy + @policy :mrf_nsfw_api def build_request_url(url) do diff --git a/lib/pleroma/web/activity_pub/mrf/object_age_policy.ex b/lib/pleroma/web/activity_pub/mrf/object_age_policy.ex index 34905fc21..00f4cf84b 100644 --- a/lib/pleroma/web/activity_pub/mrf/object_age_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/object_age_policy.ex @@ -3,13 +3,15 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.ActivityPub.MRF.ObjectAgePolicy do + @moduledoc "Filter activities depending on their age" + @behaviour Pleroma.Web.ActivityPub.MRF.Policy + alias Pleroma.Config alias Pleroma.User require Pleroma.Constants - @moduledoc "Filter activities depending on their age" - @behaviour Pleroma.Web.ActivityPub.MRF.Policy + use Pleroma.Web.ActivityPub.MRF.Policy defp check_date(%{"object" => %{"published" => published}} = activity) do with %DateTime{} = now <- DateTime.utc_now(), diff --git a/lib/pleroma/web/activity_pub/mrf/policy.ex b/lib/pleroma/web/activity_pub/mrf/policy.ex index 08bcac08a..b9a693c9e 100644 --- a/lib/pleroma/web/activity_pub/mrf/policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/policy.ex @@ -15,4 +15,12 @@ defmodule Pleroma.Web.ActivityPub.MRF.Policy do } @callback history_awareness() :: :auto | :manual @optional_callbacks config_description: 0, history_awareness: 0, id_filter: 1 + + defmacro __using__(_) do + quote do + # Otherwise Pleroma.Docs.Generator.list_behaviour_implementations fails + # since the Elixir compiler doesn't automatically load the module since v1.19. + @compile {:autoload, true} + end + end end diff --git a/lib/pleroma/web/activity_pub/mrf/quiet_reply.ex b/lib/pleroma/web/activity_pub/mrf/quiet_reply.ex index b3eb0b390..b47e9edb9 100644 --- a/lib/pleroma/web/activity_pub/mrf/quiet_reply.ex +++ b/lib/pleroma/web/activity_pub/mrf/quiet_reply.ex @@ -6,11 +6,13 @@ defmodule Pleroma.Web.ActivityPub.MRF.QuietReply do @moduledoc """ QuietReply alters the scope of activities from local users when replying by enforcing them to be "Unlisted" or "Quiet Public". This delivers the activity to all the expected recipients and instances, but it will not be published in the Federated / The Whole Known Network timelines. It will still be published to the Home timelines of the user's followers and visible to anyone who opens the thread. """ + @behaviour Pleroma.Web.ActivityPub.MRF.Policy + require Pleroma.Constants alias Pleroma.User - @behaviour Pleroma.Web.ActivityPub.MRF.Policy + use Pleroma.Web.ActivityPub.MRF.Policy @impl true def history_awareness, do: :auto diff --git a/lib/pleroma/web/activity_pub/mrf/quote_to_link_tag_policy.ex b/lib/pleroma/web/activity_pub/mrf/quote_to_link_tag_policy.ex index 2a17b6761..e1ad17585 100644 --- a/lib/pleroma/web/activity_pub/mrf/quote_to_link_tag_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/quote_to_link_tag_policy.ex @@ -6,6 +6,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.QuoteToLinkTagPolicy do @moduledoc "Force a Link tag for posts quoting another post. (may break outgoing federation of quote posts with older Pleroma versions)" @behaviour Pleroma.Web.ActivityPub.MRF.Policy + use Pleroma.Web.ActivityPub.MRF.Policy + alias Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes require Pleroma.Constants diff --git a/lib/pleroma/web/activity_pub/mrf/reject_non_public.ex b/lib/pleroma/web/activity_pub/mrf/reject_non_public.ex index 9d4a7a405..7d113dbf6 100644 --- a/lib/pleroma/web/activity_pub/mrf/reject_non_public.ex +++ b/lib/pleroma/web/activity_pub/mrf/reject_non_public.ex @@ -4,11 +4,12 @@ defmodule Pleroma.Web.ActivityPub.MRF.RejectNonPublic do @moduledoc "Rejects non-public (followers-only, direct) activities" + @behaviour Pleroma.Web.ActivityPub.MRF.Policy alias Pleroma.Config alias Pleroma.User - @behaviour Pleroma.Web.ActivityPub.MRF.Policy + use Pleroma.Web.ActivityPub.MRF.Policy require Pleroma.Constants diff --git a/lib/pleroma/web/activity_pub/mrf/remote_report_policy.ex b/lib/pleroma/web/activity_pub/mrf/remote_report_policy.ex index 8422832dd..f9f8d2298 100644 --- a/lib/pleroma/web/activity_pub/mrf/remote_report_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/remote_report_policy.ex @@ -2,6 +2,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.RemoteReportPolicy do @moduledoc "Drop remote reports if they don't contain enough information." @behaviour Pleroma.Web.ActivityPub.MRF.Policy + use Pleroma.Web.ActivityPub.MRF.Policy + alias Pleroma.Config @impl true diff --git a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex index a97e8db7b..7045fe90f 100644 --- a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex @@ -6,6 +6,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do @moduledoc "Filter activities depending on their origin instance" @behaviour Pleroma.Web.ActivityPub.MRF.Policy + use Pleroma.Web.ActivityPub.MRF.Policy + alias Pleroma.Config alias Pleroma.FollowingRelationship alias Pleroma.User diff --git a/lib/pleroma/web/activity_pub/mrf/steal_emoji_policy.ex b/lib/pleroma/web/activity_pub/mrf/steal_emoji_policy.ex index 54f0e6bc1..f1e99aead 100644 --- a/lib/pleroma/web/activity_pub/mrf/steal_emoji_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/steal_emoji_policy.ex @@ -3,12 +3,14 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.ActivityPub.MRF.StealEmojiPolicy do + @moduledoc "Detect new emojis by their shortcode and steals them" + @behaviour Pleroma.Web.ActivityPub.MRF.Policy + require Logger alias Pleroma.Config - @moduledoc "Detect new emojis by their shortcode and steals them" - @behaviour Pleroma.Web.ActivityPub.MRF.Policy + use Pleroma.Web.ActivityPub.MRF.Policy defp accept_host?(host), do: host in Config.get([:mrf_steal_emoji, :hosts], []) diff --git a/lib/pleroma/web/activity_pub/mrf/subchain_policy.ex b/lib/pleroma/web/activity_pub/mrf/subchain_policy.ex index 97acca7e8..1fe7b00e9 100644 --- a/lib/pleroma/web/activity_pub/mrf/subchain_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/subchain_policy.ex @@ -3,12 +3,14 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.ActivityPub.MRF.SubchainPolicy do + @behaviour Pleroma.Web.ActivityPub.MRF.Policy + alias Pleroma.Config alias Pleroma.Web.ActivityPub.MRF require Logger - @behaviour Pleroma.Web.ActivityPub.MRF.Policy + use Pleroma.Web.ActivityPub.MRF.Policy defp lookup_subchain(actor) do with matches <- Config.get([:mrf_subchain, :match_actor]), diff --git a/lib/pleroma/web/activity_pub/mrf/tag_policy.ex b/lib/pleroma/web/activity_pub/mrf/tag_policy.ex index c236a5a99..df5594808 100644 --- a/lib/pleroma/web/activity_pub/mrf/tag_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/tag_policy.ex @@ -3,8 +3,6 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.ActivityPub.MRF.TagPolicy do - alias Pleroma.User - @behaviour Pleroma.Web.ActivityPub.MRF.Policy @moduledoc """ Apply policies based on user tags @@ -18,6 +16,11 @@ defmodule Pleroma.Web.ActivityPub.MRF.TagPolicy do - `mrf_tag:disable-remote-subscription`: Reject non-local follow requests - `mrf_tag:disable-any-subscription`: Reject any follow requests """ + @behaviour Pleroma.Web.ActivityPub.MRF.Policy + + alias Pleroma.User + + use Pleroma.Web.ActivityPub.MRF.Policy require Pleroma.Constants diff --git a/lib/pleroma/web/activity_pub/mrf/user_allow_list_policy.ex b/lib/pleroma/web/activity_pub/mrf/user_allow_list_policy.ex index 10cc0e09d..a04ca90bd 100644 --- a/lib/pleroma/web/activity_pub/mrf/user_allow_list_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/user_allow_list_policy.ex @@ -3,11 +3,13 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.ActivityPub.MRF.UserAllowListPolicy do - alias Pleroma.Config - @moduledoc "Accept-list of users from specified instances" @behaviour Pleroma.Web.ActivityPub.MRF.Policy + alias Pleroma.Config + + use Pleroma.Web.ActivityPub.MRF.Policy + defp filter_by_list(activity, []), do: {:ok, activity} defp filter_by_list(%{"actor" => actor} = activity, allow_list) do diff --git a/lib/pleroma/web/activity_pub/mrf/vocabulary_policy.ex b/lib/pleroma/web/activity_pub/mrf/vocabulary_policy.ex index 5671e4cf3..ecba3a2c6 100644 --- a/lib/pleroma/web/activity_pub/mrf/vocabulary_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/vocabulary_policy.ex @@ -4,9 +4,10 @@ defmodule Pleroma.Web.ActivityPub.MRF.VocabularyPolicy do @moduledoc "Filter activities which belong to certain activity vocabularies" - @behaviour Pleroma.Web.ActivityPub.MRF.Policy + use Pleroma.Web.ActivityPub.MRF.Policy + @impl true def filter(%{"type" => "Undo", "object" => object} = activity) do with {:ok, _} <- filter(object) do diff --git a/lib/pleroma/web/activity_pub/views/user_view.ex b/lib/pleroma/web/activity_pub/views/user_view.ex index 8847692c8..c273043e1 100644 --- a/lib/pleroma/web/activity_pub/views/user_view.ex +++ b/lib/pleroma/web/activity_pub/views/user_view.ex @@ -68,7 +68,7 @@ defmodule Pleroma.Web.ActivityPub.UserView do capabilities = if is_boolean(user.accepts_chat_messages) do %{ - "acceptsChatMessages" => user.accepts_chat_messages + "acceptsChatMessages" => Pleroma.Chat.enabled?() and user.accepts_chat_messages } else %{} diff --git a/lib/pleroma/web/admin_api/controllers/config_controller.ex b/lib/pleroma/web/admin_api/controllers/config_controller.ex index 3eba03525..753690d45 100644 --- a/lib/pleroma/web/admin_api/controllers/config_controller.ex +++ b/lib/pleroma/web/admin_api/controllers/config_controller.ex @@ -71,7 +71,10 @@ defmodule Pleroma.Web.AdminAPI.ConfigController do end def descriptions(conn, _params) do - descriptions = Enum.filter(Pleroma.Docs.JSON.compiled_descriptions(), &whitelisted_config?/1) + descriptions = + Pleroma.Docs.JSON.compiled_descriptions() + |> Enum.filter(&allowed_config?(&1, :whitelist)) + |> Enum.reject(&allowed_config?(&1, :blacklist)) json(conn, translate_descriptions(descriptions)) end @@ -132,7 +135,8 @@ defmodule Pleroma.Web.AdminAPI.ConfigController do with :ok <- configurable_from_database() do results = configs - |> Enum.filter(&whitelisted_config?/1) + |> Enum.filter(&allowed_config?(&1, :whitelist)) + |> Enum.reject(&allowed_config?(&1, :blacklist)) |> Enum.map(fn %{group: group, key: key, delete: true} = params -> ConfigDB.delete(%{group: group, key: key, subkeys: params[:subkeys]}) @@ -174,27 +178,34 @@ defmodule Pleroma.Web.AdminAPI.ConfigController do end end - defp whitelisted_config?(":pleroma", ":database_config_whitelist"), do: false + defp allowed_config?(":pleroma", ":database_config_whitelist", _), do: true + defp allowed_config?(":pleroma", ":database_config_blacklist", _), do: true - defp whitelisted_config?(group, key) do - if whitelisted_configs = Config.get(:database_config_whitelist) do - Enum.any?(whitelisted_configs, fn - {whitelisted_group} -> - group == inspect(whitelisted_group) + defp allowed_config?(group, key, mode) do + configs = + case mode do + :whitelist -> Config.get(:database_config_whitelist) + :blacklist -> Config.get(:database_config_blacklist) + end - {whitelisted_group, whitelisted_key} -> - group == inspect(whitelisted_group) && key == inspect(whitelisted_key) + if configs do + Enum.any?(configs, fn + {allowed_group} -> + group == inspect(allowed_group) + + {allowed_group, allowed_key} -> + group == inspect(allowed_group) && key == inspect(allowed_key) end) else true end end - defp whitelisted_config?(%{group: group, key: key}) do - whitelisted_config?(group, key) + defp allowed_config?(%{group: group, key: key}, mode) do + allowed_config?(group, key, mode) end - defp whitelisted_config?(%{group: group} = config) do - whitelisted_config?(group, config[:key]) + defp allowed_config?(%{group: group} = config, mode) do + allowed_config?(group, config[:key], mode) end end diff --git a/lib/pleroma/web/api_spec/operations/chat_operation.ex b/lib/pleroma/web/api_spec/operations/chat_operation.ex index 76c2db57e..7bc39acc6 100644 --- a/lib/pleroma/web/api_spec/operations/chat_operation.ex +++ b/lib/pleroma/web/api_spec/operations/chat_operation.ex @@ -31,7 +31,8 @@ defmodule Pleroma.Web.ApiSpec.ChatOperation do "The updated chat", "application/json", Chat - ) + ), + 404 => Operation.response("Not Found", "application/json", ApiError) }, security: [ %{ @@ -56,7 +57,8 @@ defmodule Pleroma.Web.ApiSpec.ChatOperation do "The read ChatMessage", "application/json", ChatMessage - ) + ), + 404 => Operation.response("Not Found", "application/json", ApiError) }, security: [ %{ @@ -87,7 +89,8 @@ defmodule Pleroma.Web.ApiSpec.ChatOperation do "The existing chat", "application/json", Chat - ) + ), + 404 => Operation.response("Not Found", "application/json", ApiError) }, security: [ %{ @@ -118,7 +121,8 @@ defmodule Pleroma.Web.ApiSpec.ChatOperation do "The created or existing chat", "application/json", Chat - ) + ), + 404 => Operation.response("Not Found", "application/json", ApiError) }, security: [ %{ @@ -146,7 +150,8 @@ defmodule Pleroma.Web.ApiSpec.ChatOperation do Operation.parameter(:pinned, :query, BooleanLike.schema(), "Include only pinned chats") ], responses: %{ - 200 => Operation.response("The chats of the user", "application/json", chats_response()) + 200 => Operation.response("The chats of the user", "application/json", chats_response()), + 404 => Operation.response("Not Found", "application/json", ApiError) }, security: [ %{ @@ -172,7 +177,8 @@ defmodule Pleroma.Web.ApiSpec.ChatOperation do | pagination_params() ], responses: %{ - 200 => Operation.response("The chats of the user", "application/json", chats_response()) + 200 => Operation.response("The chats of the user", "application/json", chats_response()), + 404 => Operation.response("Not Found", "application/json", ApiError) }, security: [ %{ @@ -224,6 +230,7 @@ defmodule Pleroma.Web.ApiSpec.ChatOperation do ChatMessage ), 400 => Operation.response("Bad Request", "application/json", ApiError), + 404 => Operation.response("Not Found", "application/json", ApiError), 422 => Operation.response("MRF Rejection", "application/json", ApiError) }, security: [ @@ -249,7 +256,8 @@ defmodule Pleroma.Web.ApiSpec.ChatOperation do "The deleted ChatMessage", "application/json", ChatMessage - ) + ), + 404 => Operation.response("Not Found", "application/json", ApiError) }, security: [ %{ @@ -268,7 +276,8 @@ defmodule Pleroma.Web.ApiSpec.ChatOperation do Operation.parameter(:id, :path, :string, "The id of the chat", required: true) ], responses: %{ - 200 => Operation.response("The existing chat", "application/json", Chat) + 200 => Operation.response("The existing chat", "application/json", Chat), + 404 => Operation.response("Not Found", "application/json", ApiError) }, security: [ %{ @@ -287,7 +296,8 @@ defmodule Pleroma.Web.ApiSpec.ChatOperation do Operation.parameter(:id, :path, :string, "The id of the chat", required: true) ], responses: %{ - 200 => Operation.response("The existing chat", "application/json", Chat) + 200 => Operation.response("The existing chat", "application/json", Chat), + 404 => Operation.response("Not Found", "application/json", ApiError) }, security: [ %{ diff --git a/lib/pleroma/web/api_spec/operations/list_operation.ex b/lib/pleroma/web/api_spec/operations/list_operation.ex index 87189edc2..8aec1c352 100644 --- a/lib/pleroma/web/api_spec/operations/list_operation.ex +++ b/lib/pleroma/web/api_spec/operations/list_operation.ex @@ -41,7 +41,8 @@ defmodule Pleroma.Web.ApiSpec.ListOperation do responses: %{ 200 => Operation.response("List", "application/json", List), 400 => Operation.response("Error", "application/json", ApiError), - 404 => Operation.response("Error", "application/json", ApiError) + 404 => Operation.response("Error", "application/json", ApiError), + 422 => Operation.response("Error", "application/json", ApiError) } } end @@ -175,7 +176,8 @@ defmodule Pleroma.Web.ApiSpec.ListOperation do exclusive: %Schema{ type: :boolean, description: "Whether members of the list should be removed from the โ€œHomeโ€ feed" - } + }, + emoji: %Schema{type: :string, description: "List emoji", nullable: true} }, required: [:title] }, @@ -194,7 +196,8 @@ defmodule Pleroma.Web.ApiSpec.ListOperation do exclusive: %Schema{ type: :boolean, description: "Whether members of the list should be removed from the โ€œHomeโ€ feed" - } + }, + emoji: %Schema{type: :string, description: "List emoji", nullable: true} } }, required: true diff --git a/lib/pleroma/web/api_spec/operations/notification_operation.ex b/lib/pleroma/web/api_spec/operations/notification_operation.ex index 94d1f6b82..6c8f7d5e9 100644 --- a/lib/pleroma/web/api_spec/operations/notification_operation.ex +++ b/lib/pleroma/web/api_spec/operations/notification_operation.ex @@ -3,7 +3,6 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.ApiSpec.NotificationOperation do - alias OpenApiSpex.Operation alias OpenApiSpex.Operation alias OpenApiSpex.Schema alias Pleroma.Web.ApiSpec.Schemas.Account @@ -91,6 +90,79 @@ defmodule Pleroma.Web.ApiSpec.NotificationOperation do } end + def grouped_index_operation do + %Operation{ + tags: ["Notifications"], + summary: "Retrieve grouped notifications", + description: "Notifications concerning the user, grouped by supported notification types.", + operationId: "NotificationController.grouped_index", + security: [%{"oAuth" => ["read:notifications"]}], + parameters: grouped_notification_params() ++ pagination_params(), + responses: %{ + 200 => Operation.response("Grouped notifications", "application/json", grouped_result()), + 404 => Operation.response("Error", "application/json", ApiError) + } + } + end + + def show_group_operation do + %Operation{ + tags: ["Notifications"], + summary: "Retrieve a notification group", + operationId: "NotificationController.show_group", + security: [%{"oAuth" => ["read:notifications"]}], + parameters: [group_key_param()], + responses: %{ + 200 => Operation.response("Grouped notifications", "application/json", grouped_result()), + 404 => Operation.response("Error", "application/json", ApiError) + } + } + end + + def group_accounts_operation do + %Operation{ + tags: ["Notifications"], + summary: "Retrieve notification group accounts", + operationId: "NotificationController.group_accounts", + security: [%{"oAuth" => ["read:notifications"]}], + parameters: [group_key_param()], + responses: %{ + 200 => + Operation.response("Accounts", "application/json", %Schema{ + type: :array, + items: Account + }) + } + } + end + + def unread_count_operation do + %Operation{ + tags: ["Notifications"], + summary: "Retrieve unread grouped notification count", + operationId: "NotificationController.unread_count", + security: [%{"oAuth" => ["read:notifications"]}], + parameters: + grouped_notification_params() ++ + [ + Operation.parameter( + :limit, + :query, + %Schema{type: :integer}, + "Maximum number of notifications to count" + ) + ], + responses: %{ + 200 => + Operation.response("Unread notification group count", "application/json", %Schema{ + type: :object, + properties: %{count: %Schema{type: :integer}} + }), + 404 => Operation.response("Error", "application/json", ApiError) + } + } + end + def clear_operation do %Operation{ tags: ["Notifications"], @@ -114,6 +186,18 @@ defmodule Pleroma.Web.ApiSpec.NotificationOperation do } end + def dismiss_group_operation do + %Operation{ + tags: ["Notifications"], + summary: "Dismiss a notification group", + description: "Clear all notifications in a notification group from the server.", + operationId: "NotificationController.dismiss_group", + parameters: [group_key_param()], + security: [%{"oAuth" => ["write:notifications"]}], + responses: %{200 => empty_object_response()} + } + end + def dismiss_via_body_operation do %Operation{ tags: ["Notifications"], @@ -184,7 +268,7 @@ defmodule Pleroma.Web.ApiSpec.NotificationOperation do }, example: %{ "id" => "34975861", - "group-key" => "ungrouped-34975861", + "group_key" => "ungrouped-34975861", "type" => "mention", "created_at" => "2019-11-23T07:49:02.064Z", "account" => Account.schema().example, @@ -194,6 +278,73 @@ defmodule Pleroma.Web.ApiSpec.NotificationOperation do } end + defp grouped_result do + %Schema{ + title: "GroupedNotificationsResults", + type: :object, + properties: %{ + accounts: %Schema{type: :array, items: Account}, + statuses: %Schema{type: :array, items: Status}, + notification_groups: %Schema{type: :array, items: notification_group()} + }, + required: [:accounts, :statuses, :notification_groups] + } + end + + defp notification_group do + %Schema{ + title: "NotificationGroup", + type: :object, + properties: %{ + group_key: %Schema{type: :string}, + notifications_count: %Schema{type: :integer}, + type: notification_type(), + most_recent_notification_id: %Schema{type: :string}, + page_min_id: %Schema{type: :string, nullable: true}, + page_max_id: %Schema{type: :string, nullable: true}, + latest_page_notification_at: %Schema{type: :string, format: :"date-time", nullable: true}, + sample_account_ids: %Schema{type: :array, items: %Schema{type: :string}}, + status_id: %Schema{type: :string, nullable: true} + }, + required: [ + :group_key, + :notifications_count, + :type, + :most_recent_notification_id, + :sample_account_ids + ] + } + end + + defp grouped_notification_params do + [ + Operation.parameter( + :exclude_types, + :query, + %Schema{type: :array, items: notification_type()}, + "Array of types to exclude" + ), + Operation.parameter( + :account_id, + :query, + %Schema{type: :string}, + "Return only notifications received from this account" + ), + Operation.parameter( + :types, + :query, + %Schema{type: :array, items: notification_type()}, + "Include the notifications for activities with the given types" + ), + Operation.parameter( + :grouped_types, + :query, + %Schema{type: :array, items: notification_type()}, + "Notification types that may be grouped" + ) + ] + end + defp notification_type do %Schema{ type: :string, @@ -239,4 +390,11 @@ defmodule Pleroma.Web.ApiSpec.NotificationOperation do required: true ) end + + defp group_key_param do + Operation.parameter(:group_key, :path, :string, "Notification group key", + example: "favourite-123", + required: true + ) + end end diff --git a/lib/pleroma/web/api_spec/schemas/list.ex b/lib/pleroma/web/api_spec/schemas/list.ex index 5df674894..ff52b1def 100644 --- a/lib/pleroma/web/api_spec/schemas/list.ex +++ b/lib/pleroma/web/api_spec/schemas/list.ex @@ -17,11 +17,26 @@ defmodule Pleroma.Web.ApiSpec.Schemas.List do exclusive: %Schema{ type: :boolean, description: "Whether members of the list should be removed from the โ€œHomeโ€ feed" + }, + pleroma: %Schema{ + type: :object, + properties: %{ + emoji: %Schema{type: :string, description: "List emoji", nullable: true}, + emoji_url: %Schema{ + type: :string, + description: "URL of the list emoji if it's a custom emoji, null otherwise", + nullable: true + } + } } }, example: %{ "id" => "12249", - "title" => "Friends" + "title" => "Friends", + "pleroma" => %{ + "emoji" => nil, + "emoji_url" => nil + } } }) end diff --git a/lib/pleroma/web/controller_helper.ex b/lib/pleroma/web/controller_helper.ex index b15b0ea81..0c85e79f9 100644 --- a/lib/pleroma/web/controller_helper.ex +++ b/lib/pleroma/web/controller_helper.ex @@ -55,16 +55,28 @@ defmodule Pleroma.Web.ControllerHelper do # TODO: Only fetch the params from open_api_spex when everything is converted @id_keys Pagination.page_keys() -- ["limit", "order"] + @id_key_atoms Enum.map(@id_keys, &String.to_atom/1) + @drop_id_params_key :drop_id_params defp build_pagination_fields(conn, min_id, max_id, extra_params, order) do + drop_id_params? = Map.get(extra_params, @drop_id_params_key, false) + extra_params = Map.delete(extra_params, @drop_id_params_key) + + path_param_keys = + conn.path_params + |> Map.keys() + |> Enum.flat_map(&[&1, String.to_existing_atom(&1)]) + params = if Map.has_key?(conn.private, :open_api_spex) do get_in(conn, [Access.key(:private), Access.key(:open_api_spex), Access.key(:params)]) else conn.params end - |> Map.drop(Map.keys(conn.path_params) |> Enum.map(&String.to_existing_atom/1)) + |> Map.drop(path_param_keys) |> Map.merge(extra_params) - |> Map.drop(@id_keys) + # Some OpenApiSpex routes cast cursor params to atoms. Keep the historical default for + # existing endpoints, but allow grouped notifications to opt out of stale request cursors. + |> Map.drop(@id_keys ++ if(drop_id_params?, do: @id_key_atoms, else: [])) {{next_id, nid}, {prev_id, pid}} = if order == :desc, diff --git a/lib/pleroma/web/endpoint.ex b/lib/pleroma/web/endpoint.ex index 07b33f866..8552f5d0a 100644 --- a/lib/pleroma/web/endpoint.ex +++ b/lib/pleroma/web/endpoint.ex @@ -139,6 +139,8 @@ defmodule Pleroma.Web.Endpoint do from: {:pleroma, "priv/static/adminfe/"} ) + plug(Pleroma.Web.Plugs.StaticNotFoundPlug) + # Code reloading can be explicitly enabled under the # :code_reloader configuration of your endpoint. if code_reloading? do diff --git a/lib/pleroma/web/federator.ex b/lib/pleroma/web/federator.ex index 90cd2e54a..88b267331 100644 --- a/lib/pleroma/web/federator.ex +++ b/lib/pleroma/web/federator.ex @@ -89,7 +89,8 @@ defmodule Pleroma.Web.Federator do # Job Worker Callbacks - @spec perform(atom(), any()) :: {:ok, any()} | {:error, any()} + @impl true + @spec perform(atom(), any()) :: :ok | {:ok, any()} | {:error, any()} def perform(:publish_one, params) do Publisher.prepare_one(params) |> Publisher.publish_one() diff --git a/lib/pleroma/web/federator/publishing.ex b/lib/pleroma/web/federator/publishing.ex index 3a242b8dc..889b6b83e 100644 --- a/lib/pleroma/web/federator/publishing.ex +++ b/lib/pleroma/web/federator/publishing.ex @@ -3,5 +3,6 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.Federator.Publishing do + @callback perform(atom(), any()) :: :ok | {:ok, any()} | {:error, any()} @callback publish(map()) :: any() end diff --git a/lib/pleroma/web/feed/feed_view.ex b/lib/pleroma/web/feed/feed_view.ex index e1ee33d62..b409b1923 100644 --- a/lib/pleroma/web/feed/feed_view.ex +++ b/lib/pleroma/web/feed/feed_view.ex @@ -38,6 +38,20 @@ defmodule Pleroma.Web.Feed.FeedView do end end + def most_recent_update([], :rss), do: nil + + def most_recent_update(activities, :rss) do + activities + |> Enum.map(fn activity -> + case activity.object do + %Object{updated_at: updated_at} -> updated_at + _ -> activity.updated_at + end + end) + |> Enum.max_by(&NaiveDateTime.to_erl/1) + |> to_rfc2822() + end + def most_recent_update(activities, user, :atom) do (List.first(activities) || user).updated_at |> to_rfc3339() @@ -97,6 +111,67 @@ defmodule Pleroma.Web.Feed.FeedView do end end + def rss_content_encoded(data) do + data + |> feed_content_encoded() + |> String.replace("]]>", "]]>") + end + + def atom_content_encoded(data), do: feed_content_encoded(data) + + defp feed_content_encoded(data) do + base_content = + case activity_content(data) do + "" -> + (data["summary"] || data["type"] || "") + |> escape() + + content -> + content + end + + attachments_html = + if sensitive?(data) do + "" + else + (data["attachment"] || []) + |> Enum.map(&rss_attachment_preview/1) + |> Enum.reject(&(&1 == "")) + |> Enum.join("

") + end + + [base_content, attachments_html] + |> Enum.reject(&(&1 == "")) + |> Enum.join("

") + end + + defp rss_attachment_preview(attachment) do + href = attachment_href(attachment) + + if is_binary(href) and href != "" do + media_type = attachment_type(attachment) || "application/octet-stream" + escaped_href = escape(href) + name = attachment["name"] || "Attachment" + escaped_name = escape(name) + + cond do + String.starts_with?(media_type, "image/") -> + ~s(

#{escaped_name}

) + + String.starts_with?(media_type, "video/") -> + ~s(

#{escaped_name}

) + + String.starts_with?(media_type, "audio/") -> + ~s(

#{escaped_name}

) + + true -> + ~s(

#{escaped_name}

) + end + else + "" + end + end + def activity_content(%{"content" => content}) do content |> String.replace(~r/[\n\r]/, "") @@ -106,16 +181,104 @@ defmodule Pleroma.Web.Feed.FeedView do def activity_context(activity), do: escape(activity.data["context"]) + def sensitive?(data), do: data["sensitive"] in [true, "true"] + + def feed_self_url(conn) do + conn + |> Phoenix.Controller.current_url() + |> escape() + end + + def activity_link(activity, data) do + cond do + activity.local -> data["id"] || data["url"] + true -> data["external_url"] || data["id"] || data["url"] + end + end + def attachment_href(attachment) do - attachment["url"] - |> hd() + attachment + |> attachment_url_data() |> Map.get("href") end def attachment_type(attachment) do - attachment["url"] - |> hd() - |> Map.get("mediaType") + attachment + |> attachment_url_data() + |> Map.get("mediaType") || attachment["mediaType"] + end + + def attachment_size(attachment) do + attachment + |> attachment_url_data() + |> Map.get("size", 0) + end + + def attachment_size_positive(attachment) do + case attachment_size(attachment) do + size when is_integer(size) and size > 0 -> + size + + size when is_binary(size) -> + case Integer.parse(size) do + {parsed, ""} when parsed > 0 -> parsed + _ -> nil + end + + _ -> + nil + end + end + + def attachment_medium(attachment) do + case attachment_type(attachment) do + "image/" <> _rest -> "image" + "video/" <> _rest -> "video" + "audio/" <> _rest -> "audio" + _ -> "document" + end + end + + def rss_enclosure_attachment(attachments) do + Enum.find(attachments || [], &(not is_nil(attachment_size_positive(&1)))) + end + + def attachment_description(attachment) do + attachment["name"] || "" + end + + def media_content_xml(_attachment, sensitive) when sensitive in [true, "true"], do: "" + + def media_content_xml(attachment, _sensitive) do + href = escape(attachment_href(attachment) || "") + type = escape(attachment_type(attachment) || "application/octet-stream") + medium = escape(attachment_medium(attachment)) + + file_size_attr = + case attachment_size_positive(attachment) do + size when is_integer(size) -> ~s( fileSize="#{size}") + _ -> "" + end + + case attachment_description(attachment) do + "" -> + ~s(\n) + + description -> + """ + + #{escape(description)} + + """ + end + end + + defp attachment_url_data(attachment) do + case attachment["url"] do + [first | _] when is_map(first) -> first + %{} = map -> map + _ -> %{} + end end def get_href(id) do diff --git a/lib/pleroma/web/mastodon_api/controllers/notification_controller.ex b/lib/pleroma/web/mastodon_api/controllers/notification_controller.ex index afd83b785..8a5f8e7b6 100644 --- a/lib/pleroma/web/mastodon_api/controllers/notification_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/notification_controller.ex @@ -5,13 +5,17 @@ defmodule Pleroma.Web.MastodonAPI.NotificationController do use Pleroma.Web, :controller - import Pleroma.Web.ControllerHelper, only: [add_link_headers: 2] + import Pleroma.Web.ControllerHelper, only: [add_link_headers: 2, add_link_headers: 3] alias Pleroma.Notification + alias Pleroma.User + alias Pleroma.Web.MastodonAPI.AccountView alias Pleroma.Web.MastodonAPI.MastodonAPI alias Pleroma.Web.Plugs.OAuthScopesPlug - @oauth_read_actions [:show, :index] + # Mastodon's docs currently list write:notifications for group accounts, but the endpoint is + # read-only and Mastodon's implementation accepts read:notifications. Prefer least privilege. + @oauth_read_actions [:show, :index, :grouped_index, :show_group, :group_accounts, :unread_count] plug(Pleroma.Web.ApiSpec.CastAndValidate, replace_params: false) @@ -39,7 +43,7 @@ defmodule Pleroma.Web.MastodonAPI.NotificationController do # GET /api/v1/notifications def index(%{private: %{open_api_spex: %{params: %{account_id: account_id} = params}}} = conn, _) do - case Pleroma.User.get_cached_by_id(account_id) do + case User.get_cached_by_id(account_id) do %{ap_id: account_ap_id} -> params = params @@ -59,10 +63,108 @@ defmodule Pleroma.Web.MastodonAPI.NotificationController do do_get_notifications(conn, params) end + # GET /api/v2/notifications + def grouped_index( + %{private: %{open_api_spex: %{params: %{account_id: account_id} = params}}} = conn, + _ + ) do + case User.get_cached_by_id(account_id) do + %{ap_id: account_ap_id} -> + params = + params + |> Map.delete(:account_id) + |> Map.put(:account_ap_id, account_ap_id) + + do_get_grouped_notifications(conn, params) + + _ -> + conn + |> put_status(:not_found) + |> json(%{"error" => "Account is not found"}) + end + end + + def grouped_index(%{private: %{open_api_spex: %{params: params}}} = conn, _) do + do_get_grouped_notifications(conn, params) + end + + # GET /api/v2/notifications/:group_key + def show_group( + %{assigns: %{user: user}, private: %{open_api_spex: %{params: %{group_key: group_key}}}} = + conn, + _ + ) do + {notifications, notification_group_counts, notification_group_bounds} = + MastodonAPI.get_notification_group_result(user, group_key, %{}) + + if Enum.empty?(notifications) do + conn + |> put_status(:not_found) + |> json(%{"error" => "Notification group is not found"}) + else + grouped_types = if String.starts_with?(group_key, "ungrouped-"), do: [], else: nil + + render(conn, "grouped_index.json", + notification_groups: [notifications], + notification_group_counts: notification_group_counts, + notification_group_bounds: notification_group_bounds, + for: user, + grouped_types: grouped_types, + include_page_metadata: false + ) + end + end + + # GET /api/v2/notifications/:group_key/accounts + def group_accounts( + %{assigns: %{user: user}, private: %{open_api_spex: %{params: %{group_key: group_key}}}} = + conn, + _ + ) do + # Mastodon paginates this endpoint in code, but the public docs say it returns accounts of all + # notifications in the group and do not document cursor params here. Follow the documented API. + users = MastodonAPI.get_notification_group_accounts(user, group_key) + + json(conn, AccountView.render("index.json", %{users: users, for: user})) + end + + # GET /api/v2/notifications/unread_count + def unread_count( + %{private: %{open_api_spex: %{params: %{account_id: account_id} = params}}} = conn, + _ + ) do + case User.get_cached_by_id(account_id) do + %{ap_id: account_ap_id} -> + params = + params + |> Map.delete(:account_id) + |> Map.put(:account_ap_id, account_ap_id) + + do_get_unread_group_count(conn, params) + + _ -> + conn + |> put_status(:not_found) + |> json(%{"error" => "Account is not found"}) + end + end + + def unread_count(%{private: %{open_api_spex: %{params: params}}} = conn, _) do + do_get_unread_group_count(conn, params) + end + + # POST /api/v2/notifications/:group_key/dismiss + def dismiss_group( + %{assigns: %{user: user}, private: %{open_api_spex: %{params: %{group_key: group_key}}}} = + conn, + _ + ) do + MastodonAPI.dismiss_notification_group(user, group_key) + json(conn, %{}) + end + defp do_get_notifications(%{assigns: %{user: user}} = conn, params) do - params = - Map.new(params, fn {k, v} -> {to_string(k), v} end) - |> Map.put_new("types", Map.get(params, :include_types, @default_notification_types)) + params = normalize_notification_params(params) notifications = MastodonAPI.get_notifications(user, params) @@ -74,6 +176,35 @@ defmodule Pleroma.Web.MastodonAPI.NotificationController do ) end + defp do_get_grouped_notifications(%{assigns: %{user: user}} = conn, params) do + params = normalize_notification_params(params) + + {notification_groups, page_notifications, notification_group_counts, + notification_group_bounds} = + MastodonAPI.get_grouped_notification_page(user, params) + + conn + |> add_link_headers(page_notifications, %{drop_id_params: true}) + |> render("grouped_index.json", + notification_groups: notification_groups, + notification_group_counts: notification_group_counts, + notification_group_bounds: notification_group_bounds, + for: user, + grouped_types: params["grouped_types"] + ) + end + + defp do_get_unread_group_count(%{assigns: %{user: user}} = conn, params) do + params = normalize_notification_params(params) + json(conn, %{count: MastodonAPI.unread_notification_group_count(user, params)}) + end + + defp normalize_notification_params(params) do + params + |> Map.new(fn {k, v} -> {to_string(k), v} end) + |> Map.put_new("types", Map.get(params, :include_types, @default_notification_types)) + end + # GET /api/v1/notifications/:id def show(%{assigns: %{user: user}, private: %{open_api_spex: %{params: %{id: id}}}} = conn, _) do with {:ok, notification} <- Notification.get(user, id) do diff --git a/lib/pleroma/web/mastodon_api/controllers/status_controller.ex b/lib/pleroma/web/mastodon_api/controllers/status_controller.ex index 26e38aac8..6cd7469f9 100644 --- a/lib/pleroma/web/mastodon_api/controllers/status_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/status_controller.ex @@ -716,7 +716,10 @@ defmodule Pleroma.Web.MastodonAPI.StatusController do end end - defp put_application(params, %{assigns: %{token: %Token{user: %User{} = user} = token}} = _conn) do + defp put_application( + params, + %{assigns: %{token: %Token{} = token, user: %User{} = user}} = _conn + ) do if user.disclose_client do %{client_name: client_name, website: website} = Repo.preload(token, :app).app Map.put(params, :generator, %{type: "Application", name: client_name, url: website}) diff --git a/lib/pleroma/web/mastodon_api/mastodon_api.ex b/lib/pleroma/web/mastodon_api/mastodon_api.ex index c9e045d23..a65326226 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api.ex @@ -6,12 +6,16 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPI do import Ecto.Query import Ecto.Changeset + alias Pleroma.Marker alias Pleroma.Notification alias Pleroma.Pagination + alias Pleroma.Repo alias Pleroma.ScheduledActivity alias Pleroma.User alias Pleroma.Web.CommonAPI + @notification_group_sample_limit 8 + @spec follow(User.t(), User.t(), map) :: {:ok, User.t()} | {:error, String.t()} def follow(follower, followed, params \\ %{}) do result = @@ -61,29 +65,365 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPI do end def get_notifications(user, params \\ %{}) do - options = - cast_params(params) |> Map.update(:include_types, [], fn include_types -> include_types end) + user + |> notifications_query(params) + |> Pagination.fetch_paginated(params) + end - options = - if ("pleroma:report" not in options.include_types and - User.privileged?(user, :reports_manage_reports)) or - User.privileged?(user, :reports_manage_reports) do - options - else - options - |> Map.update(:exclude_types, ["pleroma:report"], fn current_exclude_types -> - current_exclude_types ++ ["pleroma:report"] - end) - end + def get_grouped_notification_page(user, params \\ %{}) do + grouped_types = + params + |> Map.get("grouped_types") + |> Notification.normalize_grouped_types() + + query = notifications_query(user, params) + {order, cursor_filters} = group_pagination(params) + + group_rows = + query + |> notification_group_rows(grouped_types, grouped_limit(params), order, cursor_filters) + + group_rows = if order == :asc, do: Enum.reverse(group_rows), else: group_rows + + page_notifications = representative_notifications(query, group_rows) + + notification_groups = + Enum.map(group_rows, ¬ification_group_sample(user, params, &1.group_key)) + + { + notification_groups, + page_notifications, + notification_group_counts(group_rows), + notification_group_bounds(group_rows) + } + end + + def get_notification_group_result(user, group_key, params \\ %{}) do + notifications = notification_group_sample(user, params, group_key) + metadata = notification_group_metadata(user, group_key, params) + + if Enum.empty?(notifications) or is_nil(metadata) do + {[], %{}, %{}} + else + { + notifications, + %{group_key => metadata.notifications_count}, + %{group_key => Map.drop(metadata, [:notifications_count])} + } + end + end + + def get_notification_group(user, group_key, params \\ %{}) + + def get_notification_group(user, "ungrouped-" <> notification_id, _params) do + case Notification.get(user, notification_id) do + {:ok, notification} -> [notification] + _ -> [] + end + end + + def get_notification_group(user, group_key, params) do + notification_group_sample(user, params, group_key) + end + + def get_notification_group_accounts(user, "ungrouped-" <> notification_id) do + with {:ok, notification} <- Notification.get(user, notification_id), + %User{} = actor <- User.get_cached_by_ap_id(notification.activity.data["actor"]) do + [actor] + else + _ -> [] + end + end + + def get_notification_group_accounts(user, group_key) do + user + |> notification_group_query(group_key, %{}) + |> exclude(:preload) + |> distinct(true) + |> select([user_actor: user_actor], user_actor) + |> Repo.all() + end + + def dismiss_notification_group(user, "ungrouped-" <> notification_id) do + Notification.destroy_multiple(user, [notification_id]) + end + + def dismiss_notification_group(%User{id: user_id}, group_key) do + Notification + |> where([n], n.user_id == ^user_id and n.group_key == ^group_key) + |> Repo.delete_all() + end + + defp group_pagination(params) do + cond do + min_id = Map.get(params, "min_id") -> + cursor_filters = [{:gt, min_id}] + + cursor_filters = + case Map.get(params, "max_id") do + nil -> cursor_filters + max_id -> [{:lt, max_id} | cursor_filters] + end + + {:asc, cursor_filters} + + since_id = Map.get(params, "since_id") -> + {:desc, [{:gt, since_id}]} + + max_id = Map.get(params, "max_id") -> + {:desc, [{:lt, max_id}]} + + true -> + {:desc, []} + end + end + + defp grouped_limit(params) do + params + |> Map.get("limit", 40) + |> parse_limit(40) + |> min(80) + end + + def unread_notification_group_count(user, params \\ %{}) do + grouped_types = + params + |> Map.get("grouped_types") + |> Notification.normalize_grouped_types() + + limit = unread_count_limit(params) + + user + |> notifications_query(params) + # The grouped API docs define unread by the notifications marker, not by Pleroma's per-row + # seen flag used by the v1 unread count. Keep this marker-based for Mastodon clients. + |> restrict_after_marker(notification_marker_last_read_id(user)) + |> notification_group_rows(grouped_types, limit, :desc, []) + |> length() + end + + defp notification_group_rows(query, grouped_types, group_limit, :desc, cursor_filters) do + query + |> notification_group_keyed_query(grouped_types) + |> group_by([n], n.group_key) + |> apply_group_cursor_filters(cursor_filters) + |> select([n], %{ + group_key: n.group_key, + representative_id: max(n.id), + notifications_count: count(n.id), + page_min_id: min(n.id), + page_max_id: max(n.id), + latest_page_notification_at: max(n.inserted_at) + }) + |> order_by([n], desc: max(n.id)) + |> limit(^group_limit) + |> Repo.all() + end + + defp notification_group_rows(query, grouped_types, group_limit, :asc, cursor_filters) do + query + |> notification_group_keyed_query(grouped_types) + |> group_by([n], n.group_key) + |> apply_group_cursor_filters(cursor_filters) + |> select([n], %{ + group_key: n.group_key, + representative_id: max(n.id), + notifications_count: count(n.id), + page_min_id: min(n.id), + page_max_id: max(n.id), + latest_page_notification_at: max(n.inserted_at) + }) + |> order_by([n], asc: max(n.id)) + |> limit(^group_limit) + |> Repo.all() + end + + defp apply_group_cursor_filters(query, []), do: query + + defp apply_group_cursor_filters(query, [{:gt, id} | rest]) do + query + |> having([n], max(n.id) > ^id) + |> apply_group_cursor_filters(rest) + end + + defp apply_group_cursor_filters(query, [{:lt, id} | rest]) do + query + |> having([n], max(n.id) < ^id) + |> apply_group_cursor_filters(rest) + end + + defp notification_group_keyed_query(query, grouped_types) do + query + |> exclude(:preload) + |> select([n], %{ + id: n.id, + inserted_at: n.inserted_at, + group_key: + fragment( + "CASE WHEN ? IS NOT NULL AND ?::text = ANY(?) THEN ? ELSE 'ungrouped-' || ?::text END", + n.group_key, + n.type, + type(^grouped_types, {:array, :string}), + n.group_key, + n.id + ) + }) + |> subquery() + end + + defp representative_notifications(_query, []), do: [] + + defp representative_notifications(query, group_rows) do + representative_ids = Enum.map(group_rows, & &1.representative_id) + + notifications_by_id = + query + |> where([n], n.id in ^representative_ids) + |> Repo.all() + |> Map.new(&{to_string(&1.id), &1}) + + group_rows + |> Enum.map(&Map.get(notifications_by_id, to_string(&1.representative_id))) + |> Enum.filter(& &1) + end + + defp notification_group_counts(group_rows) do + Map.new(group_rows, &{&1.group_key, &1.notifications_count}) + end + + defp notification_group_bounds(group_rows) do + Map.new(group_rows, fn row -> + {row.group_key, + %{ + page_min_id: row.page_min_id, + page_max_id: row.page_max_id, + latest_page_notification_at: row.latest_page_notification_at + }} + end) + end + + defp notification_group_sample(user, _params, "ungrouped-" <> notification_id) do + case Notification.get(user, notification_id) do + {:ok, notification} -> [notification] + _ -> [] + end + end + + defp notification_group_sample(user, params, group_key) do + user + |> notification_group_query(group_key, params) + |> order_by([n], desc: n.id) + |> limit(^@notification_group_sample_limit) + |> Repo.all() + end + + defp notification_group_metadata(user, "ungrouped-" <> notification_id, _params) do + case Notification.get(user, notification_id) do + {:ok, notification} -> + %{ + notifications_count: 1, + page_min_id: notification.id, + page_max_id: notification.id, + latest_page_notification_at: notification.inserted_at + } + + _ -> + nil + end + end + + defp notification_group_metadata(user, group_key, params) do + user + |> notification_group_query(group_key, params) + |> exclude(:preload) + |> select([n], %{ + notifications_count: count(n.id), + page_min_id: min(n.id), + page_max_id: max(n.id), + latest_page_notification_at: max(n.inserted_at) + }) + |> Repo.one() + |> case do + %{notifications_count: 0} -> nil + metadata -> metadata + end + end + + defp notification_group_query(user, group_key, params) do + user + |> notifications_query(params) + |> where([n], n.group_key == ^group_key) + end + + defp notification_marker_last_read_id(user) do + Marker + |> where([m], m.user_id == ^user.id and m.timeline == "notifications") + |> select([m], m.last_read_id) + |> Repo.one() + end + + defp restrict_after_marker(query, last_read_id) + when is_binary(last_read_id) and last_read_id != "" do + where(query, [n], n.id > ^last_read_id) + end + + defp restrict_after_marker(query, _last_read_id), do: query + + defp notifications_query(user, params) do + options = notification_options(user, params) user |> Notification.for_user_query(options) |> restrict(:types, options) |> restrict(:exclude_types, options) |> restrict(:account_ap_id, options) - |> Pagination.fetch_paginated(params) end + defp notification_options(user, params) do + options = + params + |> cast_params() + |> Map.update(:include_types, [], fn include_types -> include_types end) + + if "pleroma:chat_mention" in options.include_types and not Pleroma.Chat.enabled?() do + options + |> Map.update(:include_types, [], fn include_types -> + List.delete(include_types, "pleroma:chat_mention") + end) + else + options + end + + if ("pleroma:report" not in options.include_types and + User.privileged?(user, :reports_manage_reports)) or + User.privileged?(user, :reports_manage_reports) do + options + else + options + |> Map.update(:exclude_types, ["pleroma:report"], fn current_exclude_types -> + current_exclude_types ++ ["pleroma:report"] + end) + end + end + + defp unread_count_limit(params) do + params + |> Map.get("limit", 100) + |> parse_limit(100) + |> min(1000) + end + + defp parse_limit(limit, _default) when is_integer(limit) and limit > 0, do: limit + + defp parse_limit(limit, default) when is_binary(limit) do + case Integer.parse(limit) do + {limit, _} when limit > 0 -> limit + _ -> default + end + end + + defp parse_limit(_, default), do: default + def get_scheduled_activities(user, params \\ %{}) do user |> ScheduledActivity.for_user_query() @@ -95,6 +435,8 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPI do exclude_types: {:array, :string}, types: {:array, :string}, exclude_visibilities: {:array, :string}, + grouped_types: {:array, :string}, + limit: :integer, reblogs: :boolean, with_muted: :boolean, account_ap_id: :string, diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex index 5386c5a6c..1e3882fc6 100644 --- a/lib/pleroma/web/mastodon_api/views/account_view.ex +++ b/lib/pleroma/web/mastodon_api/views/account_view.ex @@ -287,6 +287,8 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do user.last_status_at && user.last_status_at |> NaiveDateTime.to_date() |> Date.to_iso8601() + accepts_chat_messages = Pleroma.Chat.enabled?() and user.accepts_chat_messages + %{ id: to_string(user.id), username: username_from_nickname(user.nickname), @@ -336,7 +338,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do relationship: relationship, skip_thread_containment: user.skip_thread_containment, background_image: image_url(user.background) |> MediaProxy.url(), - accepts_chat_messages: user.accepts_chat_messages, + accepts_chat_messages: accepts_chat_messages, favicon: favicon, avatar_description: avatar_description, header_description: header_description @@ -391,6 +393,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do |> Kernel.put_in([:source, :pleroma, :show_role], user.show_role) |> Kernel.put_in([:source, :pleroma, :no_rich_text], user.no_rich_text) |> Kernel.put_in([:source, :pleroma, :show_birthday], user.show_birthday) + |> Kernel.put_in([:source, :pleroma, :accepts_chat_messages], user.accepts_chat_messages) end defp maybe_put_settings(data, _, _, _), do: data diff --git a/lib/pleroma/web/mastodon_api/views/instance_view.ex b/lib/pleroma/web/mastodon_api/views/instance_view.ex index d64ce4fb5..437e42580 100644 --- a/lib/pleroma/web/mastodon_api/views/instance_view.ex +++ b/lib/pleroma/web/mastodon_api/views/instance_view.ex @@ -17,6 +17,8 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do reject: "suspend" } + def mastodon_api_level, do: @mastodon_api_level + def render("show.json", _) do instance = Config.get(:instance) @@ -166,6 +168,7 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do [ "pleroma_api", "mastodon_api", + "mastodon_api_grouped_notifications", "mastodon_api_streaming", "polls", "v2_suggestions", @@ -199,8 +202,12 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do end, "pleroma_emoji_reactions", "pleroma_custom_emoji_reactions", - "pleroma_chat_messages", - "pleroma:pin_chats", + if Pleroma.Chat.enabled?() do + "pleroma_chat_messages" + end, + if Pleroma.Chat.enabled?() do + "pleroma:pin_chats" + end, if Config.get([:instance, :show_reactions]) do "exposable_reactions" end, diff --git a/lib/pleroma/web/mastodon_api/views/list_view.ex b/lib/pleroma/web/mastodon_api/views/list_view.ex index 740f458d4..508169fc1 100644 --- a/lib/pleroma/web/mastodon_api/views/list_view.ex +++ b/lib/pleroma/web/mastodon_api/views/list_view.ex @@ -4,6 +4,8 @@ defmodule Pleroma.Web.MastodonAPI.ListView do use Pleroma.Web, :view + + alias Pleroma.Emoji alias Pleroma.Web.MastodonAPI.ListView def render("index.json", %{lists: lists} = opts) do @@ -11,10 +13,29 @@ defmodule Pleroma.Web.MastodonAPI.ListView do end def render("show.json", %{list: list}) do + {emoji, emoji_url} = get_emoji(list.emoji) + %{ id: to_string(list.id), title: list.title, - exclusive: list.exclusive + exclusive: list.exclusive, + pleroma: %{ + emoji: emoji, + emoji_url: emoji_url + } } end + + defp get_emoji(nil), do: {nil, nil} + + defp get_emoji(emoji) do + if Emoji.unicode?(emoji) do + {emoji, nil} + else + case Emoji.get(emoji) do + nil -> {nil, nil} + emoji_data -> {emoji, Emoji.local_url(emoji_data.file)} + end + end + end end diff --git a/lib/pleroma/web/mastodon_api/views/notification_view.ex b/lib/pleroma/web/mastodon_api/views/notification_view.ex index 27532c42a..33d4b2924 100644 --- a/lib/pleroma/web/mastodon_api/views/notification_view.ex +++ b/lib/pleroma/web/mastodon_api/views/notification_view.ex @@ -72,6 +72,58 @@ defmodule Pleroma.Web.MastodonAPI.NotificationView do safe_render_many(notifications, NotificationView, "show.json", opts) end + def render("grouped_index.json", %{notifications: notifications} = opts) do + grouped_types = Notification.normalize_grouped_types(opts[:grouped_types]) + + opts + |> Map.delete(:notifications) + |> Map.put( + :notification_groups, + Notification.group_notifications(notifications, grouped_types) + ) + |> then(&render("grouped_index.json", &1)) + end + + def render( + "grouped_index.json", + %{notification_groups: notification_groups, for: reading_user} = opts + ) do + grouped_types = Notification.normalize_grouped_types(opts[:grouped_types]) + notification_group_counts = Map.get(opts, :notification_group_counts, %{}) + notification_group_bounds = Map.get(opts, :notification_group_bounds, %{}) + include_page_metadata = Map.get(opts, :include_page_metadata, true) + + statuses = + notification_groups + |> Enum.map(&List.first/1) + |> Enum.map(&render("show.json", %{notification: &1, for: reading_user})) + |> Enum.map(& &1[:status]) + |> Enum.filter(& &1) + |> Enum.uniq_by(& &1[:id]) + + actors = + notification_groups + |> List.flatten() + |> notification_actors() + + %{ + accounts: AccountView.render("index.json", %{users: actors, for: reading_user}), + statuses: statuses, + notification_groups: + Enum.map( + notification_groups, + &render_group( + &1, + reading_user, + grouped_types, + notification_group_counts, + notification_group_bounds, + include_page_metadata + ) + ) + } + end + def render( "show.json", %{ @@ -95,7 +147,7 @@ defmodule Pleroma.Web.MastodonAPI.NotificationView do response = %{ id: to_string(notification.id), - group_key: "ungrouped-" <> to_string(notification.id), + group_key: Notification.group_key(notification), type: notification.type, created_at: CommonAPI.Utils.to_masto_date(notification.inserted_at), account: account, @@ -137,6 +189,104 @@ defmodule Pleroma.Web.MastodonAPI.NotificationView do Map.put(response, :report, report_render) end + defp render_group( + [%Notification{} = notification | _] = notifications, + _reading_user, + grouped_types, + notification_group_counts, + notification_group_bounds, + include_page_metadata + ) do + latest_notification = List.first(notifications) + oldest_notification = List.last(notifications) + status_activity = status_activity_for_group(notifications, grouped_types) + group_key = Notification.group_key(notification, grouped_types) + bounds = Map.get(notification_group_bounds, group_key, %{}) + + response = %{ + group_key: group_key, + notifications_count: Map.get(notification_group_counts, group_key, length(notifications)), + type: notification.type, + most_recent_notification_id: + bounds + |> Map.get(:page_max_id, latest_notification.id) + |> to_string(), + sample_account_ids: + notifications + |> notification_actors() + |> Enum.map(&to_string(&1.id)) + } + + response = + if include_page_metadata do + Map.merge(response, %{ + page_min_id: + bounds + |> Map.get(:page_min_id, oldest_notification.id) + |> to_string(), + page_max_id: + bounds + |> Map.get(:page_max_id, latest_notification.id) + |> to_string(), + latest_page_notification_at: + bounds + |> Map.get(:latest_page_notification_at, latest_notification.inserted_at) + |> CommonAPI.Utils.to_masto_date() + }) + else + response + end + + if status_activity do + Map.put(response, :status_id, to_string(status_activity.id)) + else + response + end + end + + defp status_activity_for_group([%Notification{} = notification | _], grouped_types) do + status_activity_for(notification, grouped_types) + end + + defp status_activity_for(%Notification{type: type, activity: activity}, _grouped_types) + when type in ["mention", "status", "poll"] do + activity + end + + defp status_activity_for(%Notification{type: type} = notification, grouped_types) + when type in ["favourite", "reblog"] do + group_key = Notification.group_key(notification, grouped_types) + + case String.split(group_key, "-", parts: 3) do + [^type, activity_id, _bucket] -> + case Activity.create_by_id_with_object(activity_id) do + %Activity{} = activity -> activity + _ -> parent_status_activity(notification.activity) + end + + _ -> + parent_status_activity(notification.activity) + end + end + + defp status_activity_for(%Notification{type: type, activity: activity}, _grouped_types) + when type in ["update", "pleroma:emoji_reaction"] do + parent_status_activity(activity) + end + + defp status_activity_for(_, _grouped_types), do: nil + + defp parent_status_activity(activity) do + Activity.get_create_by_object_ap_id(object_id_for(activity)) + end + + defp notification_actors(notifications) do + notifications + |> Enum.map(&User.get_cached_by_ap_id(&1.activity.data["actor"])) + |> Enum.filter(& &1) + |> Enum.uniq_by(& &1.id) + end + defp put_emoji(response, activity) do response |> Map.put(:emoji, activity.data["content"]) diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index 6949c5f2d..725dc2ce7 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -620,6 +620,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do String.contains?(media_type, "image") -> "image" String.contains?(media_type, "video") -> "video" String.contains?(media_type, "audio") -> "audio" + attachment["type"] == "Image" -> "image" true -> "unknown" end diff --git a/lib/pleroma/web/media_proxy/media_proxy_controller.ex b/lib/pleroma/web/media_proxy/media_proxy_controller.ex index a0aafc32e..93f692008 100644 --- a/lib/pleroma/web/media_proxy/media_proxy_controller.ex +++ b/lib/pleroma/web/media_proxy/media_proxy_controller.ex @@ -208,6 +208,7 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do defp media_proxy_opts do Config.get([:media_proxy, :proxy_opts], []) + |> Keyword.put_new(:sniff_content_type, true) end defp sandbox(conn, _params) do diff --git a/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex b/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex index ac9bb2779..9c9f17e86 100644 --- a/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex +++ b/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex @@ -41,6 +41,7 @@ defmodule Pleroma.Web.PleromaAPI.ChatController do ) plug(Pleroma.Web.ApiSpec.CastAndValidate, replace_params: false) + plug(:ensure_chats_enabled) defdelegate open_api_operation(action), to: Pleroma.Web.ApiSpec.ChatOperation @@ -244,4 +245,16 @@ defmodule Pleroma.Web.PleromaAPI.ChatController do _ -> nil end end + + defp ensure_chats_enabled(conn, _opts) do + if Chat.enabled?() do + conn + else + conn + |> put_status(:not_found) + |> put_view(Pleroma.Web.ErrorView) + |> render("404.json") + |> halt() + end + end end diff --git a/lib/pleroma/web/plugs/ensure_host_matches_plug.ex b/lib/pleroma/web/plugs/ensure_host_matches_plug.ex index ab2129f9e..960e86d4f 100644 --- a/lib/pleroma/web/plugs/ensure_host_matches_plug.ex +++ b/lib/pleroma/web/plugs/ensure_host_matches_plug.ex @@ -4,6 +4,7 @@ defmodule Pleroma.Web.Plugs.EnsureHostMatchesPlug do @moduledoc "Ensures Host header matches instance" + @behaviour Plug alias Pleroma.Web.Endpoint diff --git a/lib/pleroma/web/plugs/instance_static.ex b/lib/pleroma/web/plugs/instance_static.ex index 948188287..d2a674d39 100644 --- a/lib/pleroma/web/plugs/instance_static.ex +++ b/lib/pleroma/web/plugs/instance_static.ex @@ -4,7 +4,7 @@ defmodule Pleroma.Web.Plugs.InstanceStatic do require Pleroma.Constants - import Plug.Conn, only: [put_resp_header: 3, put_status: 2, send_resp: 3, halt: 1] + import Plug.Conn, only: [put_resp_header: 3] @moduledoc """ This is a shim to call `Plug.Static` but with runtime `from` configuration. @@ -51,37 +51,10 @@ defmodule Pleroma.Web.Plugs.InstanceStatic do |> Map.put(:from, from) |> Map.put(:content_types, false) - conn = - conn - |> set_content_type(conn.request_path) - |> Plug.Static.call(opts) + conn = set_content_type(conn, conn.request_path) - if conn.halted do - conn - else - path = String.trim_leading(conn.request_path, "/") - - if not File.exists?(file_path(path)) do - conn - |> put_status(:not_found) - |> send_404() - |> halt() - else - conn - end - end - end - - defp send_404(conn) do - if String.ends_with?(String.downcase(conn.request_path), ".json") do - conn - |> put_resp_header("content-type", "application/json") - |> send_resp(404, Jason.encode!(%{error: "not found"})) - else - conn - |> put_resp_header("content-type", "text/plain") - |> send_resp(404, "Not found") - end + # Call Plug.Static with our sanitized content-type + Plug.Static.call(conn, opts) end defp set_content_type(conn, "/emoji/" <> filepath) do diff --git a/lib/pleroma/web/plugs/o_auth_plug.ex b/lib/pleroma/web/plugs/o_auth_plug.ex index 488968691..b59ac9d3e 100644 --- a/lib/pleroma/web/plugs/o_auth_plug.ex +++ b/lib/pleroma/web/plugs/o_auth_plug.ex @@ -52,7 +52,7 @@ defmodule Pleroma.Web.Plugs.OAuthPlug do where: t.token == ^token ) - with %Token{user_id: user_id} = token_record <- Repo.one(token_query) |> Repo.preload(:user), + with %Token{user_id: user_id} = token_record <- Repo.one(token_query), false <- is_nil(user_id), %User{} = user <- User.get_cached_by_id(user_id) do {:ok, user, token_record} diff --git a/lib/pleroma/web/plugs/static_not_found_plug.ex b/lib/pleroma/web/plugs/static_not_found_plug.ex new file mode 100644 index 000000000..e2e7a622c --- /dev/null +++ b/lib/pleroma/web/plugs/static_not_found_plug.ex @@ -0,0 +1,38 @@ +# Pleroma: A lightweight social networking server +# Copyright ยฉ 2017-2026 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.Plugs.StaticNotFoundPlug do + @moduledoc """ + A small plug that enforces returning 404 on static files when they don't exist. + """ + @behaviour Plug + + require Pleroma.Constants + import Plug.Conn, only: [put_resp_header: 3, send_resp: 3, halt: 1] + + def init(opts), do: opts + + for only <- Pleroma.Constants.static_only_files() do + def call(%{request_path: "/" <> unquote(only) <> _} = conn, _opts) do + send_404(conn) + end + end + + def call(conn, _) do + conn + end + + defp send_404(conn) do + if String.ends_with?(String.downcase(conn.request_path), ".json") do + conn + |> put_resp_header("content-type", "application/json") + |> send_resp(404, Jason.encode!(%{error: "not found"})) + else + conn + |> put_resp_header("content-type", "text/plain") + |> send_resp(404, "Not found") + end + |> halt() + end +end diff --git a/lib/pleroma/web/rich_media/helpers.ex b/lib/pleroma/web/rich_media/helpers.ex index 963076510..7b4ca677f 100644 --- a/lib/pleroma/web/rich_media/helpers.ex +++ b/lib/pleroma/web/rich_media/helpers.ex @@ -4,6 +4,8 @@ defmodule Pleroma.Web.RichMedia.Helpers do alias Pleroma.Config + alias Pleroma.Gun.ConnectionPool + alias Pleroma.ReverseProxy.Client.Tesla, as: TeslaClient require Logger @@ -19,12 +21,30 @@ defmodule Pleroma.Web.RichMedia.Helpers do end defp stream(url) do - with {_, {:ok, %Tesla.Env{status: 200, body: stream_body, headers: headers}}} <- - {:get, Pleroma.HTTP.get(url, req_headers(), http_options())}, - {_, :ok} <- {:content_type, check_content_type(headers)}, - {_, :ok} <- {:content_length, check_content_length(headers)}, - {:read_stream, {:ok, body}} <- {:read_stream, read_stream(stream_body)} do - {:ok, body} + case Pleroma.HTTP.get(url, req_headers(), http_options()) do + {:ok, %Tesla.Env{status: 200, body: stream_body, headers: headers}} -> + result = + try do + with {_, :ok} <- {:content_type, check_content_type(headers)}, + {_, :ok} <- {:content_length, check_content_length(headers)}, + {:read_stream, {:ok, body}} <- {:read_stream, read_stream(stream_body)} do + {:ok, body} + end + rescue + exception -> + cleanup_stream(stream_body, :error) + reraise exception, __STACKTRACE__ + end + + cleanup_stream(stream_body, result) + result + + {:ok, %Tesla.Env{body: stream_body}} = response -> + cleanup_stream(stream_body, :rejected) + {:get, response} + + response -> + {:get, response} end end @@ -95,6 +115,10 @@ defmodule Pleroma.Web.RichMedia.Helpers do end end + defp read_stream(%{pid: pid, stream: stream, opts: opts}) do + read_chunks(pid, stream, opts, "", 0, Keyword.get(http_options(), :max_body)) + end + defp read_stream(stream) do max_body = Keyword.get(http_options(), :max_body) @@ -117,6 +141,33 @@ defmodule Pleroma.Web.RichMedia.Helpers do end end + defp read_chunks(pid, stream, opts, acc, total_bytes, max_body) do + case Tesla.Adapter.Gun.read_chunk(pid, stream, opts) do + {fin, chunk} when fin in [:fin, :nofin] -> + total_bytes = total_bytes + byte_size(chunk) + + cond do + total_bytes > max_body -> + :error + + fin == :fin -> + {:ok, acc <> chunk} + + true -> + read_chunks(pid, stream, opts, acc <> chunk, total_bytes, max_body) + end + + {:error, _reason} -> + :error + end + end + + defp cleanup_stream(%{pid: pid, stream: stream}, {:ok, _body}), + do: ConnectionPool.release_stream(pid, stream) + + defp cleanup_stream(%{pid: _pid} = client, _result), do: TeslaClient.close(client) + defp cleanup_stream(_stream, _result), do: :ok + defp http_options do [ pool: :rich_media, diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 813c5c482..5657bcad7 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -844,6 +844,16 @@ defmodule Pleroma.Web.Router do get("/directory", DirectoryController, :index) end + scope "/api/v2", Pleroma.Web.MastodonAPI do + pipe_through(:authenticated_api) + + get("/notifications", NotificationController, :grouped_index) + get("/notifications/unread_count", NotificationController, :unread_count) + get("/notifications/:group_key/accounts", NotificationController, :group_accounts) + get("/notifications/:group_key", NotificationController, :show_group) + post("/notifications/:group_key/dismiss", NotificationController, :dismiss_group) + end + scope "/api/v2", Pleroma.Web.MastodonAPI do pipe_through(:api) diff --git a/lib/pleroma/web/streamer.ex b/lib/pleroma/web/streamer.ex index 120e94751..3c154288b 100644 --- a/lib/pleroma/web/streamer.ex +++ b/lib/pleroma/web/streamer.ex @@ -195,6 +195,9 @@ defmodule Pleroma.Web.Streamer do recipients = MapSet.new(item.recipients) domain_blocks = Pleroma.Web.ActivityPub.MRF.subdomains_regex(user.domain_blocks) + following_ap_ids = + if user.domain_blocks == [], do: [], else: User.get_cached_user_friends_ap_ids(user) + with parent <- Object.normalize(item, fetch: false) || item, true <- Enum.all?([blocked_ap_ids, muted_ap_ids], &(item.actor not in &1)), true <- item.data["type"] != "Announce" || item.actor not in reblog_muted_ap_ids, @@ -205,8 +208,12 @@ defmodule Pleroma.Web.Streamer do true <- MapSet.disjoint?(recipients, recipient_blocks), %{host: item_host} <- URI.parse(item.actor), %{host: parent_host} <- URI.parse(parent.data["actor"]), - false <- Pleroma.Web.ActivityPub.MRF.subdomain_match?(domain_blocks, item_host), - false <- Pleroma.Web.ActivityPub.MRF.subdomain_match?(domain_blocks, parent_host), + false <- + item.actor not in following_ap_ids and + Pleroma.Web.ActivityPub.MRF.subdomain_match?(domain_blocks, item_host), + false <- + parent.data["actor"] not in following_ap_ids and + Pleroma.Web.ActivityPub.MRF.subdomain_match?(domain_blocks, parent_host), true <- thread_containment(item, user), false <- CommonAPI.thread_muted?(parent, user) do false @@ -289,15 +296,17 @@ defmodule Pleroma.Web.Streamer do defp do_stream(topic, {user, %MessageReference{} = cm_ref}) when topic in ["user", "user:pleroma_chat"] do - topic = "#{topic}:#{user.id}" + if Pleroma.Chat.enabled?() do + topic = "#{topic}:#{user.id}" - text = StreamerView.render("chat_update.json", %{chat_message_reference: cm_ref}, topic) + text = StreamerView.render("chat_update.json", %{chat_message_reference: cm_ref}, topic) - Registry.dispatch(@registry, topic, fn list -> - Enum.each(list, fn {pid, _auth} -> - send(pid, {:text, text}) + Registry.dispatch(@registry, topic, fn list -> + Enum.each(list, fn {pid, _auth} -> + send(pid, {:text, text}) + end) end) - end) + end end defp do_stream("user", %Activity{} = item) do diff --git a/lib/pleroma/web/templates/feed/feed/_activity.atom.eex b/lib/pleroma/web/templates/feed/feed/_activity.atom.eex index b774f7984..f94ba0793 100644 --- a/lib/pleroma/web/templates/feed/feed/_activity.atom.eex +++ b/lib/pleroma/web/templates/feed/feed/_activity.atom.eex @@ -1,9 +1,9 @@ http://activitystrea.ms/schema/1.0/note http://activitystrea.ms/schema/1.0/post - <%= @data["id"] %> + <%= escape(@data["id"] || "") %> <%= activity_title(@data, Keyword.get(@feed_config, :post_title, %{})) %> - <%= activity_description(@data) %> + <%= escape(atom_content_encoded(@data)) %> <%= to_rfc3339(@data["published"]) %> <%= to_rfc3339(@data["published"]) %> @@ -15,35 +15,26 @@ <%= escape(@data["summary"]) %> <% end %> - <%= if @activity.local do %> - - - <% else %> - - <% end %> + <%= for tag <- Pleroma.Object.hashtags(@object) do %> <% end %> - <%= for attachment <- @data["attachment"] || [] do %> - + <%= unless sensitive?(@data) do %> + <%= for attachment <- @data["attachment"] || [] do %> + <%= if size = attachment_size_positive(attachment) do %> + " type="<%= escape(attachment_type(attachment) || "application/octet-stream") %>" length="<%= size %>"/> + <% else %> + " type="<%= escape(attachment_type(attachment) || "application/octet-stream") %>"/> + <% end %> + <% end %> <% end %> <%= if @data["inReplyTo"] do %> <% end %> - <%= for id <- @activity.recipients do %> - <%= if id == Pleroma.Constants.as_public() do %> - - <% else %> - <%= unless Regex.match?(~r/^#{Pleroma.Web.Endpoint.url()}.+followers$/, id) do %> - - <% end %> - <% end %> - <% end %> - <%= for {emoji, file} <- @data["emoji"] || %{} do %> <% end %> diff --git a/lib/pleroma/web/templates/feed/feed/_activity.rss.eex b/lib/pleroma/web/templates/feed/feed/_activity.rss.eex index 7de98f736..4d71a45c7 100644 --- a/lib/pleroma/web/templates/feed/feed/_activity.rss.eex +++ b/lib/pleroma/web/templates/feed/feed/_activity.rss.eex @@ -1,45 +1,34 @@ http://activitystrea.ms/schema/1.0/note http://activitystrea.ms/schema/1.0/post - <%= @data["id"] %> + <%= escape(@data["id"] || "") %> <%= activity_title(@data, Keyword.get(@feed_config, :post_title, %{})) %> <%= activity_description(@data) %> + ]]> <%= to_rfc2822(@data["published"]) %> <%= activity_context(@activity) %> - <%= if @activity.local do %> - <%= @data["id"] %> - <% else %> - <%= @data["external_url"] %> - <% end %> - - <%= activity_context(@activity) %> + <%= escape(activity_link(@activity, @data) || "") %> <%= for tag <- Pleroma.Object.hashtags(@object) do %> - + <%= if tag && tag != "" do %> + <%= tag %> + <% end %> + <% end %> + + <%= unless sensitive?(@data) do %> + <%= with attachment when not is_nil(attachment) <- rss_enclosure_attachment(@data["attachment"]) do %> + " type="<%= escape(attachment_type(attachment) || "application/octet-stream") %>" length="<%= attachment_size_positive(attachment) %>" /> + <% end %> <% end %> <%= for attachment <- @data["attachment"] || [] do %> - + <%= media_content_xml(attachment, sensitive?(@data)) %> <% end %> <%= if @data["inReplyTo"] do %> <% end %> - - <%= for id <- @activity.recipients do %> - <%= if id == Pleroma.Constants.as_public() do %> - http://activityschema.org/collection/public - <% else %> - <%= unless Regex.match?(~r/^#{Pleroma.Web.Endpoint.url()}.+followers$/, id) do %> - <%= id %> - <% end %> - <% end %> - <% end %> - - <%= for {emoji, file} <- @data["emoji"] || %{} do %> - <%= file %> - <% end %> diff --git a/lib/pleroma/web/templates/feed/feed/_tag_activity.atom.eex b/lib/pleroma/web/templates/feed/feed/_tag_activity.atom.eex index 03c222975..6c74c2ab7 100644 --- a/lib/pleroma/web/templates/feed/feed/_tag_activity.atom.eex +++ b/lib/pleroma/web/templates/feed/feed/_tag_activity.atom.eex @@ -4,9 +4,9 @@ <%= render Phoenix.Controller.view_module(@conn), "_tag_author.atom", assigns %> - <%= @data["id"] %> + <%= escape(@data["id"] || "") %> <%= activity_title(@data, Keyword.get(@feed_config, :post_title, %{})) %> - <%= activity_description(@data) %> + <%= escape(atom_content_encoded(@data)) %> <%= to_rfc3339(@data["published"]) %> <%= to_rfc3339(@data["published"]) %> @@ -15,34 +15,25 @@ <%= if @data["summary"] != "" do %> - <%= @data["summary"] %> + <%= escape(@data["summary"]) %> <% end %> - <%= if @activity.local do %> - - - <% else %> - - <% end %> - - <%= for id <- @activity.recipients do %> - <%= if id == Pleroma.Constants.as_public() do %> - - <% else %> - <%= unless Regex.match?(~r/^#{Pleroma.Web.Endpoint.url()}.+followers$/, id) do %> - - <% end %> - <% end %> - <% end %> + <%= for tag <- Pleroma.Object.hashtags(@object) do %> <% end %> + <%= unless sensitive?(@data) do %> + <%= for attachment <- @data["attachment"] || [] do %> + <%= if size = attachment_size_positive(attachment) do %> + " type="<%= escape(attachment_type(attachment) || "application/octet-stream") %>" length="<%= size %>"/> + <% else %> + " type="<%= escape(attachment_type(attachment) || "application/octet-stream") %>"/> + <% end %> + <% end %> + <% end %> + <%= for {emoji, file} <- @data["emoji"] || %{} do %> <% end %> diff --git a/lib/pleroma/web/templates/feed/feed/_tag_activity.xml.eex b/lib/pleroma/web/templates/feed/feed/_tag_activity.xml.eex index 1b8c34b87..64e4c1d81 100644 --- a/lib/pleroma/web/templates/feed/feed/_tag_activity.xml.eex +++ b/lib/pleroma/web/templates/feed/feed/_tag_activity.xml.eex @@ -1,14 +1,24 @@ - <%= activity_title(@data, Keyword.get(@feed_config, :post_title, %{})) %> - - - <%= activity_context(@activity) %> - <%= activity_context(@activity) %> + <%= escape(activity_link(@activity, @data) || "") %> + <%= escape(activity_link(@activity, @data) || "") %> <%= to_rfc2822(@data["published"]) %> - + <%= activity_title(@data, Keyword.get(@feed_config, :post_title, %{})) %> <%= activity_description(@data) %> - <%= for attachment <- @data["attachment"] || [] do %> - + ]]> + + <%= unless sensitive?(@data) do %> + <%= with attachment when not is_nil(attachment) <- rss_enclosure_attachment(@data["attachment"]) do %> + " type="<%= escape(attachment_type(attachment) || "application/octet-stream") %>" length="<%= attachment_size_positive(attachment) %>"/> + <% end %> <% end %> + <%= for attachment <- @data["attachment"] || [] do %> + <%= media_content_xml(attachment, sensitive?(@data)) %> + <% end %> + + <%= for tag <- Pleroma.Object.hashtags(@object) do %> + <%= if tag && tag != "" do %> + <%= tag %> + <% end %> + <% end %> diff --git a/lib/pleroma/web/templates/feed/feed/tag.atom.eex b/lib/pleroma/web/templates/feed/feed/tag.atom.eex index 3449c97ff..fb04544d3 100644 --- a/lib/pleroma/web/templates/feed/feed/tag.atom.eex +++ b/lib/pleroma/web/templates/feed/feed/tag.atom.eex @@ -9,10 +9,12 @@ <%= Routes.tag_feed_url(@conn, :feed, @tag) <> ".atom" %> #<%= @tag %> - <%= Gettext.dpgettext("static_pages", "tag feed description", "These are public toots tagged with #%{tag}. You can interact with them if you have an account anywhere in the fediverse.", tag: @tag) %> + <%= with subtitle when subtitle not in [nil, ""] <- Gettext.dpgettext("static_pages", "tag feed description", "These are public toots tagged with #%{tag}. You can interact with them if you have an account anywhere in the fediverse.", tag: @tag) do %> + <%= subtitle %> + <% end %> <%= feed_logo() %> - <%= most_recent_update(@activities) %> - " type="application/atom+xml"/> + <%= most_recent_update(@activities) || to_rfc3339(NaiveDateTime.utc_now()) %> + <%= for activity <- @activities do %> <%= render Phoenix.Controller.view_module(@conn), "_tag_activity.atom", Map.merge(assigns, prepare_activity(activity, actor: true)) %> diff --git a/lib/pleroma/web/templates/feed/feed/tag.rss.eex b/lib/pleroma/web/templates/feed/feed/tag.rss.eex index a87f9bf50..b74a573e5 100644 --- a/lib/pleroma/web/templates/feed/feed/tag.rss.eex +++ b/lib/pleroma/web/templates/feed/feed/tag.rss.eex @@ -1,6 +1,8 @@ @@ -8,7 +10,12 @@ <%= Gettext.dpgettext("static_pages", "tag feed description", "These are public toots tagged with #%{tag}. You can interact with them if you have an account anywhere in the fediverse.", tag: @tag) %> <%= "#{Routes.tag_feed_url(@conn, :feed, @tag)}.rss" %> <%= feed_logo() %> + <%= feed_logo() %> 2b90d9 + <%= with last_build_date when not is_nil(last_build_date) <- most_recent_update(@activities, :rss) do %> + <%= last_build_date %> + <% end %> + <%= Pleroma.Application.named_version() %> <%= for activity <- @activities do %> <%= render Phoenix.Controller.view_module(@conn), "_tag_activity.xml", Map.merge(assigns, prepare_activity(activity)) %> <% end %> diff --git a/lib/pleroma/web/templates/feed/feed/user.atom.eex b/lib/pleroma/web/templates/feed/feed/user.atom.eex index c2c77cfed..cc6883727 100644 --- a/lib/pleroma/web/templates/feed/feed/user.atom.eex +++ b/lib/pleroma/web/templates/feed/feed/user.atom.eex @@ -8,10 +8,12 @@ <%= Routes.user_feed_url(@conn, :feed, @user.nickname) <> ".atom" %> <%= @user.nickname <> "'s timeline" %> - <%= escape(@user.bio) %> + <%= if @user.bio && @user.bio != "" do %> + <%= escape(@user.bio) %> + <% end %> <%= most_recent_update(@activities, @user, :atom) %> <%= logo(@user) %> - " type="application/atom+xml"/> + <%= render Phoenix.Controller.view_module(@conn), "_author.atom", assigns %> diff --git a/lib/pleroma/web/templates/feed/feed/user.rss.eex b/lib/pleroma/web/templates/feed/feed/user.rss.eex index b907a7e57..e2150d8b6 100644 --- a/lib/pleroma/web/templates/feed/feed/user.rss.eex +++ b/lib/pleroma/web/templates/feed/feed/user.rss.eex @@ -1,26 +1,32 @@ <%= @user.nickname <> "'s timeline" %> - <%= "#{Routes.user_feed_url(@conn, :feed, @user.nickname)}.rss" %> - " + + <%= "#{Routes.user_feed_url(@conn, :feed, @user.nickname)}.rss" %> <%= escape(@user.bio) %> <%= logo(@user) %> <%= @user.nickname <> "'s timeline" %> <%= "#{Routes.user_feed_url(@conn, :feed, @user.nickname)}.rss" %> + <%= most_recent_update(@activities, @user, :rss) %> + <%= logo(@user) %> + <%= Pleroma.Application.named_version() %> <%= render Phoenix.Controller.view_module(@conn), "_author.rss", assigns %> <%= if last_activity(@activities) do %> - <%= "#{Routes.user_feed_url(@conn, :feed, @user.nickname)}.rss?max_id=#{last_activity(@activities).id}" %> +