diff --git a/.woodpecker/changelog.yaml b/.woodpecker/changelog.yaml index 64062f17e..a3732c240 100644 --- a/.woodpecker/changelog.yaml +++ b/.woodpecker/changelog.yaml @@ -1,5 +1,6 @@ when: - event: pull_request + evaluate: 'CI_COMMIT_SOURCE_BRANCH != "weblate"' labels: platform: linux/amd64 diff --git a/.woodpecker/docker-armv7.yaml b/.woodpecker/docker-armv7.yaml new file mode 100644 index 000000000..c921a56f9 --- /dev/null +++ b/.woodpecker/docker-armv7.yaml @@ -0,0 +1,85 @@ +when: + # Temporary PR validation while arm/v7 Docker builds are being brought up. + - event: pull_request + path: [ "**/*.ex", "**/*.eex", "**/*.exs", "mix.lock", ".woodpecker/**", "Dockerfile" ] + - event: push + branch: ${CI_REPO_DEFAULT_BRANCH} + path: [ "**/*.ex", "**/*.eex", "**/*.exs", "mix.lock", ".woodpecker/**", "Dockerfile" ] + - event: tag + - event: manual + branch: ${CI_REPO_DEFAULT_BRANCH} + - event: manual + branch: stable + +# Target the CAX21 arm64 host with native arm32 userspace support, not generic arm64 runners. +labels: + platform: linux/arm + armv7: native + +variables: + docker_variables: &docker_variables + repo: pleroma/pleroma + registry: git.pleroma.social + armv7_build_settings: &armv7_build_settings + <<: *docker_variables + build_args: + - ELIXIR_IMG=arm32v7/elixir + - ELIXIR_TAG=1.17.3-otp-26-alpine + - ALPINE_IMG=arm32v7/alpine + - ALPINE_VER=3.20 + extra_opts: --custom-platform=linux/arm/v7 + armv7_push_settings: &armv7_push_settings + <<: *armv7_build_settings + username: + from_secret: pleroma-ci-user + password: + from_secret: pleroma-ci-password + +steps: + docker-armv7-pr: + image: woodpeckerci/plugin-kaniko:2.3.1 + when: + - event: pull_request + settings: + <<: *armv7_build_settings + dry_run: true + tags: + - armv7-pr-check + + docker-develop-armv7: + image: woodpeckerci/plugin-kaniko:2.3.1 + when: + - event: push + branch: ${CI_REPO_DEFAULT_BRANCH} + - event: manual + branch: ${CI_REPO_DEFAULT_BRANCH} + settings: + <<: *armv7_push_settings + tags: + - latest-armv7 + - develop-armv7 + - ${CI_COMMIT_SHA:0:8}-armv7 + + docker-stable-armv7: + image: woodpeckerci/plugin-kaniko:2.3.1 + when: + - evaluate: 'CI_PIPELINE_EVENT == "manual" && CI_COMMIT_BRANCH == "stable" && CI_COMMIT_TAG == ""' + settings: + <<: *armv7_push_settings + tags: + - latest-armv7 + - stable-armv7 + - ${CI_COMMIT_SHA:0:8}-armv7 + + docker-stable-tag-armv7: + image: woodpeckerci/plugin-kaniko:2.3.1 + when: + - event: tag + - evaluate: 'CI_PIPELINE_EVENT == "manual" && CI_COMMIT_BRANCH == "stable" && CI_COMMIT_TAG != ""' + settings: + <<: *armv7_push_settings + tags: + - latest-armv7 + - stable-armv7 + - ${CI_COMMIT_SHA:0:8}-armv7 + - ${CI_COMMIT_TAG}-armv7 diff --git a/.woodpecker/docker.yaml b/.woodpecker/docker.yaml index abc6bfa3b..9a5be78e2 100644 --- a/.woodpecker/docker.yaml +++ b/.woodpecker/docker.yaml @@ -62,6 +62,18 @@ steps: - stable-amd64 - ${CI_COMMIT_SHA:0:8}-amd64 + # FE workflow runs only on linux/amd64 + docker-stable-e2e-amd64: + image: *kaniko_image + when: + - evaluate: 'platform == "linux/amd64" && CI_PIPELINE_EVENT == "tag"' + - evaluate: 'platform == "linux/amd64" && CI_PIPELINE_EVENT == "manual" && CI_COMMIT_BRANCH == "stable"' + settings: + <<: *docker_variables + dockerfile: Dockerfile-e2e + tags: + - stable-e2e + docker-stable-tag-amd64: image: *kaniko_image when: diff --git a/.woodpecker/lint.yaml b/.woodpecker/lint.yaml index 0ab7441a8..8a5718c5d 100644 --- a/.woodpecker/lint.yaml +++ b/.woodpecker/lint.yaml @@ -1,6 +1,7 @@ when: - event: pull_request path: [ "**/*.ex", "**/*.eex", "**/*.exs", "mix.lock", ".woodpecker/**" ] + evaluate: 'CI_COMMIT_SOURCE_BRANCH != "weblate"' - event: push branch: ${CI_REPO_DEFAULT_BRANCH} path: [ "**/*.ex", "**/*.eex", "**/*.exs", "mix.lock", ".woodpecker/**" ] diff --git a/.woodpecker/unit-testing-elixir-1.15.yaml b/.woodpecker/unit-testing-elixir-1.15.yaml index a4a8fc266..a5bcc0ef3 100644 --- a/.woodpecker/unit-testing-elixir-1.15.yaml +++ b/.woodpecker/unit-testing-elixir-1.15.yaml @@ -1,6 +1,7 @@ when: - event: pull_request path: [ "**/*.ex", "**/*.eex", "**/*.exs", "mix.lock", ".woodpecker/**" ] + evaluate: 'CI_COMMIT_SOURCE_BRANCH != "weblate"' - event: push branch: ${CI_REPO_DEFAULT_BRANCH} path: [ "**/*.ex", "**/*.eex", "**/*.exs", "mix.lock", ".woodpecker/**" ] diff --git a/.woodpecker/unit-testing-elixir-1.19.yaml b/.woodpecker/unit-testing-elixir-1.19.yaml index 462e2ee5d..efab97223 100644 --- a/.woodpecker/unit-testing-elixir-1.19.yaml +++ b/.woodpecker/unit-testing-elixir-1.19.yaml @@ -1,6 +1,7 @@ when: - event: pull_request path: [ "**/*.ex", "**/*.eex", "**/*.exs", "mix.lock", ".woodpecker/**" ] + evaluate: 'CI_COMMIT_SOURCE_BRANCH != "weblate"' - event: push branch: ${CI_REPO_DEFAULT_BRANCH} path: [ "**/*.ex", "**/*.eex", "**/*.exs", "mix.lock", ".woodpecker/**" ] diff --git a/Dockerfile b/Dockerfile index 11a17b7ad..5fd1cc0a1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,8 +3,10 @@ ARG ELIXIR_IMG=hexpm/elixir ARG ELIXIR_VER=1.17.3 ARG ERLANG_VER=26.2.5.6 ARG ALPINE_VER=3.17.9 +ARG ELIXIR_TAG=${ELIXIR_VER}-erlang-${ERLANG_VER}-alpine-${ALPINE_VER} +ARG ALPINE_IMG=alpine -FROM ${ELIXIR_IMG}:${ELIXIR_VER}-erlang-${ERLANG_VER}-alpine-${ALPINE_VER} AS build +FROM ${ELIXIR_IMG}:${ELIXIR_TAG} AS build COPY . . @@ -20,7 +22,7 @@ RUN apk add git gcc g++ musl-dev make cmake file-dev vips-dev &&\ mkdir release &&\ mix release --path release -FROM alpine:${ALPINE_VER} +FROM ${ALPINE_IMG}:${ALPINE_VER} ARG BUILD_DATE ARG VCS_REF diff --git a/Dockerfile-e2e b/Dockerfile-e2e new file mode 100644 index 000000000..df8bb39aa --- /dev/null +++ b/Dockerfile-e2e @@ -0,0 +1,61 @@ +# https://hub.docker.com/r/hexpm/elixir/tags +ARG ELIXIR_IMG=docker.io/hexpm/elixir +ARG ALPINE_IMG=docker.io/alpine +ARG ELIXIR_VER=1.18.4 +ARG ERLANG_VER=27.3.4.11 +ARG ALPINE_VER=3.23.4 + +FROM ${ELIXIR_IMG}:${ELIXIR_VER}-erlang-${ERLANG_VER}-alpine-${ALPINE_VER} AS build + +COPY . . + +ENV MIX_ENV=prod +ENV VIX_COMPILATION_MODE=PLATFORM_PROVIDED_LIBVIPS + +RUN apk add git gcc g++ musl-dev make cmake file-dev vips-dev &&\ + echo "import Config" > config/prod.secret.exs &&\ + mix local.hex --force &&\ + mix local.rebar --force &&\ + mix deps.clean --all &&\ + mix deps.get --only prod &&\ + mkdir release &&\ + mix release --path release + +FROM ${ALPINE_IMG}:${ALPINE_VER} + +ARG BUILD_DATE +ARG VCS_REF + +LABEL maintainer="ops@pleroma.social" \ + org.opencontainers.image.title="pleroma" \ + org.opencontainers.image.description="Pleroma FE E2E test image" \ + org.opencontainers.image.authors="ops@pleroma.social" \ + org.opencontainers.image.vendor="pleroma.social" \ + org.opencontainers.image.documentation="https://git.pleroma.social/pleroma/pleroma" \ + org.opencontainers.image.licenses="AGPL-3.0" \ + org.opencontainers.image.url="https://pleroma.social" \ + org.opencontainers.image.revision=$VCS_REF \ + org.opencontainers.image.created=$BUILD_DATE + +ARG HOME=/opt/pleroma +ARG DATA=/var/lib/pleroma + +RUN apk update &&\ + apk add exiftool ffmpeg vips libmagic ncurses postgresql-client &&\ + adduser --system --shell /bin/false --home ${HOME} pleroma &&\ + mkdir -p ${DATA}/uploads &&\ + mkdir -p ${DATA}/static &&\ + chown -R pleroma ${DATA} &&\ + mkdir -p /etc/pleroma &&\ + chown -R pleroma /etc/pleroma + +USER pleroma + +COPY --from=build --chown=pleroma:0 /release ${HOME} + +COPY --chown=pleroma --chmod=640 ./config/docker.exs /etc/pleroma/config.exs +COPY ./docker-entrypoint-e2e.sh ${HOME} + +EXPOSE 4000 + +ENTRYPOINT ["/opt/pleroma/docker-entrypoint-e2e.sh"] diff --git a/changelog.d/docker-e2e-image.skip b/changelog.d/docker-e2e-image.skip new file mode 100644 index 000000000..e69de29bb diff --git a/changelog.d/featured-collection-page.fix b/changelog.d/featured-collection-page.fix new file mode 100644 index 000000000..b611a4137 --- /dev/null +++ b/changelog.d/featured-collection-page.fix @@ -0,0 +1 @@ +Support fetching featured collection items from the first collection page diff --git a/changelog.d/grouped-notifications-api.add b/changelog.d/grouped-notifications-api.add new file mode 100644 index 000000000..8c2cc5397 --- /dev/null +++ b/changelog.d/grouped-notifications-api.add @@ -0,0 +1 @@ +Add Mastodon-compatible grouped notifications API endpoints under `/api/v2/notifications` for newly-created notifications diff --git a/changelog.d/grouped_notifications_flag.skip b/changelog.d/grouped_notifications_flag.skip new file mode 100644 index 000000000..e69de29bb diff --git a/changelog.d/mastodon-websocket-protocol.fix b/changelog.d/mastodon-websocket-protocol.fix new file mode 100644 index 000000000..66dab16ed --- /dev/null +++ b/changelog.d/mastodon-websocket-protocol.fix @@ -0,0 +1 @@ +Echo Mastodon-style `Sec-WebSocket-Protocol` tokens in streaming WebSocket handshakes. diff --git a/changelog.d/mfm-extend.skip b/changelog.d/mfm-extend.skip new file mode 100644 index 000000000..e69de29bb diff --git a/changelog.d/weblate-ci.skip b/changelog.d/weblate-ci.skip new file mode 100644 index 000000000..e69de29bb diff --git a/changelog.d/wss-necroposts.fix b/changelog.d/wss-necroposts.fix new file mode 100644 index 000000000..743f9087c --- /dev/null +++ b/changelog.d/wss-necroposts.fix @@ -0,0 +1 @@ +RichMedia: Fix backfill causing old posts to show up on timelines by disabling it in MastoAPI StatusView diff --git a/docker-entrypoint-e2e.sh b/docker-entrypoint-e2e.sh new file mode 100755 index 000000000..d7aa332f7 --- /dev/null +++ b/docker-entrypoint-e2e.sh @@ -0,0 +1,70 @@ +#!/bin/ash +set -eu + +SEED_SENTINEL_PATH=/var/lib/pleroma/.e2e_seeded +CONFIG_OVERRIDE_PATH=/var/lib/pleroma/config.exs + +echo '-- Waiting for database...' +while ! pg_isready -U "${DB_USER:-pleroma}" -d "postgres://${DB_HOST:-db}:${DB_PORT:-5432}/${DB_NAME:-pleroma}" -t 1; do + sleep 1s +done + +echo '-- Writing E2E config overrides...' +cat > "$CONFIG_OVERRIDE_PATH" </dev/null; then + kill -TERM "$PLEROMA_PID" + wait "$PLEROMA_PID" || true + fi +} + +trap cleanup INT TERM + +echo '-- Waiting for API...' +api_ok=false +for _i in $(seq 1 120); do + if wget -qO- http://127.0.0.1:4000/api/v1/instance >/dev/null 2>&1; then + api_ok=true + break + fi + sleep 1s +done + +if [ "$api_ok" != true ]; then + echo 'Timed out waiting for Pleroma API to become available' + exit 1 +fi + +wait "$PLEROMA_PID" diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex index b8693c3a8..afd750823 100644 --- a/lib/pleroma/notification.ex +++ b/lib/pleroma/notification.ex @@ -35,6 +35,7 @@ defmodule Pleroma.Notification do # remember to add a migration to add it to the `notifications_type` enum # as well. field(:type, :string) + field(:group_key, :string) belongs_to(:user, User, type: FlakeId.Ecto.CompatType) belongs_to(:activity, Activity, type: FlakeId.Ecto.CompatType) @@ -49,7 +50,7 @@ defmodule Pleroma.Notification do |> type_from_activity() notification - |> changeset(%{type: type}) + |> changeset(%{type: type, group_key: grouped_notification_key(type, activity)}) |> Repo.update() end end @@ -62,6 +63,99 @@ defmodule Pleroma.Notification do |> Repo.aggregate(:count, :id) end + @groupable_notification_types ~w{favourite follow reblog} + @group_bucket_seconds 12 * 60 * 60 + + def groupable_notification_types, do: @groupable_notification_types + + def group_notifications(notifications, grouped_types \\ nil) do + grouped_types = normalize_grouped_types(grouped_types) + + {group_keys, grouped_notifications} = + Enum.reduce(notifications, {[], %{}}, fn notification, + {group_keys, grouped_notifications} -> + group_key = group_key(notification, grouped_types) + + if Map.has_key?(grouped_notifications, group_key) do + {group_keys, Map.update!(grouped_notifications, group_key, &[notification | &1])} + else + {[group_key | group_keys], Map.put(grouped_notifications, group_key, [notification])} + end + end) + + group_keys + |> Enum.reverse() + |> Enum.map(fn group_key -> + grouped_notifications + |> Map.fetch!(group_key) + |> Enum.reverse() + end) + end + + def group_key(notification, grouped_types \\ nil) + + def group_key(%Notification{} = notification, nil) do + group_key(notification, @groupable_notification_types) + end + + def group_key(%Notification{type: type, group_key: group_key} = notification, grouped_types) do + grouped_types = normalize_grouped_types(grouped_types) + + if type in @groupable_notification_types and type in grouped_types and is_binary(group_key) do + group_key + else + ungrouped_group_key(notification) + end + end + + def normalize_grouped_types(nil), do: @groupable_notification_types + def normalize_grouped_types(types) when is_list(types), do: types + def normalize_grouped_types(type), do: [type] + + defp ungrouped_group_key(%Notification{id: id}), do: "ungrouped-#{id}" + + defp grouped_notification_key(type, activity) when type in @groupable_notification_types do + with target_id when is_binary(target_id) <- group_target_id(type, activity) do + # Mastodon uses Redis to reuse a recent bucket for rolling 12h windows. Pleroma keeps keys + # deterministic and self-contained; clients must treat group_key as opaque either way. + "#{type}-#{target_id}-#{group_time_bucket(activity)}" + end + end + + defp grouped_notification_key(_type, _activity), do: nil + + defp group_time_bucket(%Activity{inserted_at: inserted_at}) when not is_nil(inserted_at) do + inserted_at + |> NaiveDateTime.to_erl() + |> :calendar.datetime_to_gregorian_seconds() + |> div(@group_bucket_seconds) + end + + defp group_time_bucket(_), + do: group_time_bucket(%Activity{inserted_at: NaiveDateTime.utc_now()}) + + defp group_target_id(type, activity) when type in ["favourite", "reblog"] do + with object_id when is_binary(object_id) <- object_id_for(activity), + %Activity{id: id} <- Activity.get_create_by_object_ap_id(object_id) do + to_string(id) + else + _ -> nil + end + end + + defp group_target_id("follow", %{data: %{"object" => ap_id}}) do + case User.get_cached_by_ap_id(ap_id) do + %User{id: id} -> to_string(id) + _ -> nil + end + end + + defp group_target_id(_, _), do: nil + + defp object_id_for(%{data: %{"object" => %{"id" => id}}}) when is_binary(id), do: id + defp object_id_for(%{data: %{"object" => id}}) when is_binary(id), do: id + defp object_id_for(_), do: nil + @notification_types ~w{ favourite follow @@ -79,7 +173,7 @@ defmodule Pleroma.Notification do def changeset(%Notification{} = notification, attrs) do notification - |> cast(attrs, [:seen, :type]) + |> cast(attrs, [:seen, :type, :group_key]) |> validate_inclusion(:type, @notification_types) end @@ -458,7 +552,8 @@ defmodule Pleroma.Notification do user_id: user.id, activity: activity, seen: mark_as_read?(activity, user), - type: type + type: type, + group_key: grouped_notification_key(type, activity) }) |> Marker.multi_set_last_read_id(user, "notifications") |> Repo.transaction() diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 0b513ee16..1e8bfe3dd 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -1881,11 +1881,15 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do end end + @featured_collection_types ["OrderedCollection", "Collection"] + @featured_collection_page_types ["OrderedCollectionPage", "CollectionPage"] + @featured_collection_item_types @featured_collection_types ++ @featured_collection_page_types + def pin_data_from_featured_collection(%{ "type" => type, "orderedItems" => objects }) - when type in ["OrderedCollection", "Collection"] do + when type in @featured_collection_item_types do Map.new(objects, fn %{"id" => object_ap_id} -> {object_ap_id, NaiveDateTime.utc_now()} object_ap_id when is_binary(object_ap_id) -> {object_ap_id, NaiveDateTime.utc_now()} @@ -1903,7 +1907,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do def fetch_and_prepare_featured_from_ap_id(ap_id) do with {:ok, data} <- Fetcher.fetch_and_contain_remote_object_from_id(ap_id) do - {:ok, pin_data_from_featured_collection(data)} + {:ok, prepare_featured_collection(data)} else e -> Logger.error("Could not decode featured collection at fetch #{ap_id}, #{inspect(e)}") @@ -1911,6 +1915,34 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do end end + defp prepare_featured_collection(%{"orderedItems" => objects} = data) when is_list(objects) do + pin_data_from_featured_collection(data) + end + + defp prepare_featured_collection(%{ + "type" => type, + "first" => %{"type" => page_type} = first + }) + when type in @featured_collection_types and page_type in @featured_collection_page_types do + pin_data_from_featured_collection(first) + end + + defp prepare_featured_collection(%{"type" => type, "first" => first}) + when type in @featured_collection_types and is_binary(first) do + case Fetcher.fetch_and_contain_remote_object_from_id(first) do + {:ok, data} -> + pin_data_from_featured_collection(data) + + e -> + Logger.error("Could not decode featured collection page at fetch #{first}, #{inspect(e)}") + %{} + end + end + + defp prepare_featured_collection(data) do + pin_data_from_featured_collection(data) + end + def enqueue_pin_fetches(%{pinned_objects: pins}) do # enqueue a task to fetch all pinned objects Enum.each(pins, fn {ap_id, _} -> diff --git a/lib/pleroma/web/api_spec/operations/notification_operation.ex b/lib/pleroma/web/api_spec/operations/notification_operation.ex index 94d1f6b82..6c8f7d5e9 100644 --- a/lib/pleroma/web/api_spec/operations/notification_operation.ex +++ b/lib/pleroma/web/api_spec/operations/notification_operation.ex @@ -3,7 +3,6 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.ApiSpec.NotificationOperation do - alias OpenApiSpex.Operation alias OpenApiSpex.Operation alias OpenApiSpex.Schema alias Pleroma.Web.ApiSpec.Schemas.Account @@ -91,6 +90,79 @@ defmodule Pleroma.Web.ApiSpec.NotificationOperation do } end + def grouped_index_operation do + %Operation{ + tags: ["Notifications"], + summary: "Retrieve grouped notifications", + description: "Notifications concerning the user, grouped by supported notification types.", + operationId: "NotificationController.grouped_index", + security: [%{"oAuth" => ["read:notifications"]}], + parameters: grouped_notification_params() ++ pagination_params(), + responses: %{ + 200 => Operation.response("Grouped notifications", "application/json", grouped_result()), + 404 => Operation.response("Error", "application/json", ApiError) + } + } + end + + def show_group_operation do + %Operation{ + tags: ["Notifications"], + summary: "Retrieve a notification group", + operationId: "NotificationController.show_group", + security: [%{"oAuth" => ["read:notifications"]}], + parameters: [group_key_param()], + responses: %{ + 200 => Operation.response("Grouped notifications", "application/json", grouped_result()), + 404 => Operation.response("Error", "application/json", ApiError) + } + } + end + + def group_accounts_operation do + %Operation{ + tags: ["Notifications"], + summary: "Retrieve notification group accounts", + operationId: "NotificationController.group_accounts", + security: [%{"oAuth" => ["read:notifications"]}], + parameters: [group_key_param()], + responses: %{ + 200 => + Operation.response("Accounts", "application/json", %Schema{ + type: :array, + items: Account + }) + } + } + end + + def unread_count_operation do + %Operation{ + tags: ["Notifications"], + summary: "Retrieve unread grouped notification count", + operationId: "NotificationController.unread_count", + security: [%{"oAuth" => ["read:notifications"]}], + parameters: + grouped_notification_params() ++ + [ + Operation.parameter( + :limit, + :query, + %Schema{type: :integer}, + "Maximum number of notifications to count" + ) + ], + responses: %{ + 200 => + Operation.response("Unread notification group count", "application/json", %Schema{ + type: :object, + properties: %{count: %Schema{type: :integer}} + }), + 404 => Operation.response("Error", "application/json", ApiError) + } + } + end + def clear_operation do %Operation{ tags: ["Notifications"], @@ -114,6 +186,18 @@ defmodule Pleroma.Web.ApiSpec.NotificationOperation do } end + def dismiss_group_operation do + %Operation{ + tags: ["Notifications"], + summary: "Dismiss a notification group", + description: "Clear all notifications in a notification group from the server.", + operationId: "NotificationController.dismiss_group", + parameters: [group_key_param()], + security: [%{"oAuth" => ["write:notifications"]}], + responses: %{200 => empty_object_response()} + } + end + def dismiss_via_body_operation do %Operation{ tags: ["Notifications"], @@ -184,7 +268,7 @@ defmodule Pleroma.Web.ApiSpec.NotificationOperation do }, example: %{ "id" => "34975861", - "group-key" => "ungrouped-34975861", + "group_key" => "ungrouped-34975861", "type" => "mention", "created_at" => "2019-11-23T07:49:02.064Z", "account" => Account.schema().example, @@ -194,6 +278,73 @@ defmodule Pleroma.Web.ApiSpec.NotificationOperation do } end + defp grouped_result do + %Schema{ + title: "GroupedNotificationsResults", + type: :object, + properties: %{ + accounts: %Schema{type: :array, items: Account}, + statuses: %Schema{type: :array, items: Status}, + notification_groups: %Schema{type: :array, items: notification_group()} + }, + required: [:accounts, :statuses, :notification_groups] + } + end + + defp notification_group do + %Schema{ + title: "NotificationGroup", + type: :object, + properties: %{ + group_key: %Schema{type: :string}, + notifications_count: %Schema{type: :integer}, + type: notification_type(), + most_recent_notification_id: %Schema{type: :string}, + page_min_id: %Schema{type: :string, nullable: true}, + page_max_id: %Schema{type: :string, nullable: true}, + latest_page_notification_at: %Schema{type: :string, format: :"date-time", nullable: true}, + sample_account_ids: %Schema{type: :array, items: %Schema{type: :string}}, + status_id: %Schema{type: :string, nullable: true} + }, + required: [ + :group_key, + :notifications_count, + :type, + :most_recent_notification_id, + :sample_account_ids + ] + } + end + + defp grouped_notification_params do + [ + Operation.parameter( + :exclude_types, + :query, + %Schema{type: :array, items: notification_type()}, + "Array of types to exclude" + ), + Operation.parameter( + :account_id, + :query, + %Schema{type: :string}, + "Return only notifications received from this account" + ), + Operation.parameter( + :types, + :query, + %Schema{type: :array, items: notification_type()}, + "Include the notifications for activities with the given types" + ), + Operation.parameter( + :grouped_types, + :query, + %Schema{type: :array, items: notification_type()}, + "Notification types that may be grouped" + ) + ] + end + defp notification_type do %Schema{ type: :string, @@ -239,4 +390,11 @@ defmodule Pleroma.Web.ApiSpec.NotificationOperation do required: true ) end + + defp group_key_param do + Operation.parameter(:group_key, :path, :string, "Notification group key", + example: "favourite-123", + required: true + ) + end end diff --git a/lib/pleroma/web/controller_helper.ex b/lib/pleroma/web/controller_helper.ex index b15b0ea81..0c85e79f9 100644 --- a/lib/pleroma/web/controller_helper.ex +++ b/lib/pleroma/web/controller_helper.ex @@ -55,16 +55,28 @@ defmodule Pleroma.Web.ControllerHelper do # TODO: Only fetch the params from open_api_spex when everything is converted @id_keys Pagination.page_keys() -- ["limit", "order"] + @id_key_atoms Enum.map(@id_keys, &String.to_atom/1) + @drop_id_params_key :drop_id_params defp build_pagination_fields(conn, min_id, max_id, extra_params, order) do + drop_id_params? = Map.get(extra_params, @drop_id_params_key, false) + extra_params = Map.delete(extra_params, @drop_id_params_key) + + path_param_keys = + conn.path_params + |> Map.keys() + |> Enum.flat_map(&[&1, String.to_existing_atom(&1)]) + params = if Map.has_key?(conn.private, :open_api_spex) do get_in(conn, [Access.key(:private), Access.key(:open_api_spex), Access.key(:params)]) else conn.params end - |> Map.drop(Map.keys(conn.path_params) |> Enum.map(&String.to_existing_atom/1)) + |> Map.drop(path_param_keys) |> Map.merge(extra_params) - |> Map.drop(@id_keys) + # Some OpenApiSpex routes cast cursor params to atoms. Keep the historical default for + # existing endpoints, but allow grouped notifications to opt out of stale request cursors. + |> Map.drop(@id_keys ++ if(drop_id_params?, do: @id_key_atoms, else: [])) {{next_id, nid}, {prev_id, pid}} = if order == :desc, diff --git a/lib/pleroma/web/endpoint.ex b/lib/pleroma/web/endpoint.ex index 36e78e1e2..07b33f866 100644 --- a/lib/pleroma/web/endpoint.ex +++ b/lib/pleroma/web/endpoint.ex @@ -9,8 +9,8 @@ defmodule Pleroma.Web.Endpoint do alias Pleroma.Config - socket("/api/v1/streaming", Pleroma.Web.MastodonAPI.WebsocketHandler, - longpoll: false, + plug(Pleroma.Web.MastodonAPI.WebsocketPlug, + path: "/api/v1/streaming", websocket: [ path: "/", compress: false, @@ -169,8 +169,7 @@ defmodule Pleroma.Web.Endpoint do else: "pleroma_key" extra = - Config.get([__MODULE__, :extra_cookie_attrs]) - |> Enum.join(";") + Enum.join(Config.get([__MODULE__, :extra_cookie_attrs]), ";") # The session will be stored in the cookie and signed, # this means its contents can be read but not tampered with. diff --git a/lib/pleroma/web/mastodon_api/controllers/notification_controller.ex b/lib/pleroma/web/mastodon_api/controllers/notification_controller.ex index afd83b785..8a5f8e7b6 100644 --- a/lib/pleroma/web/mastodon_api/controllers/notification_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/notification_controller.ex @@ -5,13 +5,17 @@ defmodule Pleroma.Web.MastodonAPI.NotificationController do use Pleroma.Web, :controller - import Pleroma.Web.ControllerHelper, only: [add_link_headers: 2] + import Pleroma.Web.ControllerHelper, only: [add_link_headers: 2, add_link_headers: 3] alias Pleroma.Notification + alias Pleroma.User + alias Pleroma.Web.MastodonAPI.AccountView alias Pleroma.Web.MastodonAPI.MastodonAPI alias Pleroma.Web.Plugs.OAuthScopesPlug - @oauth_read_actions [:show, :index] + # Mastodon's docs currently list write:notifications for group accounts, but the endpoint is + # read-only and Mastodon's implementation accepts read:notifications. Prefer least privilege. + @oauth_read_actions [:show, :index, :grouped_index, :show_group, :group_accounts, :unread_count] plug(Pleroma.Web.ApiSpec.CastAndValidate, replace_params: false) @@ -39,7 +43,7 @@ defmodule Pleroma.Web.MastodonAPI.NotificationController do # GET /api/v1/notifications def index(%{private: %{open_api_spex: %{params: %{account_id: account_id} = params}}} = conn, _) do - case Pleroma.User.get_cached_by_id(account_id) do + case User.get_cached_by_id(account_id) do %{ap_id: account_ap_id} -> params = params @@ -59,10 +63,108 @@ defmodule Pleroma.Web.MastodonAPI.NotificationController do do_get_notifications(conn, params) end + # GET /api/v2/notifications + def grouped_index( + %{private: %{open_api_spex: %{params: %{account_id: account_id} = params}}} = conn, + _ + ) do + case User.get_cached_by_id(account_id) do + %{ap_id: account_ap_id} -> + params = + params + |> Map.delete(:account_id) + |> Map.put(:account_ap_id, account_ap_id) + + do_get_grouped_notifications(conn, params) + + _ -> + conn + |> put_status(:not_found) + |> json(%{"error" => "Account is not found"}) + end + end + + def grouped_index(%{private: %{open_api_spex: %{params: params}}} = conn, _) do + do_get_grouped_notifications(conn, params) + end + + # GET /api/v2/notifications/:group_key + def show_group( + %{assigns: %{user: user}, private: %{open_api_spex: %{params: %{group_key: group_key}}}} = + conn, + _ + ) do + {notifications, notification_group_counts, notification_group_bounds} = + MastodonAPI.get_notification_group_result(user, group_key, %{}) + + if Enum.empty?(notifications) do + conn + |> put_status(:not_found) + |> json(%{"error" => "Notification group is not found"}) + else + grouped_types = if String.starts_with?(group_key, "ungrouped-"), do: [], else: nil + + render(conn, "grouped_index.json", + notification_groups: [notifications], + notification_group_counts: notification_group_counts, + notification_group_bounds: notification_group_bounds, + for: user, + grouped_types: grouped_types, + include_page_metadata: false + ) + end + end + + # GET /api/v2/notifications/:group_key/accounts + def group_accounts( + %{assigns: %{user: user}, private: %{open_api_spex: %{params: %{group_key: group_key}}}} = + conn, + _ + ) do + # Mastodon paginates this endpoint in code, but the public docs say it returns accounts of all + # notifications in the group and do not document cursor params here. Follow the documented API. + users = MastodonAPI.get_notification_group_accounts(user, group_key) + + json(conn, AccountView.render("index.json", %{users: users, for: user})) + end + + # GET /api/v2/notifications/unread_count + def unread_count( + %{private: %{open_api_spex: %{params: %{account_id: account_id} = params}}} = conn, + _ + ) do + case User.get_cached_by_id(account_id) do + %{ap_id: account_ap_id} -> + params = + params + |> Map.delete(:account_id) + |> Map.put(:account_ap_id, account_ap_id) + + do_get_unread_group_count(conn, params) + + _ -> + conn + |> put_status(:not_found) + |> json(%{"error" => "Account is not found"}) + end + end + + def unread_count(%{private: %{open_api_spex: %{params: params}}} = conn, _) do + do_get_unread_group_count(conn, params) + end + + # POST /api/v2/notifications/:group_key/dismiss + def dismiss_group( + %{assigns: %{user: user}, private: %{open_api_spex: %{params: %{group_key: group_key}}}} = + conn, + _ + ) do + MastodonAPI.dismiss_notification_group(user, group_key) + json(conn, %{}) + end + defp do_get_notifications(%{assigns: %{user: user}} = conn, params) do - params = - Map.new(params, fn {k, v} -> {to_string(k), v} end) - |> Map.put_new("types", Map.get(params, :include_types, @default_notification_types)) + params = normalize_notification_params(params) notifications = MastodonAPI.get_notifications(user, params) @@ -74,6 +176,35 @@ defmodule Pleroma.Web.MastodonAPI.NotificationController do ) end + defp do_get_grouped_notifications(%{assigns: %{user: user}} = conn, params) do + params = normalize_notification_params(params) + + {notification_groups, page_notifications, notification_group_counts, + notification_group_bounds} = + MastodonAPI.get_grouped_notification_page(user, params) + + conn + |> add_link_headers(page_notifications, %{drop_id_params: true}) + |> render("grouped_index.json", + notification_groups: notification_groups, + notification_group_counts: notification_group_counts, + notification_group_bounds: notification_group_bounds, + for: user, + grouped_types: params["grouped_types"] + ) + end + + defp do_get_unread_group_count(%{assigns: %{user: user}} = conn, params) do + params = normalize_notification_params(params) + json(conn, %{count: MastodonAPI.unread_notification_group_count(user, params)}) + end + + defp normalize_notification_params(params) do + params + |> Map.new(fn {k, v} -> {to_string(k), v} end) + |> Map.put_new("types", Map.get(params, :include_types, @default_notification_types)) + end + # GET /api/v1/notifications/:id def show(%{assigns: %{user: user}, private: %{open_api_spex: %{params: %{id: id}}}} = conn, _) do with {:ok, notification} <- Notification.get(user, id) do diff --git a/lib/pleroma/web/mastodon_api/mastodon_api.ex b/lib/pleroma/web/mastodon_api/mastodon_api.ex index c9e045d23..d6f2dd3a3 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api.ex @@ -6,12 +6,16 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPI do import Ecto.Query import Ecto.Changeset + alias Pleroma.Marker alias Pleroma.Notification alias Pleroma.Pagination + alias Pleroma.Repo alias Pleroma.ScheduledActivity alias Pleroma.User alias Pleroma.Web.CommonAPI + @notification_group_sample_limit 8 + @spec follow(User.t(), User.t(), map) :: {:ok, User.t()} | {:error, String.t()} def follow(follower, followed, params \\ %{}) do result = @@ -61,29 +65,356 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPI do end def get_notifications(user, params \\ %{}) do - options = - cast_params(params) |> Map.update(:include_types, [], fn include_types -> include_types end) + user + |> notifications_query(params) + |> Pagination.fetch_paginated(params) + end - options = - if ("pleroma:report" not in options.include_types and - User.privileged?(user, :reports_manage_reports)) or - User.privileged?(user, :reports_manage_reports) do - options - else - options - |> Map.update(:exclude_types, ["pleroma:report"], fn current_exclude_types -> - current_exclude_types ++ ["pleroma:report"] - end) - end + def get_grouped_notification_page(user, params \\ %{}) do + grouped_types = + params + |> Map.get("grouped_types") + |> Notification.normalize_grouped_types() + + query = notifications_query(user, params) + {order, cursor_filters} = group_pagination(params) + + group_rows = + query + |> notification_group_rows(grouped_types, grouped_limit(params), order, cursor_filters) + + group_rows = if order == :asc, do: Enum.reverse(group_rows), else: group_rows + + page_notifications = representative_notifications(query, group_rows) + + notification_groups = + Enum.map(group_rows, ¬ification_group_sample(user, params, &1.group_key)) + + { + notification_groups, + page_notifications, + notification_group_counts(group_rows), + notification_group_bounds(group_rows) + } + end + + def get_notification_group_result(user, group_key, params \\ %{}) do + notifications = notification_group_sample(user, params, group_key) + metadata = notification_group_metadata(user, group_key, params) + + if Enum.empty?(notifications) or is_nil(metadata) do + {[], %{}, %{}} + else + { + notifications, + %{group_key => metadata.notifications_count}, + %{group_key => Map.drop(metadata, [:notifications_count])} + } + end + end + + def get_notification_group(user, group_key, params \\ %{}) + + def get_notification_group(user, "ungrouped-" <> notification_id, _params) do + case Notification.get(user, notification_id) do + {:ok, notification} -> [notification] + _ -> [] + end + end + + def get_notification_group(user, group_key, params) do + notification_group_sample(user, params, group_key) + end + + def get_notification_group_accounts(user, "ungrouped-" <> notification_id) do + with {:ok, notification} <- Notification.get(user, notification_id), + %User{} = actor <- User.get_cached_by_ap_id(notification.activity.data["actor"]) do + [actor] + else + _ -> [] + end + end + + def get_notification_group_accounts(user, group_key) do + user + |> notification_group_query(group_key, %{}) + |> exclude(:preload) + |> distinct(true) + |> select([user_actor: user_actor], user_actor) + |> Repo.all() + end + + def dismiss_notification_group(user, "ungrouped-" <> notification_id) do + Notification.destroy_multiple(user, [notification_id]) + end + + def dismiss_notification_group(%User{id: user_id}, group_key) do + Notification + |> where([n], n.user_id == ^user_id and n.group_key == ^group_key) + |> Repo.delete_all() + end + + defp group_pagination(params) do + cond do + min_id = Map.get(params, "min_id") -> + cursor_filters = [{:gt, min_id}] + + cursor_filters = + case Map.get(params, "max_id") do + nil -> cursor_filters + max_id -> [{:lt, max_id} | cursor_filters] + end + + {:asc, cursor_filters} + + since_id = Map.get(params, "since_id") -> + {:desc, [{:gt, since_id}]} + + max_id = Map.get(params, "max_id") -> + {:desc, [{:lt, max_id}]} + + true -> + {:desc, []} + end + end + + defp grouped_limit(params) do + params + |> Map.get("limit", 40) + |> parse_limit(40) + |> min(80) + end + + def unread_notification_group_count(user, params \\ %{}) do + grouped_types = + params + |> Map.get("grouped_types") + |> Notification.normalize_grouped_types() + + limit = unread_count_limit(params) + + user + |> notifications_query(params) + # The grouped API docs define unread by the notifications marker, not by Pleroma's per-row + # seen flag used by the v1 unread count. Keep this marker-based for Mastodon clients. + |> restrict_after_marker(notification_marker_last_read_id(user)) + |> notification_group_rows(grouped_types, limit, :desc, []) + |> length() + end + + defp notification_group_rows(query, grouped_types, group_limit, :desc, cursor_filters) do + query + |> notification_group_keyed_query(grouped_types) + |> group_by([n], n.group_key) + |> apply_group_cursor_filters(cursor_filters) + |> select([n], %{ + group_key: n.group_key, + representative_id: max(n.id), + notifications_count: count(n.id), + page_min_id: min(n.id), + page_max_id: max(n.id), + latest_page_notification_at: max(n.inserted_at) + }) + |> order_by([n], desc: max(n.id)) + |> limit(^group_limit) + |> Repo.all() + end + + defp notification_group_rows(query, grouped_types, group_limit, :asc, cursor_filters) do + query + |> notification_group_keyed_query(grouped_types) + |> group_by([n], n.group_key) + |> apply_group_cursor_filters(cursor_filters) + |> select([n], %{ + group_key: n.group_key, + representative_id: max(n.id), + notifications_count: count(n.id), + page_min_id: min(n.id), + page_max_id: max(n.id), + latest_page_notification_at: max(n.inserted_at) + }) + |> order_by([n], asc: max(n.id)) + |> limit(^group_limit) + |> Repo.all() + end + + defp apply_group_cursor_filters(query, []), do: query + + defp apply_group_cursor_filters(query, [{:gt, id} | rest]) do + query + |> having([n], max(n.id) > ^id) + |> apply_group_cursor_filters(rest) + end + + defp apply_group_cursor_filters(query, [{:lt, id} | rest]) do + query + |> having([n], max(n.id) < ^id) + |> apply_group_cursor_filters(rest) + end + + defp notification_group_keyed_query(query, grouped_types) do + query + |> exclude(:preload) + |> select([n], %{ + id: n.id, + inserted_at: n.inserted_at, + group_key: + fragment( + "CASE WHEN ? IS NOT NULL AND ?::text = ANY(?) THEN ? ELSE 'ungrouped-' || ?::text END", + n.group_key, + n.type, + type(^grouped_types, {:array, :string}), + n.group_key, + n.id + ) + }) + |> subquery() + end + + defp representative_notifications(_query, []), do: [] + + defp representative_notifications(query, group_rows) do + representative_ids = Enum.map(group_rows, & &1.representative_id) + + notifications_by_id = + query + |> where([n], n.id in ^representative_ids) + |> Repo.all() + |> Map.new(&{to_string(&1.id), &1}) + + group_rows + |> Enum.map(&Map.get(notifications_by_id, to_string(&1.representative_id))) + |> Enum.filter(& &1) + end + + defp notification_group_counts(group_rows) do + Map.new(group_rows, &{&1.group_key, &1.notifications_count}) + end + + defp notification_group_bounds(group_rows) do + Map.new(group_rows, fn row -> + {row.group_key, + %{ + page_min_id: row.page_min_id, + page_max_id: row.page_max_id, + latest_page_notification_at: row.latest_page_notification_at + }} + end) + end + + defp notification_group_sample(user, _params, "ungrouped-" <> notification_id) do + case Notification.get(user, notification_id) do + {:ok, notification} -> [notification] + _ -> [] + end + end + + defp notification_group_sample(user, params, group_key) do + user + |> notification_group_query(group_key, params) + |> order_by([n], desc: n.id) + |> limit(^@notification_group_sample_limit) + |> Repo.all() + end + + defp notification_group_metadata(user, "ungrouped-" <> notification_id, _params) do + case Notification.get(user, notification_id) do + {:ok, notification} -> + %{ + notifications_count: 1, + page_min_id: notification.id, + page_max_id: notification.id, + latest_page_notification_at: notification.inserted_at + } + + _ -> + nil + end + end + + defp notification_group_metadata(user, group_key, params) do + user + |> notification_group_query(group_key, params) + |> exclude(:preload) + |> select([n], %{ + notifications_count: count(n.id), + page_min_id: min(n.id), + page_max_id: max(n.id), + latest_page_notification_at: max(n.inserted_at) + }) + |> Repo.one() + |> case do + %{notifications_count: 0} -> nil + metadata -> metadata + end + end + + defp notification_group_query(user, group_key, params) do + user + |> notifications_query(params) + |> where([n], n.group_key == ^group_key) + end + + defp notification_marker_last_read_id(user) do + Marker + |> where([m], m.user_id == ^user.id and m.timeline == "notifications") + |> select([m], m.last_read_id) + |> Repo.one() + end + + defp restrict_after_marker(query, last_read_id) + when is_binary(last_read_id) and last_read_id != "" do + where(query, [n], n.id > ^last_read_id) + end + + defp restrict_after_marker(query, _last_read_id), do: query + + defp notifications_query(user, params) do + options = notification_options(user, params) user |> Notification.for_user_query(options) |> restrict(:types, options) |> restrict(:exclude_types, options) |> restrict(:account_ap_id, options) - |> Pagination.fetch_paginated(params) end + defp notification_options(user, params) do + options = + params + |> cast_params() + |> Map.update(:include_types, [], fn include_types -> include_types end) + + if ("pleroma:report" not in options.include_types and + User.privileged?(user, :reports_manage_reports)) or + User.privileged?(user, :reports_manage_reports) do + options + else + options + |> Map.update(:exclude_types, ["pleroma:report"], fn current_exclude_types -> + current_exclude_types ++ ["pleroma:report"] + end) + end + end + + defp unread_count_limit(params) do + params + |> Map.get("limit", 100) + |> parse_limit(100) + |> min(1000) + end + + defp parse_limit(limit, _default) when is_integer(limit) and limit > 0, do: limit + + defp parse_limit(limit, default) when is_binary(limit) do + case Integer.parse(limit) do + {limit, _} when limit > 0 -> limit + _ -> default + end + end + + defp parse_limit(_, default), do: default + def get_scheduled_activities(user, params \\ %{}) do user |> ScheduledActivity.for_user_query() @@ -95,6 +426,8 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPI do exclude_types: {:array, :string}, types: {:array, :string}, exclude_visibilities: {:array, :string}, + grouped_types: {:array, :string}, + limit: :integer, reblogs: :boolean, with_muted: :boolean, account_ap_id: :string, diff --git a/lib/pleroma/web/mastodon_api/views/instance_view.ex b/lib/pleroma/web/mastodon_api/views/instance_view.ex index d64ce4fb5..1a0f63ffa 100644 --- a/lib/pleroma/web/mastodon_api/views/instance_view.ex +++ b/lib/pleroma/web/mastodon_api/views/instance_view.ex @@ -17,6 +17,8 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do reject: "suspend" } + def mastodon_api_level, do: @mastodon_api_level + def render("show.json", _) do instance = Config.get(:instance) @@ -166,6 +168,7 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do [ "pleroma_api", "mastodon_api", + "mastodon_api_grouped_notifications", "mastodon_api_streaming", "polls", "v2_suggestions", diff --git a/lib/pleroma/web/mastodon_api/views/notification_view.ex b/lib/pleroma/web/mastodon_api/views/notification_view.ex index 27532c42a..33d4b2924 100644 --- a/lib/pleroma/web/mastodon_api/views/notification_view.ex +++ b/lib/pleroma/web/mastodon_api/views/notification_view.ex @@ -72,6 +72,58 @@ defmodule Pleroma.Web.MastodonAPI.NotificationView do safe_render_many(notifications, NotificationView, "show.json", opts) end + def render("grouped_index.json", %{notifications: notifications} = opts) do + grouped_types = Notification.normalize_grouped_types(opts[:grouped_types]) + + opts + |> Map.delete(:notifications) + |> Map.put( + :notification_groups, + Notification.group_notifications(notifications, grouped_types) + ) + |> then(&render("grouped_index.json", &1)) + end + + def render( + "grouped_index.json", + %{notification_groups: notification_groups, for: reading_user} = opts + ) do + grouped_types = Notification.normalize_grouped_types(opts[:grouped_types]) + notification_group_counts = Map.get(opts, :notification_group_counts, %{}) + notification_group_bounds = Map.get(opts, :notification_group_bounds, %{}) + include_page_metadata = Map.get(opts, :include_page_metadata, true) + + statuses = + notification_groups + |> Enum.map(&List.first/1) + |> Enum.map(&render("show.json", %{notification: &1, for: reading_user})) + |> Enum.map(& &1[:status]) + |> Enum.filter(& &1) + |> Enum.uniq_by(& &1[:id]) + + actors = + notification_groups + |> List.flatten() + |> notification_actors() + + %{ + accounts: AccountView.render("index.json", %{users: actors, for: reading_user}), + statuses: statuses, + notification_groups: + Enum.map( + notification_groups, + &render_group( + &1, + reading_user, + grouped_types, + notification_group_counts, + notification_group_bounds, + include_page_metadata + ) + ) + } + end + def render( "show.json", %{ @@ -95,7 +147,7 @@ defmodule Pleroma.Web.MastodonAPI.NotificationView do response = %{ id: to_string(notification.id), - group_key: "ungrouped-" <> to_string(notification.id), + group_key: Notification.group_key(notification), type: notification.type, created_at: CommonAPI.Utils.to_masto_date(notification.inserted_at), account: account, @@ -137,6 +189,104 @@ defmodule Pleroma.Web.MastodonAPI.NotificationView do Map.put(response, :report, report_render) end + defp render_group( + [%Notification{} = notification | _] = notifications, + _reading_user, + grouped_types, + notification_group_counts, + notification_group_bounds, + include_page_metadata + ) do + latest_notification = List.first(notifications) + oldest_notification = List.last(notifications) + status_activity = status_activity_for_group(notifications, grouped_types) + group_key = Notification.group_key(notification, grouped_types) + bounds = Map.get(notification_group_bounds, group_key, %{}) + + response = %{ + group_key: group_key, + notifications_count: Map.get(notification_group_counts, group_key, length(notifications)), + type: notification.type, + most_recent_notification_id: + bounds + |> Map.get(:page_max_id, latest_notification.id) + |> to_string(), + sample_account_ids: + notifications + |> notification_actors() + |> Enum.map(&to_string(&1.id)) + } + + response = + if include_page_metadata do + Map.merge(response, %{ + page_min_id: + bounds + |> Map.get(:page_min_id, oldest_notification.id) + |> to_string(), + page_max_id: + bounds + |> Map.get(:page_max_id, latest_notification.id) + |> to_string(), + latest_page_notification_at: + bounds + |> Map.get(:latest_page_notification_at, latest_notification.inserted_at) + |> CommonAPI.Utils.to_masto_date() + }) + else + response + end + + if status_activity do + Map.put(response, :status_id, to_string(status_activity.id)) + else + response + end + end + + defp status_activity_for_group([%Notification{} = notification | _], grouped_types) do + status_activity_for(notification, grouped_types) + end + + defp status_activity_for(%Notification{type: type, activity: activity}, _grouped_types) + when type in ["mention", "status", "poll"] do + activity + end + + defp status_activity_for(%Notification{type: type} = notification, grouped_types) + when type in ["favourite", "reblog"] do + group_key = Notification.group_key(notification, grouped_types) + + case String.split(group_key, "-", parts: 3) do + [^type, activity_id, _bucket] -> + case Activity.create_by_id_with_object(activity_id) do + %Activity{} = activity -> activity + _ -> parent_status_activity(notification.activity) + end + + _ -> + parent_status_activity(notification.activity) + end + end + + defp status_activity_for(%Notification{type: type, activity: activity}, _grouped_types) + when type in ["update", "pleroma:emoji_reaction"] do + parent_status_activity(activity) + end + + defp status_activity_for(_, _grouped_types), do: nil + + defp parent_status_activity(activity) do + Activity.get_create_by_object_ap_id(object_id_for(activity)) + end + + defp notification_actors(notifications) do + notifications + |> Enum.map(&User.get_cached_by_ap_id(&1.activity.data["actor"])) + |> Enum.filter(& &1) + |> Enum.uniq_by(& &1.id) + end + defp put_emoji(response, activity) do response |> Map.put(:emoji, activity.data["content"]) diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index 09af60865..6949c5f2d 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -28,9 +28,13 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do # This is a naive way to do this, just spawning a process per activity # to fetch the preview. However it should be fine considering # pagination is restricted to 40 activities at a time - defp fetch_rich_media_for_activities(activities) do + # Force disable Websockets streaming for backfill jobs, + # otherwise old posts can show up on timelines. + defp fetch_rich_media_for_activities(activities, opts) do + opts = Map.put(opts, :stream, false) + Enum.each(activities, fn activity -> - Card.get_by_activity(activity) + Card.get_by_activity(activity, opts) end) end @@ -113,7 +117,8 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do activities = Enum.filter(opts.activities, & &1) # Start prefetching rich media before doing anything else - fetch_rich_media_for_activities(activities) + fetch_rich_media_for_activities(activities, opts) + replied_to_activities = get_replied_to_activities(activities) quoted_activities = get_quoted_activities(activities) @@ -361,8 +366,10 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do summary = object.data["summary"] || "" + # Force disable Websockets streaming for backfill jobs which the below call will create, + # otherwise old posts can show up on timelines. card = - case Card.get_by_activity(activity) do + case Card.get_by_activity(activity, Map.put(opts, :stream, false)) do %Card{} = result -> render("card.json", result) _ -> nil end diff --git a/lib/pleroma/web/mastodon_api/websocket_handler.ex b/lib/pleroma/web/mastodon_api/websocket_handler.ex index 2b698bd5d..3dc862a5a 100644 --- a/lib/pleroma/web/mastodon_api/websocket_handler.ex +++ b/lib/pleroma/web/mastodon_api/websocket_handler.ex @@ -67,9 +67,10 @@ defmodule Pleroma.Web.MastodonAPI.WebsocketHandler do @impl Phoenix.Socket.Transport def handle_in({text, [opcode: :text]}, state) do - with {:ok, %{} = event} <- Jason.decode(text) do - handle_client_event(event, state) - else + case Jason.decode(text) do + {:ok, %{} = event} -> + handle_client_event(event, state) + _ -> Logger.error("#{__MODULE__} received non-JSON event: #{inspect(text)}") {:ok, state} @@ -85,11 +86,11 @@ defmodule Pleroma.Web.MastodonAPI.WebsocketHandler do def handle_info({:render_with_user, view, template, item, topic}, state) do user = %User{} = User.get_cached_by_ap_id(state.user.ap_id) - unless Streamer.filtered_by_user?(user, item) do + if Streamer.filtered_by_user?(user, item) do + {:ok, state} + else message = view.render(template, item, user, topic) {:push, {:text, message}, %{state | user: user}} - else - {:ok, state} end end @@ -253,7 +254,7 @@ defmodule Pleroma.Web.MastodonAPI.WebsocketHandler do defp find_sec_websocket_protocol(sec_headers) do Enum.find_value(sec_headers, fn - {"sec-websocket-protocol", token} -> token + {"sec-websocket-protocol", protocols} -> protocols |> Plug.Conn.Utils.list() |> List.first() _ -> nil end) end diff --git a/lib/pleroma/web/mastodon_api/websocket_plug.ex b/lib/pleroma/web/mastodon_api/websocket_plug.ex new file mode 100644 index 000000000..c5fb5b748 --- /dev/null +++ b/lib/pleroma/web/mastodon_api/websocket_plug.ex @@ -0,0 +1,104 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.MastodonAPI.WebsocketPlug do + @moduledoc """ + A Phoenix 1.8 compatible WebSocket transport for Mastodon streaming. + + It mirrors Phoenix.Transports.WebSocket, but echoes a successfully authenticated + Mastodon-style Sec-WebSocket-Protocol token so browser clients accept the handshake. + """ + + @behaviour Plug + + import Plug.Conn + + alias Phoenix.Socket.Transport + alias Pleroma.Web.Endpoint + alias Pleroma.Web.MastodonAPI.WebsocketHandler + + @connect_info_opts [:check_csrf] + + @impl Plug + def init(opts) do + path = String.split(Keyword.fetch!(opts, :path), "/", trim: true) + websocket = Keyword.fetch!(opts, :websocket) + config = Transport.load_config(websocket, Phoenix.Transports.WebSocket) + + {path, config} + end + + @impl Plug + def call(%{method: "GET", path_info: path} = conn, {path, opts}) do + conn + |> fetch_query_params() + |> Transport.code_reload(Endpoint, opts) + |> Transport.transport_log(opts[:transport_log]) + |> Transport.check_origin(WebsocketHandler, Endpoint, opts) + |> connect(opts) + end + + def call(%{path_info: path} = conn, {path, _opts}) do + conn + |> send_resp(400, "") + |> halt() + end + + def call(conn, _opts), do: conn + + defp connect(%{halted: true} = conn, _opts), do: conn + + defp connect(%{params: params} = conn, opts) do + keys = Keyword.get(opts, :connect_info, []) + + connect_info = + Transport.connect_info(conn, Endpoint, keys, Keyword.take(opts, @connect_info_opts)) + + config = %{ + endpoint: Endpoint, + transport: :websocket, + options: opts, + params: params, + connect_info: connect_info + } + + case WebsocketHandler.connect(config) do + {:ok, arg} -> + try do + conn + |> echo_sec_websocket_protocol() + |> WebSockAdapter.upgrade(WebsocketHandler, arg, opts) + |> halt() + rescue + e in WebSockAdapter.UpgradeError -> + conn + |> send_resp(400, e.message) + |> halt() + end + + :error -> + conn + |> send_resp(403, "") + |> halt() + + {:error, reason} -> + {m, f, args} = opts[:error_handler] + + halt(apply(m, f, [conn, reason | args])) + end + end + + defp echo_sec_websocket_protocol(conn) do + case get_req_header(conn, "sec-websocket-protocol") do + [protocols | _] -> + case Plug.Conn.Utils.list(protocols) do + [protocol | _] -> put_resp_header(conn, "sec-websocket-protocol", protocol) + [] -> conn + end + + [] -> + conn + end + end +end diff --git a/lib/pleroma/web/rich_media/backfill.ex b/lib/pleroma/web/rich_media/backfill.ex index 1cd90629f..a100b39ab 100644 --- a/lib/pleroma/web/rich_media/backfill.ex +++ b/lib/pleroma/web/rich_media/backfill.ex @@ -11,6 +11,8 @@ defmodule Pleroma.Web.RichMedia.Backfill do require Logger + @callback run(map()) :: :ok | Parser.parse_errors() | Helpers.get_errors() + @cachex Pleroma.Config.get([:cachex, :provider], Cachex) @stream_out_impl Pleroma.Config.get( [__MODULE__, :stream_out], @@ -26,11 +28,7 @@ defmodule Pleroma.Web.RichMedia.Backfill do {:ok, card} = Card.create(url, fields) maybe_schedule_expiration(url, fields) - - with %{"activity_id" => activity_id} <- args, - false <- is_nil(activity_id) do - stream_update(args) - end + maybe_update_stream(args) warm_cache(url_hash, card) :ok @@ -55,12 +53,17 @@ defmodule Pleroma.Web.RichMedia.Backfill do end end - defp stream_update(%{"activity_id" => activity_id}) do + defp maybe_update_stream(%{"activity_id" => activity_id, "stream" => true}) + when is_binary(activity_id) do Pleroma.Activity.get_by_id(activity_id) |> Pleroma.Activity.normalize() |> @stream_out_impl.stream_out() end + # Streamer.stream_out returns noop when unsupported activity type is requested to be streamed. + # Do the same here for unwanted streaming + defp maybe_update_stream(_), do: :noop + defp warm_cache(key, val), do: @cachex.put(:rich_media_cache, key, val) defp negative_cache(key, ttl \\ :timer.minutes(15)), diff --git a/lib/pleroma/web/rich_media/card.ex b/lib/pleroma/web/rich_media/card.ex index 6b4bb9555..1e9e66ec1 100644 --- a/lib/pleroma/web/rich_media/card.ex +++ b/lib/pleroma/web/rich_media/card.ex @@ -91,7 +91,18 @@ defmodule Pleroma.Web.RichMedia.Card do nil -> activity_id = Keyword.get(opts, :activity_id, nil) - RichMediaWorker.new(%{"op" => "backfill", "url" => url, "activity_id" => activity_id}) + # Nested opts, first layer comes from get_by_activity/2 as Keyword, + # second from API views/Federation as Map. + # Provide default Map when called directly. + opts = Keyword.get(opts, :opts, %{}) + stream = Map.get(opts, :stream, true) + + RichMediaWorker.new(%{ + "op" => "backfill", + "url" => url, + "activity_id" => activity_id, + "stream" => stream + }) |> Oban.insert() nil @@ -112,9 +123,11 @@ defmodule Pleroma.Web.RichMedia.Card do end end - @spec get_by_activity(Activity.t()) :: t() | nil | :error + @spec get_by_activity(Activity.t(), %{}) :: t() | nil | :error + def get_by_activity(activity, opts \\ %{}) + # Fake/Draft activity - def get_by_activity(%Activity{id: "pleroma:fakeid"} = activity) do + def get_by_activity(%Activity{id: "pleroma:fakeid"} = activity, _opts) do with {_, true} <- {:config, @config_impl.get([:rich_media, :enabled])}, %Object{} = object <- Object.normalize(activity, fetch: false), url when not is_nil(url) <- HTML.extract_first_external_url_from_object(object) do @@ -138,13 +151,13 @@ defmodule Pleroma.Web.RichMedia.Card do end end - def get_by_activity(activity) do + def get_by_activity(activity, opts) do with %Object{} = object <- Object.normalize(activity, fetch: false), {_, nil} <- {:cached, get_cached_url(object, activity.id)} do nil else {:cached, url} -> - get_or_backfill_by_url(url, activity_id: activity.id) + get_or_backfill_by_url(url, activity_id: activity.id, opts: opts) _ -> :error diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 813c5c482..5657bcad7 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -844,6 +844,16 @@ defmodule Pleroma.Web.Router do get("/directory", DirectoryController, :index) end + scope "/api/v2", Pleroma.Web.MastodonAPI do + pipe_through(:authenticated_api) + + get("/notifications", NotificationController, :grouped_index) + get("/notifications/unread_count", NotificationController, :unread_count) + get("/notifications/:group_key/accounts", NotificationController, :group_accounts) + get("/notifications/:group_key", NotificationController, :show_group) + post("/notifications/:group_key/dismiss", NotificationController, :dismiss_group) + end + scope "/api/v2", Pleroma.Web.MastodonAPI do pipe_through(:api) diff --git a/priv/repo/migrations/20200602125217_add_group_key_to_notifications.exs b/priv/repo/migrations/20200602125217_add_group_key_to_notifications.exs new file mode 100644 index 000000000..7c7d529e0 --- /dev/null +++ b/priv/repo/migrations/20200602125217_add_group_key_to_notifications.exs @@ -0,0 +1,15 @@ +defmodule Pleroma.Repo.Migrations.AddGroupKeyToNotifications do + use Ecto.Migration + + def change do + alter table(:notifications) do + add(:group_key, :string) + end + + create_if_not_exists( + index(:notifications, [:user_id, :group_key, "id desc nulls last"], + where: "group_key IS NOT NULL" + ) + ) + end +end diff --git a/priv/scrubbers/default.ex b/priv/scrubbers/default.ex index 342ef9944..8e2d16650 100644 --- a/priv/scrubbers/default.ex +++ b/priv/scrubbers/default.ex @@ -46,6 +46,7 @@ defmodule Pleroma.HTML.Scrubber.Default do Meta.allow_tag_with_these_attributes(:b, ["lang"]) Meta.allow_tag_with_these_attributes(:blockquote, ["lang"]) Meta.allow_tag_with_these_attributes(:br, ["lang"]) + Meta.allow_tag_with_these_attributes(:center, []) Meta.allow_tag_with_these_attributes(:cite, ["lang"]) Meta.allow_tag_with_these_attributes(:code, ["lang"]) Meta.allow_tag_with_these_attributes(:del, ["lang"]) @@ -83,48 +84,57 @@ defmodule Pleroma.HTML.Scrubber.Default do "quote-inline", "invisible", "ellipsis", - "mfm-center", - "mfm-flip", - "mfm-font", - "mfm-blur", - "mfm-rotate", - "mfm-x2", - "mfm-x3", - "mfm-x4", - "mfm-position", - "mfm-scale", - "mfm-fg", - "mfm-bg", + "mfm-tada", "mfm-jelly", "mfm-twitch", "mfm-shake", "mfm-spin", "mfm-jump", "mfm-bounce", + "mfm-flip", + "mfm-x2", + "mfm-x3", + "mfm-x4", + "mfm-scale", + "mfm-position", + "mfm-fg", + "mfm-bg", + "mfm-border", + "mfm-font", + "mfm-blur", "mfm-rainbow", - "mfm-tada", - "mfm-sparkle" + "mfm-sparkle", + "mfm-rotate", + "mfm-ruby", + "mfm-unixtime", + # Exists in Akkoma but not Misskey? + "mfm-center" ]) Meta.allow_tag_with_this_attribute_values(:p, "class", ["quote-inline"]) Meta.allow_tag_with_these_attributes(:span, [ "lang", - "data-mfm-h", - "data-mfm-v", + "data-mfm-speed", + "data-mfm-delay", + "data-mfm-left", + "data-mfm-alternate", "data-mfm-x", "data-mfm-y", - "data-mfm-alternate", - "data-mfm-speed", - "data-mfm-deg", - "data-mfm-left", + "data-mfm-h", + "data-mfm-v", + "data-mfm-color", + "data-mfm-width", + "data-mfm-style", + "data-mfm-radius", + "data-mfm-noclip", "data-mfm-serif", "data-mfm-monospace", "data-mfm-cursive", "data-mfm-fantasy", "data-mfm-emoji", "data-mfm-math", - "data-mfm-color" + "data-mfm-deg" ]) Meta.allow_tag_with_this_attribute_values(:code, "class", ["inline"]) diff --git a/test/pleroma/integration/mastodon_websocket_test.exs b/test/pleroma/integration/mastodon_websocket_test.exs index 078bb643c..de88e5002 100644 --- a/test/pleroma/integration/mastodon_websocket_test.exs +++ b/test/pleroma/integration/mastodon_websocket_test.exs @@ -11,6 +11,7 @@ defmodule Pleroma.Integration.MastodonWebsocketTest do alias Pleroma.Integration.WebsocketClient alias Pleroma.Web.CommonAPI + alias Pleroma.Web.MastodonAPI.StatusView alias Pleroma.Web.OAuth @moduletag needs_streamer: true, capture_log: true @@ -31,6 +32,48 @@ defmodule Pleroma.Integration.MastodonWebsocketTest do WebsocketClient.start_link(self(), path, headers) end + defp raw_websocket_handshake(qs, headers) do + uri = URI.parse(@path <> qs) + port = uri.port || 80 + path = uri.path <> if(uri.query, do: "?" <> uri.query, else: "") + + default_headers = [ + {"host", "#{uri.host}:#{port}"}, + {"upgrade", "websocket"}, + {"connection", "Upgrade"}, + {"sec-websocket-key", Base.encode64(:crypto.strong_rand_bytes(16))}, + {"sec-websocket-version", "13"} + ] + + request = [ + "GET #{path} HTTP/1.1\r\n", + Enum.map(default_headers ++ headers, fn {name, value} -> "#{name}: #{value}\r\n" end), + "\r\n" + ] + + with {:ok, socket} <- + :gen_tcp.connect(String.to_charlist(uri.host), port, [:binary, active: false], 1_000), + :ok <- :gen_tcp.send(socket, request), + {:ok, response} <- :gen_tcp.recv(socket, 0, 1_000) do + :gen_tcp.close(socket) + {:ok, parse_http_response(response)} + end + end + + defp parse_http_response(response) do + [headers | _] = String.split(response, "\r\n\r\n", parts: 2) + [status_line | header_lines] = String.split(headers, "\r\n") + [_, status | _] = String.split(status_line, " ") + + headers = + Enum.map(header_lines, fn line -> + [name, value] = String.split(line, ":", parts: 2) + {String.downcase(name), String.trim(value)} + end) + + %{status: String.to_integer(status), headers: headers} + end + defp decode_json(json) do with {:ok, %{"event" => event, "payload" => payload_text}} <- Jason.decode(json), {:ok, payload} <- Jason.decode(payload_text) do @@ -85,9 +128,7 @@ defmodule Pleroma.Integration.MastodonWebsocketTest do assert json["payload"] assert {:ok, json} = Jason.decode(json["payload"]) - view_json = - Pleroma.Web.MastodonAPI.StatusView.render("show.json", activity: activity, for: nil) - |> atom_key_to_string() + view_json = atom_key_to_string(StatusView.render("show.json", activity: activity, for: nil)) assert json == view_json end @@ -114,10 +155,7 @@ defmodule Pleroma.Integration.MastodonWebsocketTest do assert json["payload"] assert {:ok, json} = Jason.decode(json["payload"]) - view_json = - Pleroma.Web.MastodonAPI.StatusView.render("show.json", activity: activity, for: nil) - |> Jason.encode!() - |> Jason.decode!() + view_json = atom_key_to_string(StatusView.render("show.json", activity: activity, for: nil)) assert json == view_json end @@ -279,6 +317,34 @@ defmodule Pleroma.Integration.MastodonWebsocketTest do end) end + test "echoes the Sec-WebSocket-Protocol token in the handshake", %{token: token} do + assert {:ok, %{status: 101, headers: headers}} = + raw_websocket_handshake("?stream=user", [ + {"sec-websocket-protocol", token.token} + ]) + + assert {"sec-websocket-protocol", token.token} in headers + end + + test "echoes the selected Sec-WebSocket-Protocol token", %{token: token} do + assert {:ok, %{status: 101, headers: headers}} = + raw_websocket_handshake("?stream=user", [ + {"sec-websocket-protocol", "#{token.token}, phoenix"} + ]) + + assert {"sec-websocket-protocol", token.token} in headers + end + + test "does not echo an invalid Sec-WebSocket-Protocol token", %{token: token} do + assert {:ok, %{status: 401, headers: headers}} = + raw_websocket_handshake("?stream=user", [ + {"sec-websocket-protocol", "invalid"} + ]) + + refute {"sec-websocket-protocol", token.token} in headers + refute List.keymember?(headers, "sec-websocket-protocol", 0) + end + test "prefers sec-websocket-protocol token over query access_token", %{ token: token, user: user @@ -450,12 +516,12 @@ defmodule Pleroma.Integration.MastodonWebsocketTest do assert {:ok, json} = Jason.decode(json["payload"]) view_json = - Pleroma.Web.MastodonAPI.StatusView.render("show.json", - activity: activity, - for: reading_user + atom_key_to_string( + StatusView.render("show.json", + activity: activity, + for: reading_user + ) ) - |> Jason.encode!() - |> Jason.decode!() assert json == view_json end @@ -478,12 +544,12 @@ defmodule Pleroma.Integration.MastodonWebsocketTest do activity = Pleroma.Activity.normalize(activity) view_json = - Pleroma.Web.MastodonAPI.StatusView.render("show.json", - activity: activity, - for: reading_user + atom_key_to_string( + StatusView.render("show.json", + activity: activity, + for: reading_user + ) ) - |> Jason.encode!() - |> Jason.decode!() assert {:ok, %{"event" => "status.update", "payload" => ^view_json}} = decode_json(raw_json) end diff --git a/test/pleroma/web/activity_pub/activity_pub_test.exs b/test/pleroma/web/activity_pub/activity_pub_test.exs index 9aafc41a5..dd7a82a55 100644 --- a/test/pleroma/web/activity_pub/activity_pub_test.exs +++ b/test/pleroma/web/activity_pub/activity_pub_test.exs @@ -415,6 +415,105 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do assert %{data: %{"id" => ^object_url}} = Object.get_by_ap_id(object_url) end + test "fetches user featured collection from the first collection page" do + ap_id = "https://example.com/users/lain" + featured_url = "https://example.com/users/lain/collections/featured" + first_page_url = "#{featured_url}?page=true" + + user_data = + "test/fixtures/users_mock/user.json" + |> File.read!() + |> String.replace("{{nickname}}", "lain") + |> Jason.decode!() + |> Map.put("featured", featured_url) + |> Jason.encode!() + + object_id = Ecto.UUID.generate() + object_url = "https://example.com/objects/#{object_id}" + + featured_data = + Jason.encode!(%{ + "id" => featured_url, + "type" => "OrderedCollection", + "first" => first_page_url + }) + + first_page_data = + Jason.encode!(%{ + "id" => first_page_url, + "type" => "OrderedCollectionPage", + "partOf" => featured_url, + "orderedItems" => [object_url] + }) + + object_data = + "test/fixtures/statuses/note.json" + |> File.read!() + |> String.replace("{{object_id}}", object_id) + |> String.replace("{{nickname}}", "lain") + + Tesla.Mock.mock(fn + %{ + method: :get, + url: ^ap_id + } -> + %Tesla.Env{ + status: 200, + body: user_data, + headers: [{"content-type", "application/activity+json"}] + } + + %{ + method: :get, + url: ^featured_url + } -> + %Tesla.Env{ + status: 200, + body: featured_data, + headers: [{"content-type", "application/activity+json"}] + } + + %{ + method: :get, + url: ^first_page_url + } -> + %Tesla.Env{ + status: 200, + body: first_page_data, + headers: [{"content-type", "application/activity+json"}] + } + + %{ + method: :get, + url: ^object_url + } -> + %Tesla.Env{ + status: 200, + body: object_data, + headers: [{"content-type", "application/activity+json"}] + } + end) + + refute capture_log(fn -> + {:ok, user} = ActivityPub.make_user_from_ap_id(ap_id) + + assert_enqueued( + worker: Pleroma.Workers.RemoteFetcherWorker, + args: %{ + "op" => "fetch_remote", + "id" => object_url, + "depth" => 1 + } + ) + + Pleroma.Tests.ObanHelpers.perform_all() + + assert user.featured_address == featured_url + assert Map.has_key?(user.pinned_objects, object_url) + assert %{data: %{"id" => ^object_url}} = Object.get_by_ap_id(object_url) + end) =~ "Could not parse featured collection" + end + test "fetches user birthday information from misskey" do user_id = "https://misskey.io/@mkljczk" @@ -2818,4 +2917,38 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do "first" => "https://social.example/users/alice/collections/featured?page=true" }) end + + test "fetch_and_prepare_featured_from_ap_id handles embedded first collection pages" do + featured_url = "https://social.example/users/alice/collections/featured" + first_page_url = "#{featured_url}?page=true" + object_url = "https://social.example/objects/1" + + featured_data = + Jason.encode!(%{ + "id" => featured_url, + "type" => "OrderedCollection", + "first" => %{ + "id" => first_page_url, + "type" => "OrderedCollectionPage", + "partOf" => featured_url, + "orderedItems" => [object_url] + } + }) + + Tesla.Mock.mock(fn + %{method: :get, url: ^featured_url} -> + %Tesla.Env{ + status: 200, + body: featured_data, + headers: [{"content-type", "application/activity+json"}] + } + end) + + refute capture_log(fn -> + assert {:ok, pinned_objects} = + ActivityPub.fetch_and_prepare_featured_from_ap_id(featured_url) + + assert Map.has_key?(pinned_objects, object_url) + end) =~ "Could not parse featured collection" + end end diff --git a/test/pleroma/web/mastodon_api/controllers/notification_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/notification_controller_test.exs index dd4b7389e..55df41ec6 100644 --- a/test/pleroma/web/mastodon_api/controllers/notification_controller_test.exs +++ b/test/pleroma/web/mastodon_api/controllers/notification_controller_test.exs @@ -577,6 +577,298 @@ defmodule Pleroma.Web.MastodonAPI.NotificationControllerTest do assert [%{"id" => ^notification4_id}, %{"id" => ^notification3_id}] = result end + describe "GET /api/v2/notifications" do + test "groups favourite notifications for the same status" do + %{user: user, conn: conn} = oauth_access(["read:notifications"]) + other_user1 = insert(:user) + other_user2 = insert(:user) + + {:ok, status} = CommonAPI.post(user, %{status: "hello"}) + {:ok, _} = CommonAPI.favorite(status.id, other_user1) + {:ok, _} = CommonAPI.favorite(status.id, other_user2) + + notifications = Repo.all(Notification) + notification_ids = Enum.map(notifications, &to_string(&1.id)) + assert [persisted_group_key] = notifications |> Enum.map(& &1.group_key) |> Enum.uniq() + assert is_binary(persisted_group_key) + + result = + conn + |> get("/api/v2/notifications") + |> json_response_and_validate_schema(200) + + assert [%{"id" => account_id1}, %{"id" => account_id2}] = result["accounts"] + assert account_id1 in [other_user1.id, other_user2.id] + assert account_id2 in [other_user1.id, other_user2.id] + assert [%{"id" => status_id}] = result["statuses"] + assert status_id == status.id + + assert [group] = result["notification_groups"] + assert group["type"] == "favourite" + assert group["notifications_count"] == 2 + assert group["status_id"] == status.id + assert group["most_recent_notification_id"] in notification_ids + assert group["page_min_id"] in notification_ids + assert group["page_max_id"] in notification_ids + assert group["latest_page_notification_at"] + assert Enum.sort(group["sample_account_ids"]) == Enum.sort([other_user1.id, other_user2.id]) + group_key = group["group_key"] + assert group_key == persisted_group_key + + assert [%{"group_key" => ^group_key}, %{"group_key" => ^group_key}] = + conn + |> get("/api/v1/notifications") + |> json_response_and_validate_schema(200) + + assert %{"notification_groups" => [shown_group]} = + conn + |> get("/api/v2/notifications/#{group_key}") + |> json_response_and_validate_schema(200) + + assert shown_group["group_key"] == group_key + assert shown_group["notifications_count"] == 2 + refute Map.has_key?(shown_group, "page_min_id") + refute Map.has_key?(shown_group, "page_max_id") + refute Map.has_key?(shown_group, "latest_page_notification_at") + end + + test "round-trips ungrouped group keys when grouped_types excludes a type" do + %{user: user, conn: conn} = oauth_access(["read:notifications"]) + other_user = insert(:user) + + {:ok, status} = CommonAPI.post(user, %{status: "hello"}) + {:ok, _} = CommonAPI.favorite(status.id, other_user) + + %{"notification_groups" => [%{"group_key" => "ungrouped-" <> _ = group_key}]} = + conn + |> get("/api/v2/notifications?grouped_types[]=reblog") + |> json_response_and_validate_schema(200) + + assert %{ + "notification_groups" => [%{"group_key" => ^group_key, "notifications_count" => 1}] + } = + conn + |> get("/api/v2/notifications/#{group_key}") + |> json_response_and_validate_schema(200) + end + + test "keeps legacy notifications without persisted group keys ungrouped" do + %{user: user, conn: conn} = oauth_access(["read:notifications"]) + other_user1 = insert(:user) + other_user2 = insert(:user) + + {:ok, status} = CommonAPI.post(user, %{status: "hello"}) + {:ok, favorite1} = CommonAPI.favorite(status.id, other_user1) + {:ok, favorite2} = CommonAPI.favorite(status.id, other_user2) + + notification_ids = + [favorite1, favorite2] + |> Enum.map(&get_notification_id_by_activity/1) + + Repo.update_all(Notification, set: [group_key: nil]) + + expected_group_keys = Enum.map(notification_ids, &"ungrouped-#{&1}") + + assert [%{"group_key" => group_key1}, %{"group_key" => group_key2}] = + conn + |> get("/api/v1/notifications") + |> json_response_and_validate_schema(200) + + assert Enum.sort([group_key1, group_key2]) == Enum.sort(expected_group_keys) + + %{"notification_groups" => groups} = + conn + |> get("/api/v2/notifications") + |> json_response_and_validate_schema(200) + + assert Enum.sort(Enum.map(groups, & &1["group_key"])) == Enum.sort(expected_group_keys) + assert Enum.all?(groups, &(&1["notifications_count"] == 1)) + + group_key = List.first(expected_group_keys) + + assert %{ + "notification_groups" => [%{"group_key" => ^group_key, "notifications_count" => 1}] + } = + conn + |> get("/api/v2/notifications/#{group_key}") + |> json_response_and_validate_schema(200) + end + + test "paginates notification groups instead of raw notifications" do + %{user: user, conn: conn} = oauth_access(["read:notifications"]) + other_user1 = insert(:user) + other_user2 = insert(:user) + other_user3 = insert(:user) + + {:ok, older_status} = CommonAPI.post(user, %{status: "older"}) + {:ok, _} = CommonAPI.favorite(older_status.id, other_user3) + + {:ok, newer_status} = CommonAPI.post(user, %{status: "newer"}) + {:ok, _} = CommonAPI.favorite(newer_status.id, other_user1) + {:ok, _} = CommonAPI.favorite(newer_status.id, other_user2) + older_status_id = older_status.id + newer_status_id = newer_status.id + + %{"notification_groups" => groups} = + conn + |> get("/api/v2/notifications?limit=2") + |> json_response_and_validate_schema(200) + + assert [ + %{"status_id" => ^newer_status_id, "notifications_count" => 2}, + %{"status_id" => ^older_status_id, "notifications_count" => 1} + ] = groups + end + + test "uses group-level cursors when paginating notification groups" do + %{user: user, conn: conn} = oauth_access(["read:notifications"]) + other_user1 = insert(:user) + other_user2 = insert(:user) + other_user3 = insert(:user) + + {:ok, older_status} = CommonAPI.post(user, %{status: "older"}) + {:ok, _} = CommonAPI.favorite(older_status.id, other_user3) + + {:ok, newer_status} = CommonAPI.post(user, %{status: "newer"}) + {:ok, _} = CommonAPI.favorite(newer_status.id, other_user1) + {:ok, _} = CommonAPI.favorite(newer_status.id, other_user2) + older_status_id = older_status.id + newer_status_id = newer_status.id + + assert %{ + "notification_groups" => [ + %{ + "status_id" => ^newer_status_id, + "notifications_count" => 2, + "most_recent_notification_id" => cursor + } + ] + } = + conn + |> get("/api/v2/notifications?limit=1") + |> json_response_and_validate_schema(200) + + assert %{ + "notification_groups" => [ + %{"status_id" => ^older_status_id, "notifications_count" => 1} + ] + } = + conn + |> get("/api/v2/notifications?limit=1&max_id=#{cursor}") + |> json_response_and_validate_schema(200) + end + + test "returns total notification count for a partially represented group" do + %{user: user, conn: conn} = oauth_access(["read:notifications"]) + other_user1 = insert(:user) + other_user2 = insert(:user) + other_user3 = insert(:user) + + {:ok, grouped_status} = CommonAPI.post(user, %{status: "grouped"}) + {:ok, _} = CommonAPI.favorite(grouped_status.id, other_user1) + + {:ok, other_status} = CommonAPI.post(user, %{status: "other"}) + {:ok, _} = CommonAPI.favorite(other_status.id, other_user3) + {:ok, _} = CommonAPI.favorite(grouped_status.id, other_user2) + grouped_status_id = grouped_status.id + + assert %{ + "notification_groups" => [ + %{"status_id" => ^grouped_status_id, "notifications_count" => 2} + ] + } = + conn + |> get("/api/v2/notifications?limit=1") + |> json_response_and_validate_schema(200) + end + + test "counts unread notification groups" do + %{user: user, conn: conn} = oauth_access(["read:notifications"]) + other_user1 = insert(:user) + other_user2 = insert(:user) + mentioner = insert(:user) + + {:ok, status} = CommonAPI.post(user, %{status: "hello"}) + {:ok, _} = CommonAPI.favorite(status.id, other_user1) + {:ok, _} = CommonAPI.favorite(status.id, other_user2) + {:ok, _} = CommonAPI.post(mentioner, %{status: "hi @#{user.nickname}"}) + + assert %{"count" => 2} = + conn + |> get("/api/v2/notifications/unread_count") + |> json_response_and_validate_schema(200) + end + + test "counts unread notification groups newer than the notification marker" do + %{user: user, conn: conn} = oauth_access(["read:notifications"]) + other_user = insert(:user) + mentioner = insert(:user) + + {:ok, status} = CommonAPI.post(user, %{status: "older"}) + {:ok, favorite_activity} = CommonAPI.favorite(status.id, other_user) + marker_id = get_notification_id_by_activity(favorite_activity) + + {:ok, _} = CommonAPI.post(mentioner, %{status: "newer @#{user.nickname}"}) + + {:ok, _} = + Pleroma.Marker.upsert(user, %{"notifications" => %{"last_read_id" => marker_id}}) + + assert %{"count" => 1} = + conn + |> get("/api/v2/notifications/unread_count") + |> json_response_and_validate_schema(200) + end + + test "does not preserve stale cursor params in grouped notification link headers" do + %{user: user, conn: conn} = oauth_access(["read:notifications"]) + other_user = insert(:user) + + {:ok, activity1} = CommonAPI.post(other_user, %{status: "one @#{user.nickname}"}) + {:ok, _activity2} = CommonAPI.post(other_user, %{status: "two @#{user.nickname}"}) + {:ok, activity3} = CommonAPI.post(other_user, %{status: "three @#{user.nickname}"}) + + notification1_id = get_notification_id_by_activity(activity1) + notification3_id = get_notification_id_by_activity(activity3) + + conn = get(conn, "/api/v2/notifications?since_id=#{notification1_id}&limit=1") + + assert [link_header] = get_resp_header(conn, "link") + assert link_header =~ ~r/max_id=#{notification3_id}/ + refute link_header =~ "since_id=" + end + + test "lists accounts from a notification group" do + %{user: user, conn: conn} = oauth_access(["read:notifications"]) + + {other_users, %{"group_key" => group_key, "sample_account_ids" => sample_account_ids}} = + create_favourite_notification_group(user, conn) + + assert length(sample_account_ids) == 8 + + account_ids = + conn + |> get("/api/v2/notifications/#{group_key}/accounts") + |> json_response_and_validate_schema(200) + |> Enum.map(& &1["id"]) + + assert Enum.sort(account_ids) == Enum.sort(Enum.map(other_users, & &1.id)) + end + + test "dismisses a notification group" do + %{user: user, conn: conn} = oauth_access(["read:notifications", "write:notifications"]) + + {_other_users, %{"group_key" => group_key}} = + create_favourite_notification_group(user, conn) + + assert %{} = + conn + |> post("/api/v2/notifications/#{group_key}/dismiss") + |> json_response_and_validate_schema(200) + + assert [] = Notification.for_user(user) + end + end + test "doesn't see notifications after muting user with notifications" do %{user: user, conn: conn} = oauth_access(["read:notifications"]) user2 = insert(:user) @@ -708,6 +1000,29 @@ defmodule Pleroma.Web.MastodonAPI.NotificationControllerTest do |> to_string() end + defp create_favourite_notification_group(user, conn) do + other_users = insert_list(9, :user) + + {:ok, status} = CommonAPI.post(user, %{status: "hello"}) + + Enum.each(other_users, fn other_user -> + {:ok, _} = CommonAPI.favorite(status.id, other_user) + end) + + %{ + "notification_groups" => [ + %{ + "notifications_count" => 9 + } = group + ] + } = + conn + |> get("/api/v2/notifications") + |> json_response_and_validate_schema(200) + + {other_users, group} + end + defp params_to_query(%{} = params) do Enum.map_join(params, "&", fn {k, v} when is_list(v) -> Enum.map_join(v, "&", &"#{k}[]=#{&1}") diff --git a/test/pleroma/web/mastodon_api/views/instance_view_test.exs b/test/pleroma/web/mastodon_api/views/instance_view_test.exs new file mode 100644 index 000000000..66485be0e --- /dev/null +++ b/test/pleroma/web/mastodon_api/views/instance_view_test.exs @@ -0,0 +1,675 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2026 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.MastodonAPI.InstanceViewTest do + use Pleroma.DataCase, async: true + + import Pleroma.Factory + + alias Pleroma.Language.LanguageDetectorMock + alias Pleroma.StaticStubbedConfigMock + alias Pleroma.Web.MastodonAPI.InstanceView + + import Mox + + @common_information_keys [ + :languages, + :rules, + :title, + :version + ] + + # Used for filtering out values from configuration.json render in configuration2.json render + @configuration2_filter [ + {:accounts, + [ + :max_pinned_statuses, + :max_profile_fields, + :profile_field_name_limit, + :profile_field_value_limit + ]}, + {:statuses, [:characters_reserved_per_url]}, + {:urls, [:streaming, :status]}, + {:vapid, [:public_key]}, + {:translation, [:enabled]}, + {:timelines_access, [:live_feeds, :hashtag_feeds, :trending_link_feeds]} + ] + + @pleroma_configuration_filter [{:metadata, [:features, :federation]}] + + @pleroma_configuration2_filter [ + {:metadata, + [ + :avatar_upload_limit, + :background_upload_limit, + :banner_upload_limit, + :background_image, + :chat_limit, + :description_limit, + :shout_limit + ]} + ] + + @show_keys @common_information_keys ++ + [ + :uri, + :description, + :short_description, + :email, + :urls, + :stats, + :thumbnail, + :registrations, + :approval_required, + :contact_account, + :configuration, + # Extra (not present in Mastodon) + :max_toot_chars, + :max_media_attachments, + :poll_limits, + :upload_limit, + :avatar_upload_limit, + :background_upload_limit, + :banner_upload_limit, + :background_image, + :shout_limit, + :description_limit, + :chat_limit, + :pleroma + ] + + # Checked in other check_* functions + @show_filter @common_information_keys ++ [:contact_account, :configuration, :pleroma] + + @show2_keys @common_information_keys ++ + [ + :domain, + :source_url, + :description, + :usage, + :thumbnail, + :configuration, + :registrations, + :contact, + # Extra (not present in Mastodon) + :pleroma + ] + + @show2_filter @common_information_keys ++ + [:thumbnail, :configuration, :registrations, :contact, :pleroma] + + @features [ + "pleroma_api", + "mastodon_api", + "mastodon_api_grouped_notifications", + "mastodon_api_streaming", + "polls", + "v2_suggestions", + "pleroma_explicit_addressing", + "shareable_emoji_packs", + "multifetch", + "pleroma:api/v1/notifications:include_types_filter", + "editing", + "quote_posting", + "pleroma_emoji_reactions", + "pleroma_custom_emoji_reactions", + "pleroma_chat_messages", + "pleroma:pin_chats", + "pleroma:get:main/ostatus", + "pleroma:group_actors", + "pleroma:bookmark_folders", + "pleroma:block_expiration" + ] + + @configurable_features [ + "blockers_visible", + "media_proxy", + "gopher", + "chat", + "shout", + "relay", + "safe_dm_mentions", + "exposable_reactions", + "profile_directory", + "pleroma:language_detection" + ] + @configurable_features_flags [ + [:activitypub, :blockers_visible], + [:media_proxy, :enabled], + [:gopher, :enabled], + [:shout, :enabled], + [:instance, :allow_relay], + [:instance, :safe_dm_mentions], + [:instance, :show_reactions], + [:instance, :profile_directory] + ] + + # Filters out values from one layer deep nested Maps. + # Filtering types: + # - :inclusive -> filters out everything that is not in the filters + # - :discriminative -> filters out what is in the filters + defp filter_render(input, filter, type, acc) + + defp filter_render(_input_rest, [], type, acc) when type == :inclusive, do: acc + + defp filter_render(input_rest, [], type, acc) when type == :discriminative, + do: Map.merge(input_rest, acc) + + defp filter_render(input, filter, type, acc) do + {{filter_key, filters}, filter_rest} = List.pop_at(filter, 0) + {config_key_values, input_rest} = Map.pop!(input, filter_key) + + filtered = + case type do + :inclusive -> + Map.reject(config_key_values, fn {key, _} -> key not in filters end) + + :discriminative -> + Map.reject(config_key_values, fn {key, _} -> key in filters end) + end + + filter_render(input_rest, filter_rest, type, Map.put(acc, filter_key, filtered)) + end + + defp check_common_information(info) do + filtered_info = Map.reject(info, fn {key, _value} -> key not in @common_information_keys end) + + expected = %{ + languages: ["en"], + rules: InstanceView.render("rules.json", %{}), + title: "Pleroma", + version: + InstanceView.mastodon_api_level() <> + " (compatible; #{Pleroma.Application.named_version()})" + } + + Map.equal?(expected, filtered_info) + end + + defp check_thumbnail2(info) do + thumbnail = info[:thumbnail] + + expected = %{ + url: + URI.merge(Pleroma.Web.Endpoint.url(), "http://localhost:4001/instance/thumbnail.jpeg") + |> to_string + } + + expected == thumbnail + end + + defp check_contact(info, user) do + contact = info[:contact] + + expected = %{ + email: "admin@example.com", + account: Pleroma.Web.MastodonAPI.AccountView.render("show.json", %{user: user, for: nil}) + } + + Map.equal?(expected, contact) + end + + defp check_contact_account(info, user) do + contact = info[:contact_account] + view = Pleroma.Web.MastodonAPI.AccountView.render("show.json", %{user: user, for: nil}) + + contact == view + end + + defp check_configuration(info) do + configuration = info[:configuration] + + expected = %{ + accounts: %{ + max_featured_tags: 0 + }, + statuses: %{ + max_characters: 5_000, + max_media_attachments: 1_000 + }, + media_attachments: %{ + image_size_limit: 16_000_000, + video_size_limit: 16_000_000, + supported_mime_types: ["application/octet-stream"] + }, + polls: %{ + max_options: 20, + max_characters_per_option: 200, + min_expiration: 0, + max_expiration: 365 * 24 * 60 * 60 + } + } + + Map.equal?(expected, configuration) + end + + defp check_configuration2(info) do + configuration = + info[:configuration] + |> filter_render(@configuration2_filter, :inclusive, %{}) + + expected = %{ + accounts: %{ + max_pinned_statuses: 1, + max_profile_fields: 10, + profile_field_name_limit: 512, + profile_field_value_limit: 2048 + }, + statuses: %{ + characters_reserved_per_url: 0 + }, + urls: %{ + streaming: Pleroma.Web.Endpoint.websocket_url(), + status: nil + }, + vapid: %{ + public_key: Keyword.get(Pleroma.Web.Push.vapid_config(), :public_key) + }, + translation: %{enabled: true}, + timelines_access: %{ + live_feeds: %{local: "public", remote: "public"}, + hashtag_feeds: %{local: "public", remote: "public"}, + # not implemented in Pleroma + trending_link_feeds: %{ + local: "disabled", + remote: "disabled" + } + } + } + + Map.equal?(expected, configuration) + end + + defp check_registrations2(info) do + registrations = info[:registrations] + + expected = %{ + enabled: true, + approval_required: false, + message: nil, + url: nil + } + + Map.equal?(expected, registrations) + end + + defp check_pleroma_configuration(info) do + configuration = + info[:pleroma] + |> filter_render(@pleroma_configuration_filter, :discriminative, %{}) + + expected = %{ + metadata: %{ + account_activation_required: false, + fields_limits: %{ + max_fields: 10, + max_remote_fields: 20, + name_length: 512, + value_length: 2048 + }, + post_formats: [ + "text/plain", + "text/html", + "text/markdown", + "text/bbcode", + "text/x.misskeymarkdown" + ], + birthday_required: false, + birthday_min_age: 0, + translation: %{ + source_languages: ["en", "pl"], + target_languages: ["en", "pl"] + }, + base_urls: %{}, + markup: %{ + allow_inline_images: true, + allow_headings: false, + allow_tables: false + } + }, + stats: %{mau: Pleroma.User.active_user_count()}, + vapid_public_key: Keyword.get(Pleroma.Web.Push.vapid_config(), :public_key) + } + + Map.equal?(expected, configuration) + end + + defp check_pleroma_configuration2(info) do + configuration = + info[:pleroma] + |> filter_render(@pleroma_configuration2_filter, :inclusive, %{}) + + expected = %{ + metadata: %{ + avatar_upload_limit: 2_000_000, + background_upload_limit: 4_000_000, + banner_upload_limit: 4_000_000, + background_image: Pleroma.Web.Endpoint.url() <> "/images/city.jpg", + chat_limit: 5_000, + description_limit: 5_000, + shout_limit: 5_000 + } + } + + Map.equal?(expected, configuration) + end + + defp check_show(info) do + filter = @show_filter + filtered_info = Map.reject(info, fn {key, _value} -> key in filter end) + + expected = %{ + uri: Pleroma.Web.WebFinger.host(), + description: "Pleroma: An efficient and flexible fediverse server", + short_description: "", + email: "admin@example.com", + urls: %{ + streaming_api: Pleroma.Web.Endpoint.websocket_url() + }, + stats: Pleroma.Stats.get_stats(), + thumbnail: "http://localhost:4001/instance/thumbnail.jpeg", + registrations: true, + approval_required: false, + max_toot_chars: 5_000, + max_media_attachments: 1_000, + poll_limits: %{ + max_options: 20, + max_option_chars: 200, + min_expiration: 0, + max_expiration: 365 * 24 * 60 * 60 + }, + upload_limit: 16_000_000, + avatar_upload_limit: 2_000_000, + background_upload_limit: 4_000_000, + banner_upload_limit: 4_000_000, + background_image: Pleroma.Web.Endpoint.url() <> "/images/city.jpg", + shout_limit: 5_000, + description_limit: 5_000, + chat_limit: 5_000 + } + + Map.equal?(expected, filtered_info) + end + + defp check_show2(info) do + filter = @show2_filter + filtered_info = Map.reject(info, fn {key, _value} -> key in filter end) + + expected = %{ + domain: Pleroma.Web.WebFinger.host(), + source_url: Pleroma.Application.repository(), + description: "", + usage: %{users: %{active_month: Pleroma.User.active_user_count()}} + } + + Map.equal?(expected, filtered_info) + end + + # When this fails, a new feature flag was probably added. Add it to the macros above. + describe "uncofigurable features" do + setup do: Enum.each(@configurable_features_flags, &clear_config(&1, false)) + + test "returns always enabled features" do + features = @features + + assert Enum.sort(InstanceView.features()) == Enum.sort(features) + end + end + + describe "all features" do + setup do + Enum.each(@configurable_features_flags, &clear_config(&1, true)) + + # Stub the StaticStubbedConfigMock to return our mock for the provider + StaticStubbedConfigMock + |> stub(:get, fn + [Pleroma.Language.LanguageDetector, :provider] -> LanguageDetectorMock + _other -> nil + end) + + # Stub the LanguageDetectorMock with default implementations + LanguageDetectorMock + |> stub(:missing_dependencies, fn -> [] end) + |> stub(:configured?, fn -> true end) + + :ok + end + + test "returns all features including configurable ones" do + features = @features ++ @configurable_features + + assert Enum.sort(InstanceView.features()) == Enum.sort(features) + end + end + + describe "federation/0" do + setup do: clear_config([:mrf, :transparency], true) + + test "quarantined instances" do + clear_config([:instance, :quarantined_instances], [{"quarantine.example.com", "crimes"}]) + + output = InstanceView.federation() + assert %{quarantined_instances: ["quarantine.example.com"]} = output + + assert %{ + quarantined_instances_info: %{ + "quarantined_instances" => %{"quarantine.example.com" => %{"reason" => "crimes"}} + } + } = output + end + + test "rejected instances" do + clear_config([:instance, :rejected_instances], [ + {"rejected.example.com", "not enough #cofe posting"} + ]) + + output = InstanceView.federation() + + assert %{ + rejected_instances: %{ + "rejected.example.com" => %{"reason" => "not enough #cofe posting"} + } + } = output + end + + test "federating" do + clear_config([:instance, :federating], true) + + output = InstanceView.federation() + assert %{enabled: true} = output + end + + test "transparency" do + clear_config([:instance, :quarantined_instances], [{"quarantine.example.com", "crimes"}]) + clear_config([:instance, :federating], true) + clear_config([:mrf, :transparency], false) + + clear_config([:instance, :rejected_instances], [ + {"rejected.example.com", "not enough #cofe posting"} + ]) + + output = InstanceView.federation() + assert %{enabled: true} == output + end + + test "MRFs" do + clear_config([:mrf, :policies], [Pleroma.Web.ActivityPub.MRF.SimplePolicy]) + clear_config([:mrf_simple, :reject], [{"rejected.example.com", "not enough #cofe posting"}]) + + clear_config([:mrf, :transparency_exclusions], [ + {"media1.example.com", "the Fediverse doesn't need to know"} + ]) + + clear_config([:mrf_simple, :media_removal], [ + {"media1.example.com", "NSFW"}, + {"media2.example.com", "usual suspects"} + ]) + + output = InstanceView.federation() + + expected = %{ + mrf_simple: %{ + reject: ["rejected.example.com"], + media_removal: ["media2.example.com"] + }, + mrf_hashtag: %{ + sensitive: ["nsfw"] + }, + exclusions: true, + mrf_policies: ["SimplePolicy", "HashtagPolicy"], + mrf_simple_info: %{ + reject: %{ + "rejected.example.com" => %{"reason" => "not enough #cofe posting"} + }, + media_removal: %{"media2.example.com" => %{"reason" => "usual suspects"}} + } + } + + assert expected.mrf_simple.reject == output.mrf_simple.reject + assert expected.mrf_simple.media_removal == output.mrf_simple.media_removal + assert expected.mrf_simple_info == output.mrf_simple_info + assert expected.exclusions == output.exclusions + end + end + + describe "render show.json/show2.json" do + setup do + Pleroma.Rule.create(%{text: "rule 1"}) + Pleroma.Rule.create(%{text: "rule 2"}) + + user = insert(:user) + + # contact in show2.json + clear_config([:instance, :contact_username], user.nickname) + + # Pleroma.Language.Translation.configured?() -> true + clear_config([Pleroma.Language.Translation, :provider], TranslationMock) + + # Deprecated, but still used in view + clear_config([:instance, :chat_limit], 5_000) + + %{user: user} + end + + test "renders show.json properly", %{user: user} do + output = InstanceView.render("show.json", %{}) + expected_keys = Enum.sort(@show_keys) + + assert check_common_information(output) + assert check_show(output) + assert check_contact_account(output, user) + assert check_configuration(output) + assert check_pleroma_configuration(output) + assert expected_keys == Enum.sort(Map.keys(output)) + end + + test "renders show2.json properly", %{user: user} do + output = InstanceView.render("show2.json", %{}) + expected_keys = Enum.sort(@show2_keys) + + assert check_common_information(output) + assert check_show2(output) + assert check_thumbnail2(output) + assert check_configuration2(output) + assert check_registrations2(output) + assert check_contact(output, user) + assert check_pleroma_configuration2(output) + assert expected_keys == Enum.sort(Map.keys(output)) + end + end + + describe "render rule.json" do + setup do + rule = Pleroma.Rule.create(%{text: "hambaga"}) + + %{rule: rule} + end + + test "renders properly", %{rule: rule} do + output = InstanceView.render("rule.json", %{rule: rule}) + + assert %{text: "hambaga"} = output + end + end + + describe "render rules.json" do + setup do + Pleroma.Rule.create(%{text: "rule 1"}) + Pleroma.Rule.create(%{text: "rule 2"}) + Pleroma.Rule.create(%{text: "rule 3"}) + Pleroma.Rule.create(%{text: "rule 4"}) + + :ok + end + + test "renders properly" do + output = InstanceView.render("rules.json", %{}) + keys = Enum.map(output, fn %{text: text} -> text end) + + assert length(keys) == 4 + assert Enum.all?(output, fn %{text: "rule " <> num} -> String.to_integer(num) end) + end + end + + describe "render domain_blocks.json" do + setup do + clear_config([:mrf, :transparency], true) + clear_config([:mrf_simple, :media_removal], [{"media1.example.com", "NSFW"}]) + + clear_config([:mrf, :transparency_exclusions], [ + {"removed1.example.com", "the Fediverse doesn't need to know"} + ]) + + clear_config([:mrf_simple, :reject], [ + {"rejected.example.com", "not enough #cofe posting"}, + {"rejected-without-reason.example.com", ""} + ]) + + clear_config([:mrf_simple, :federated_timeline_removal], [ + {"removed1.example.com", "spam"}, + {"removed2.example.com", "more spam"} + ]) + + :ok + end + + test "renders properly" do + output = InstanceView.render("domain_blocks.json", %{}) + + # Media removal is not in @block_severities in the view. + # NOTE: Order of these two matters! + expected = [ + %{ + domain: "removed2.example.com", + comment: "more spam", + severity: "silence", + digest: "5f6649cfc780fac5172dbbcc8a552953f6621a14368d82728b6e8faf51295dc0" + }, + %{ + domain: "rejected.example.com", + comment: "not enough #cofe posting", + severity: "suspend", + digest: "c15082d347fb7eccd251e199a98dcdce94a8dfc1d33d6ece80889eed374b6f92" + }, + %{ + domain: "rejected-without-reason.example.com", + severity: "suspend", + digest: "48297155018c626c8867ca0bba4a2621eccdf51c6f7ac91b7099ed8b2ee9863f" + } + ] + + assert expected == output + end + end + + describe "render translation_languages.json" do + setup do: clear_config([Pleroma.Language.Translation, :provider], TranslationMock) + + test "returns language matrix" do + output = InstanceView.render("translation_languages.json", %{}) + + assert %{"en" => ["pl"], "pl" => ["en"]} == output + end + end +end diff --git a/test/pleroma/web/mastodon_api/views/notification_view_test.exs b/test/pleroma/web/mastodon_api/views/notification_view_test.exs index c8287bb79..2d0801589 100644 --- a/test/pleroma/web/mastodon_api/views/notification_view_test.exs +++ b/test/pleroma/web/mastodon_api/views/notification_view_test.exs @@ -98,10 +98,11 @@ defmodule Pleroma.Web.MastodonAPI.NotificationViewTest do {:ok, favorite_activity} = CommonAPI.favorite(create_activity.id, another_user) {:ok, [notification]} = Notification.create_notifications(favorite_activity) create_activity = Activity.get_by_id(create_activity.id) + assert is_binary(notification.group_key) expected = %{ id: to_string(notification.id), - group_key: "ungrouped-#{to_string(notification.id)}", + group_key: Notification.group_key(notification), pleroma: %{is_seen: false, is_muted: false}, type: "favourite", account: AccountView.render("show.json", %{user: another_user, for: user}), @@ -119,10 +120,11 @@ defmodule Pleroma.Web.MastodonAPI.NotificationViewTest do {:ok, reblog_activity} = CommonAPI.repeat(create_activity.id, another_user) {:ok, [notification]} = Notification.create_notifications(reblog_activity) reblog_activity = Activity.get_by_id(create_activity.id) + assert is_binary(notification.group_key) expected = %{ id: to_string(notification.id), - group_key: "ungrouped-#{to_string(notification.id)}", + group_key: Notification.group_key(notification), pleroma: %{is_seen: false, is_muted: false}, type: "reblog", account: AccountView.render("show.json", %{user: another_user, for: user}), @@ -138,10 +140,11 @@ defmodule Pleroma.Web.MastodonAPI.NotificationViewTest do followed = insert(:user) {:ok, followed, follower, _activity} = CommonAPI.follow(followed, follower) notification = Notification |> Repo.one() |> Repo.preload(:activity) + assert is_binary(notification.group_key) expected = %{ id: to_string(notification.id), - group_key: "ungrouped-#{to_string(notification.id)}", + group_key: Notification.group_key(notification), pleroma: %{is_seen: false, is_muted: false}, type: "follow", account: AccountView.render("show.json", %{user: follower, for: followed}), @@ -333,7 +336,7 @@ defmodule Pleroma.Web.MastodonAPI.NotificationViewTest do expected = %{ id: to_string(notification.id), - group_key: "ungrouped-#{to_string(notification.id)}", + group_key: Notification.group_key(notification), pleroma: %{is_seen: true, is_muted: true}, type: "favourite", account: AccountView.render("show.json", %{user: another_user, for: user}), diff --git a/test/pleroma/web/node_info_test.exs b/test/pleroma/web/node_info_test.exs index afe4ebb36..e2bf3400a 100644 --- a/test/pleroma/web/node_info_test.exs +++ b/test/pleroma/web/node_info_test.exs @@ -160,6 +160,7 @@ defmodule Pleroma.Web.NodeInfoTest do default_features = [ "pleroma_api", "mastodon_api", + "mastodon_api_grouped_notifications", "mastodon_api_streaming", "polls", "pleroma_explicit_addressing", diff --git a/test/pleroma/web/rich_media/backfill_test.exs b/test/pleroma/web/rich_media/backfill_test.exs index 6d221fcf5..071f9b48a 100644 --- a/test/pleroma/web/rich_media/backfill_test.exs +++ b/test/pleroma/web/rich_media/backfill_test.exs @@ -5,12 +5,22 @@ defmodule Pleroma.Web.RichMedia.BackfillTest do use Pleroma.DataCase + alias Pleroma.Tests.ObanHelpers + alias Pleroma.Web.CommonAPI alias Pleroma.Web.RichMedia.Backfill alias Pleroma.Web.RichMedia.Card import Mox + import Pleroma.Factory - setup_all do: clear_config([:rich_media, :enabled], true) + setup do + clear_config([:rich_media, :enabled], true) + + Mox.stub_with(Pleroma.UnstubbedConfigMock, Pleroma.Test.StaticConfig) + Mox.stub_with(Pleroma.CachexMock, Pleroma.NullCache) + + :ok + end test "sets a negative cache entry for an error" do url = "https://bad.example.com/" @@ -23,4 +33,139 @@ defmodule Pleroma.Web.RichMedia.BackfillTest do Backfill.run(%{"url" => url}) end + + test "sets a warm_cache entry" do + url = "https://good.example.com/" + url_hash = Card.url_to_hash(url) + + Tesla.Mock.mock(fn %{url: ^url} -> + {:ok, + %Tesla.Env{ + status: 200, + body: "" + }} + end) + + Pleroma.CachexMock + |> expect(:put, fn :rich_media_cache, + ^url_hash, + %Pleroma.Web.RichMedia.Card{url_hash: ^url_hash} -> + {:ok, true} + end) + + Backfill.run(%{"url" => url}) + end + + test "streams out update when stream == true" do + url = "https://example.com" + user = insert(:user) + + Tesla.Mock.mock(fn %{url: ^url} -> + {:ok, + %Tesla.Env{ + status: 200, + body: "" + }} + end) + + {:ok, activity} = CommonAPI.post(user, %{status: "#cofe #{url}"}) + + Pleroma.CachexMock + |> expect(:put, fn :rich_media_cache, _, _ -> {:ok, true} end) + + Pleroma.Web.ActivityPub.ActivityPubMock + |> expect(:stream_out, fn %Pleroma.Activity{id: id} -> + assert id == activity.id + :ok + end) + + Backfill.run(%{"url" => url, "activity_id" => activity.id, "stream" => true}) + end + + test "does not stream out update when stream == false" do + url = "https://example.com" + user = insert(:user) + + Tesla.Mock.mock(fn %{url: ^url} -> + {:ok, + %Tesla.Env{ + status: 200, + body: "" + }} + end) + + {:ok, activity} = CommonAPI.post(user, %{status: "#cofe #{url}"}) + + Pleroma.CachexMock + |> expect(:put, fn :rich_media_cache, _, _ -> {:ok, true} end) + + Pleroma.Web.ActivityPub.ActivityPubMock + |> deny(:stream_out, 1) + + Backfill.run(%{"url" => url, "activity_id" => "#{activity.data["id"]}", "stream" => false}) + end + + # NOTE: Below two MastoAPI tests cover almost the same code paths. + # index.json will always prefetch rich media, while show.json will try to get the card and + # fetch it when it isn't cached (both use Card.get_by_activity in the end). + # So if index.json doesn't fetch the rich media, show.json will when it renders the post, + # hence why index.json test will only call ActivityPub.stream_out twice, + # if streaming is re-enabled for in both. + test "does not stream out in MastoAPI StatusView index" do + url = "https://example.com" + user = insert(:user) + + Tesla.Mock.mock(fn %{url: ^url} -> + {:ok, + %Tesla.Env{ + status: 200, + body: "" + }} + end) + + # CommonAPI federation processing will stream out once as a new post + Pleroma.Web.ActivityPub.ActivityPubMock + |> expect(:stream_out, 1, fn _ -> :ok end) + + {:ok, activity} = CommonAPI.post(user, %{status: "#cofe #{url}"}) + ObanHelpers.perform_all() + + # Clear cache to force backfill below + Pleroma.Activity.HTML.invalidate_cache_for(activity.id) + Pleroma.Web.RichMedia.Card.delete(url) + + Pleroma.Web.MastodonAPI.StatusView.render("index.json", %{ + activities: [activity], + as: :activity + }) + + ObanHelpers.perform_all() + end + + test "does not stream out in MastoAPI StatusView show" do + url = "https://example.com" + user = insert(:user) + + Tesla.Mock.mock(fn %{url: ^url} -> + {:ok, + %Tesla.Env{ + status: 200, + body: "" + }} + end) + + # CommonAPI federation processing will stream out once as a new post + Pleroma.Web.ActivityPub.ActivityPubMock + |> expect(:stream_out, 1, fn _ -> :ok end) + + {:ok, activity} = CommonAPI.post(user, %{status: "#cofe #{url}"}) + ObanHelpers.perform_all() + + # Clear cache to force backfill below + Pleroma.Activity.HTML.invalidate_cache_for(activity.id) + Pleroma.Web.RichMedia.Card.delete(url) + + Pleroma.Web.MastodonAPI.StatusView.render("show.json", activity: activity) + ObanHelpers.perform_all() + end end