diff --git a/.woodpecker/changelog.yaml b/.woodpecker/changelog.yaml index a3732c240..64062f17e 100644 --- a/.woodpecker/changelog.yaml +++ b/.woodpecker/changelog.yaml @@ -1,6 +1,5 @@ when: - event: pull_request - evaluate: 'CI_COMMIT_SOURCE_BRANCH != "weblate"' labels: platform: linux/amd64 diff --git a/.woodpecker/docker-armv7.yaml b/.woodpecker/docker-armv7.yaml deleted file mode 100644 index c921a56f9..000000000 --- a/.woodpecker/docker-armv7.yaml +++ /dev/null @@ -1,85 +0,0 @@ -when: - # Temporary PR validation while arm/v7 Docker builds are being brought up. - - event: pull_request - path: [ "**/*.ex", "**/*.eex", "**/*.exs", "mix.lock", ".woodpecker/**", "Dockerfile" ] - - event: push - branch: ${CI_REPO_DEFAULT_BRANCH} - path: [ "**/*.ex", "**/*.eex", "**/*.exs", "mix.lock", ".woodpecker/**", "Dockerfile" ] - - event: tag - - event: manual - branch: ${CI_REPO_DEFAULT_BRANCH} - - event: manual - branch: stable - -# Target the CAX21 arm64 host with native arm32 userspace support, not generic arm64 runners. -labels: - platform: linux/arm - armv7: native - -variables: - docker_variables: &docker_variables - repo: pleroma/pleroma - registry: git.pleroma.social - armv7_build_settings: &armv7_build_settings - <<: *docker_variables - build_args: - - ELIXIR_IMG=arm32v7/elixir - - ELIXIR_TAG=1.17.3-otp-26-alpine - - ALPINE_IMG=arm32v7/alpine - - ALPINE_VER=3.20 - extra_opts: --custom-platform=linux/arm/v7 - armv7_push_settings: &armv7_push_settings - <<: *armv7_build_settings - username: - from_secret: pleroma-ci-user - password: - from_secret: pleroma-ci-password - -steps: - docker-armv7-pr: - image: woodpeckerci/plugin-kaniko:2.3.1 - when: - - event: pull_request - settings: - <<: *armv7_build_settings - dry_run: true - tags: - - armv7-pr-check - - docker-develop-armv7: - image: woodpeckerci/plugin-kaniko:2.3.1 - when: - - event: push - branch: ${CI_REPO_DEFAULT_BRANCH} - - event: manual - branch: ${CI_REPO_DEFAULT_BRANCH} - settings: - <<: *armv7_push_settings - tags: - - latest-armv7 - - develop-armv7 - - ${CI_COMMIT_SHA:0:8}-armv7 - - docker-stable-armv7: - image: woodpeckerci/plugin-kaniko:2.3.1 - when: - - evaluate: 'CI_PIPELINE_EVENT == "manual" && CI_COMMIT_BRANCH == "stable" && CI_COMMIT_TAG == ""' - settings: - <<: *armv7_push_settings - tags: - - latest-armv7 - - stable-armv7 - - ${CI_COMMIT_SHA:0:8}-armv7 - - docker-stable-tag-armv7: - image: woodpeckerci/plugin-kaniko:2.3.1 - when: - - event: tag - - evaluate: 'CI_PIPELINE_EVENT == "manual" && CI_COMMIT_BRANCH == "stable" && CI_COMMIT_TAG != ""' - settings: - <<: *armv7_push_settings - tags: - - latest-armv7 - - stable-armv7 - - ${CI_COMMIT_SHA:0:8}-armv7 - - ${CI_COMMIT_TAG}-armv7 diff --git a/.woodpecker/docker.yaml b/.woodpecker/docker.yaml index 9a5be78e2..abc6bfa3b 100644 --- a/.woodpecker/docker.yaml +++ b/.woodpecker/docker.yaml @@ -62,18 +62,6 @@ steps: - stable-amd64 - ${CI_COMMIT_SHA:0:8}-amd64 - # FE workflow runs only on linux/amd64 - docker-stable-e2e-amd64: - image: *kaniko_image - when: - - evaluate: 'platform == "linux/amd64" && CI_PIPELINE_EVENT == "tag"' - - evaluate: 'platform == "linux/amd64" && CI_PIPELINE_EVENT == "manual" && CI_COMMIT_BRANCH == "stable"' - settings: - <<: *docker_variables - dockerfile: Dockerfile-e2e - tags: - - stable-e2e - docker-stable-tag-amd64: image: *kaniko_image when: diff --git a/.woodpecker/lint.yaml b/.woodpecker/lint.yaml index 8a5718c5d..0ab7441a8 100644 --- a/.woodpecker/lint.yaml +++ b/.woodpecker/lint.yaml @@ -1,7 +1,6 @@ when: - event: pull_request path: [ "**/*.ex", "**/*.eex", "**/*.exs", "mix.lock", ".woodpecker/**" ] - evaluate: 'CI_COMMIT_SOURCE_BRANCH != "weblate"' - event: push branch: ${CI_REPO_DEFAULT_BRANCH} path: [ "**/*.ex", "**/*.eex", "**/*.exs", "mix.lock", ".woodpecker/**" ] diff --git a/.woodpecker/unit-testing-elixir-1.15.yaml b/.woodpecker/unit-testing-elixir-1.15.yaml index a5bcc0ef3..a4a8fc266 100644 --- a/.woodpecker/unit-testing-elixir-1.15.yaml +++ b/.woodpecker/unit-testing-elixir-1.15.yaml @@ -1,7 +1,6 @@ when: - event: pull_request path: [ "**/*.ex", "**/*.eex", "**/*.exs", "mix.lock", ".woodpecker/**" ] - evaluate: 'CI_COMMIT_SOURCE_BRANCH != "weblate"' - event: push branch: ${CI_REPO_DEFAULT_BRANCH} path: [ "**/*.ex", "**/*.eex", "**/*.exs", "mix.lock", ".woodpecker/**" ] diff --git a/.woodpecker/unit-testing-elixir-1.18.yaml b/.woodpecker/unit-testing-elixir-1.18.yaml index 6e576fad9..9ad9eebc9 100644 --- a/.woodpecker/unit-testing-elixir-1.18.yaml +++ b/.woodpecker/unit-testing-elixir-1.18.yaml @@ -1,7 +1,6 @@ when: - event: pull_request path: [ "**/*.ex", "**/*.eex", "**/*.exs", "mix.lock", ".woodpecker/**" ] - evaluate: 'CI_COMMIT_SOURCE_BRANCH != "weblate"' - event: push branch: ${CI_REPO_DEFAULT_BRANCH} path: [ "**/*.ex", "**/*.eex", "**/*.exs", "mix.lock", ".woodpecker/**" ] diff --git a/Dockerfile b/Dockerfile index 5fd1cc0a1..11a17b7ad 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,10 +3,8 @@ ARG ELIXIR_IMG=hexpm/elixir ARG ELIXIR_VER=1.17.3 ARG ERLANG_VER=26.2.5.6 ARG ALPINE_VER=3.17.9 -ARG ELIXIR_TAG=${ELIXIR_VER}-erlang-${ERLANG_VER}-alpine-${ALPINE_VER} -ARG ALPINE_IMG=alpine -FROM ${ELIXIR_IMG}:${ELIXIR_TAG} AS build +FROM ${ELIXIR_IMG}:${ELIXIR_VER}-erlang-${ERLANG_VER}-alpine-${ALPINE_VER} AS build COPY . . @@ -22,7 +20,7 @@ RUN apk add git gcc g++ musl-dev make cmake file-dev vips-dev &&\ mkdir release &&\ mix release --path release -FROM ${ALPINE_IMG}:${ALPINE_VER} +FROM alpine:${ALPINE_VER} ARG BUILD_DATE ARG VCS_REF diff --git a/Dockerfile-e2e b/Dockerfile-e2e deleted file mode 100644 index df8bb39aa..000000000 --- a/Dockerfile-e2e +++ /dev/null @@ -1,61 +0,0 @@ -# https://hub.docker.com/r/hexpm/elixir/tags -ARG ELIXIR_IMG=docker.io/hexpm/elixir -ARG ALPINE_IMG=docker.io/alpine -ARG ELIXIR_VER=1.18.4 -ARG ERLANG_VER=27.3.4.11 -ARG ALPINE_VER=3.23.4 - -FROM ${ELIXIR_IMG}:${ELIXIR_VER}-erlang-${ERLANG_VER}-alpine-${ALPINE_VER} AS build - -COPY . . - -ENV MIX_ENV=prod -ENV VIX_COMPILATION_MODE=PLATFORM_PROVIDED_LIBVIPS - -RUN apk add git gcc g++ musl-dev make cmake file-dev vips-dev &&\ - echo "import Config" > config/prod.secret.exs &&\ - mix local.hex --force &&\ - mix local.rebar --force &&\ - mix deps.clean --all &&\ - mix deps.get --only prod &&\ - mkdir release &&\ - mix release --path release - -FROM ${ALPINE_IMG}:${ALPINE_VER} - -ARG BUILD_DATE -ARG VCS_REF - -LABEL maintainer="ops@pleroma.social" \ - org.opencontainers.image.title="pleroma" \ - org.opencontainers.image.description="Pleroma FE E2E test image" \ - org.opencontainers.image.authors="ops@pleroma.social" \ - org.opencontainers.image.vendor="pleroma.social" \ - org.opencontainers.image.documentation="https://git.pleroma.social/pleroma/pleroma" \ - org.opencontainers.image.licenses="AGPL-3.0" \ - org.opencontainers.image.url="https://pleroma.social" \ - org.opencontainers.image.revision=$VCS_REF \ - org.opencontainers.image.created=$BUILD_DATE - -ARG HOME=/opt/pleroma -ARG DATA=/var/lib/pleroma - -RUN apk update &&\ - apk add exiftool ffmpeg vips libmagic ncurses postgresql-client &&\ - adduser --system --shell /bin/false --home ${HOME} pleroma &&\ - mkdir -p ${DATA}/uploads &&\ - mkdir -p ${DATA}/static &&\ - chown -R pleroma ${DATA} &&\ - mkdir -p /etc/pleroma &&\ - chown -R pleroma /etc/pleroma - -USER pleroma - -COPY --from=build --chown=pleroma:0 /release ${HOME} - -COPY --chown=pleroma --chmod=640 ./config/docker.exs /etc/pleroma/config.exs -COPY ./docker-entrypoint-e2e.sh ${HOME} - -EXPOSE 4000 - -ENTRYPOINT ["/opt/pleroma/docker-entrypoint-e2e.sh"] diff --git a/changelog.d/docker-e2e-image.skip b/changelog.d/docker-e2e-image.skip deleted file mode 100644 index e69de29bb..000000000 diff --git a/changelog.d/featured-collection-page.fix b/changelog.d/featured-collection-page.fix deleted file mode 100644 index b611a4137..000000000 --- a/changelog.d/featured-collection-page.fix +++ /dev/null @@ -1 +0,0 @@ -Support fetching featured collection items from the first collection page diff --git a/changelog.d/mastodon-websocket-protocol.fix b/changelog.d/mastodon-websocket-protocol.fix deleted file mode 100644 index 66dab16ed..000000000 --- a/changelog.d/mastodon-websocket-protocol.fix +++ /dev/null @@ -1 +0,0 @@ -Echo Mastodon-style `Sec-WebSocket-Protocol` tokens in streaming WebSocket handshakes. diff --git a/changelog.d/weblate-ci.skip b/changelog.d/weblate-ci.skip deleted file mode 100644 index e69de29bb..000000000 diff --git a/changelog.d/wss-necroposts.fix b/changelog.d/wss-necroposts.fix deleted file mode 100644 index 743f9087c..000000000 --- a/changelog.d/wss-necroposts.fix +++ /dev/null @@ -1 +0,0 @@ -RichMedia: Fix backfill causing old posts to show up on timelines by disabling it in MastoAPI StatusView diff --git a/docker-entrypoint-e2e.sh b/docker-entrypoint-e2e.sh deleted file mode 100755 index d7aa332f7..000000000 --- a/docker-entrypoint-e2e.sh +++ /dev/null @@ -1,70 +0,0 @@ -#!/bin/ash -set -eu - -SEED_SENTINEL_PATH=/var/lib/pleroma/.e2e_seeded -CONFIG_OVERRIDE_PATH=/var/lib/pleroma/config.exs - -echo '-- Waiting for database...' -while ! pg_isready -U "${DB_USER:-pleroma}" -d "postgres://${DB_HOST:-db}:${DB_PORT:-5432}/${DB_NAME:-pleroma}" -t 1; do - sleep 1s -done - -echo '-- Writing E2E config overrides...' -cat > "$CONFIG_OVERRIDE_PATH" </dev/null; then - kill -TERM "$PLEROMA_PID" - wait "$PLEROMA_PID" || true - fi -} - -trap cleanup INT TERM - -echo '-- Waiting for API...' -api_ok=false -for _i in $(seq 1 120); do - if wget -qO- http://127.0.0.1:4000/api/v1/instance >/dev/null 2>&1; then - api_ok=true - break - fi - sleep 1s -done - -if [ "$api_ok" != true ]; then - echo 'Timed out waiting for Pleroma API to become available' - exit 1 -fi - -wait "$PLEROMA_PID" diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 1e8bfe3dd..0b513ee16 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -1881,15 +1881,11 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do end end - @featured_collection_types ["OrderedCollection", "Collection"] - @featured_collection_page_types ["OrderedCollectionPage", "CollectionPage"] - @featured_collection_item_types @featured_collection_types ++ @featured_collection_page_types - def pin_data_from_featured_collection(%{ "type" => type, "orderedItems" => objects }) - when type in @featured_collection_item_types do + when type in ["OrderedCollection", "Collection"] do Map.new(objects, fn %{"id" => object_ap_id} -> {object_ap_id, NaiveDateTime.utc_now()} object_ap_id when is_binary(object_ap_id) -> {object_ap_id, NaiveDateTime.utc_now()} @@ -1907,7 +1903,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do def fetch_and_prepare_featured_from_ap_id(ap_id) do with {:ok, data} <- Fetcher.fetch_and_contain_remote_object_from_id(ap_id) do - {:ok, prepare_featured_collection(data)} + {:ok, pin_data_from_featured_collection(data)} else e -> Logger.error("Could not decode featured collection at fetch #{ap_id}, #{inspect(e)}") @@ -1915,34 +1911,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do end end - defp prepare_featured_collection(%{"orderedItems" => objects} = data) when is_list(objects) do - pin_data_from_featured_collection(data) - end - - defp prepare_featured_collection(%{ - "type" => type, - "first" => %{"type" => page_type} = first - }) - when type in @featured_collection_types and page_type in @featured_collection_page_types do - pin_data_from_featured_collection(first) - end - - defp prepare_featured_collection(%{"type" => type, "first" => first}) - when type in @featured_collection_types and is_binary(first) do - case Fetcher.fetch_and_contain_remote_object_from_id(first) do - {:ok, data} -> - pin_data_from_featured_collection(data) - - e -> - Logger.error("Could not decode featured collection page at fetch #{first}, #{inspect(e)}") - %{} - end - end - - defp prepare_featured_collection(data) do - pin_data_from_featured_collection(data) - end - def enqueue_pin_fetches(%{pinned_objects: pins}) do # enqueue a task to fetch all pinned objects Enum.each(pins, fn {ap_id, _} -> diff --git a/lib/pleroma/web/endpoint.ex b/lib/pleroma/web/endpoint.ex index 07b33f866..36e78e1e2 100644 --- a/lib/pleroma/web/endpoint.ex +++ b/lib/pleroma/web/endpoint.ex @@ -9,8 +9,8 @@ defmodule Pleroma.Web.Endpoint do alias Pleroma.Config - plug(Pleroma.Web.MastodonAPI.WebsocketPlug, - path: "/api/v1/streaming", + socket("/api/v1/streaming", Pleroma.Web.MastodonAPI.WebsocketHandler, + longpoll: false, websocket: [ path: "/", compress: false, @@ -169,7 +169,8 @@ defmodule Pleroma.Web.Endpoint do else: "pleroma_key" extra = - Enum.join(Config.get([__MODULE__, :extra_cookie_attrs]), ";") + Config.get([__MODULE__, :extra_cookie_attrs]) + |> Enum.join(";") # The session will be stored in the cookie and signed, # this means its contents can be read but not tampered with. diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index 6949c5f2d..09af60865 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -28,13 +28,9 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do # This is a naive way to do this, just spawning a process per activity # to fetch the preview. However it should be fine considering # pagination is restricted to 40 activities at a time - # Force disable Websockets streaming for backfill jobs, - # otherwise old posts can show up on timelines. - defp fetch_rich_media_for_activities(activities, opts) do - opts = Map.put(opts, :stream, false) - + defp fetch_rich_media_for_activities(activities) do Enum.each(activities, fn activity -> - Card.get_by_activity(activity, opts) + Card.get_by_activity(activity) end) end @@ -117,8 +113,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do activities = Enum.filter(opts.activities, & &1) # Start prefetching rich media before doing anything else - fetch_rich_media_for_activities(activities, opts) - + fetch_rich_media_for_activities(activities) replied_to_activities = get_replied_to_activities(activities) quoted_activities = get_quoted_activities(activities) @@ -366,10 +361,8 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do summary = object.data["summary"] || "" - # Force disable Websockets streaming for backfill jobs which the below call will create, - # otherwise old posts can show up on timelines. card = - case Card.get_by_activity(activity, Map.put(opts, :stream, false)) do + case Card.get_by_activity(activity) do %Card{} = result -> render("card.json", result) _ -> nil end diff --git a/lib/pleroma/web/mastodon_api/websocket_handler.ex b/lib/pleroma/web/mastodon_api/websocket_handler.ex index 3dc862a5a..2b698bd5d 100644 --- a/lib/pleroma/web/mastodon_api/websocket_handler.ex +++ b/lib/pleroma/web/mastodon_api/websocket_handler.ex @@ -67,10 +67,9 @@ defmodule Pleroma.Web.MastodonAPI.WebsocketHandler do @impl Phoenix.Socket.Transport def handle_in({text, [opcode: :text]}, state) do - case Jason.decode(text) do - {:ok, %{} = event} -> - handle_client_event(event, state) - + with {:ok, %{} = event} <- Jason.decode(text) do + handle_client_event(event, state) + else _ -> Logger.error("#{__MODULE__} received non-JSON event: #{inspect(text)}") {:ok, state} @@ -86,11 +85,11 @@ defmodule Pleroma.Web.MastodonAPI.WebsocketHandler do def handle_info({:render_with_user, view, template, item, topic}, state) do user = %User{} = User.get_cached_by_ap_id(state.user.ap_id) - if Streamer.filtered_by_user?(user, item) do - {:ok, state} - else + unless Streamer.filtered_by_user?(user, item) do message = view.render(template, item, user, topic) {:push, {:text, message}, %{state | user: user}} + else + {:ok, state} end end @@ -254,7 +253,7 @@ defmodule Pleroma.Web.MastodonAPI.WebsocketHandler do defp find_sec_websocket_protocol(sec_headers) do Enum.find_value(sec_headers, fn - {"sec-websocket-protocol", protocols} -> protocols |> Plug.Conn.Utils.list() |> List.first() + {"sec-websocket-protocol", token} -> token _ -> nil end) end diff --git a/lib/pleroma/web/mastodon_api/websocket_plug.ex b/lib/pleroma/web/mastodon_api/websocket_plug.ex deleted file mode 100644 index c5fb5b748..000000000 --- a/lib/pleroma/web/mastodon_api/websocket_plug.ex +++ /dev/null @@ -1,104 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2022 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Web.MastodonAPI.WebsocketPlug do - @moduledoc """ - A Phoenix 1.8 compatible WebSocket transport for Mastodon streaming. - - It mirrors Phoenix.Transports.WebSocket, but echoes a successfully authenticated - Mastodon-style Sec-WebSocket-Protocol token so browser clients accept the handshake. - """ - - @behaviour Plug - - import Plug.Conn - - alias Phoenix.Socket.Transport - alias Pleroma.Web.Endpoint - alias Pleroma.Web.MastodonAPI.WebsocketHandler - - @connect_info_opts [:check_csrf] - - @impl Plug - def init(opts) do - path = String.split(Keyword.fetch!(opts, :path), "/", trim: true) - websocket = Keyword.fetch!(opts, :websocket) - config = Transport.load_config(websocket, Phoenix.Transports.WebSocket) - - {path, config} - end - - @impl Plug - def call(%{method: "GET", path_info: path} = conn, {path, opts}) do - conn - |> fetch_query_params() - |> Transport.code_reload(Endpoint, opts) - |> Transport.transport_log(opts[:transport_log]) - |> Transport.check_origin(WebsocketHandler, Endpoint, opts) - |> connect(opts) - end - - def call(%{path_info: path} = conn, {path, _opts}) do - conn - |> send_resp(400, "") - |> halt() - end - - def call(conn, _opts), do: conn - - defp connect(%{halted: true} = conn, _opts), do: conn - - defp connect(%{params: params} = conn, opts) do - keys = Keyword.get(opts, :connect_info, []) - - connect_info = - Transport.connect_info(conn, Endpoint, keys, Keyword.take(opts, @connect_info_opts)) - - config = %{ - endpoint: Endpoint, - transport: :websocket, - options: opts, - params: params, - connect_info: connect_info - } - - case WebsocketHandler.connect(config) do - {:ok, arg} -> - try do - conn - |> echo_sec_websocket_protocol() - |> WebSockAdapter.upgrade(WebsocketHandler, arg, opts) - |> halt() - rescue - e in WebSockAdapter.UpgradeError -> - conn - |> send_resp(400, e.message) - |> halt() - end - - :error -> - conn - |> send_resp(403, "") - |> halt() - - {:error, reason} -> - {m, f, args} = opts[:error_handler] - - halt(apply(m, f, [conn, reason | args])) - end - end - - defp echo_sec_websocket_protocol(conn) do - case get_req_header(conn, "sec-websocket-protocol") do - [protocols | _] -> - case Plug.Conn.Utils.list(protocols) do - [protocol | _] -> put_resp_header(conn, "sec-websocket-protocol", protocol) - [] -> conn - end - - [] -> - conn - end - end -end diff --git a/lib/pleroma/web/rich_media/backfill.ex b/lib/pleroma/web/rich_media/backfill.ex index a100b39ab..1cd90629f 100644 --- a/lib/pleroma/web/rich_media/backfill.ex +++ b/lib/pleroma/web/rich_media/backfill.ex @@ -11,8 +11,6 @@ defmodule Pleroma.Web.RichMedia.Backfill do require Logger - @callback run(map()) :: :ok | Parser.parse_errors() | Helpers.get_errors() - @cachex Pleroma.Config.get([:cachex, :provider], Cachex) @stream_out_impl Pleroma.Config.get( [__MODULE__, :stream_out], @@ -28,7 +26,11 @@ defmodule Pleroma.Web.RichMedia.Backfill do {:ok, card} = Card.create(url, fields) maybe_schedule_expiration(url, fields) - maybe_update_stream(args) + + with %{"activity_id" => activity_id} <- args, + false <- is_nil(activity_id) do + stream_update(args) + end warm_cache(url_hash, card) :ok @@ -53,17 +55,12 @@ defmodule Pleroma.Web.RichMedia.Backfill do end end - defp maybe_update_stream(%{"activity_id" => activity_id, "stream" => true}) - when is_binary(activity_id) do + defp stream_update(%{"activity_id" => activity_id}) do Pleroma.Activity.get_by_id(activity_id) |> Pleroma.Activity.normalize() |> @stream_out_impl.stream_out() end - # Streamer.stream_out returns noop when unsupported activity type is requested to be streamed. - # Do the same here for unwanted streaming - defp maybe_update_stream(_), do: :noop - defp warm_cache(key, val), do: @cachex.put(:rich_media_cache, key, val) defp negative_cache(key, ttl \\ :timer.minutes(15)), diff --git a/lib/pleroma/web/rich_media/card.ex b/lib/pleroma/web/rich_media/card.ex index 1e9e66ec1..6b4bb9555 100644 --- a/lib/pleroma/web/rich_media/card.ex +++ b/lib/pleroma/web/rich_media/card.ex @@ -91,18 +91,7 @@ defmodule Pleroma.Web.RichMedia.Card do nil -> activity_id = Keyword.get(opts, :activity_id, nil) - # Nested opts, first layer comes from get_by_activity/2 as Keyword, - # second from API views/Federation as Map. - # Provide default Map when called directly. - opts = Keyword.get(opts, :opts, %{}) - stream = Map.get(opts, :stream, true) - - RichMediaWorker.new(%{ - "op" => "backfill", - "url" => url, - "activity_id" => activity_id, - "stream" => stream - }) + RichMediaWorker.new(%{"op" => "backfill", "url" => url, "activity_id" => activity_id}) |> Oban.insert() nil @@ -123,11 +112,9 @@ defmodule Pleroma.Web.RichMedia.Card do end end - @spec get_by_activity(Activity.t(), %{}) :: t() | nil | :error - def get_by_activity(activity, opts \\ %{}) - + @spec get_by_activity(Activity.t()) :: t() | nil | :error # Fake/Draft activity - def get_by_activity(%Activity{id: "pleroma:fakeid"} = activity, _opts) do + def get_by_activity(%Activity{id: "pleroma:fakeid"} = activity) do with {_, true} <- {:config, @config_impl.get([:rich_media, :enabled])}, %Object{} = object <- Object.normalize(activity, fetch: false), url when not is_nil(url) <- HTML.extract_first_external_url_from_object(object) do @@ -151,13 +138,13 @@ defmodule Pleroma.Web.RichMedia.Card do end end - def get_by_activity(activity, opts) do + def get_by_activity(activity) do with %Object{} = object <- Object.normalize(activity, fetch: false), {_, nil} <- {:cached, get_cached_url(object, activity.id)} do nil else {:cached, url} -> - get_or_backfill_by_url(url, activity_id: activity.id, opts: opts) + get_or_backfill_by_url(url, activity_id: activity.id) _ -> :error diff --git a/priv/scrubbers/default.ex b/priv/scrubbers/default.ex index 8e2d16650..8963d99e8 100644 --- a/priv/scrubbers/default.ex +++ b/priv/scrubbers/default.ex @@ -46,7 +46,6 @@ defmodule Pleroma.HTML.Scrubber.Default do Meta.allow_tag_with_these_attributes(:b, ["lang"]) Meta.allow_tag_with_these_attributes(:blockquote, ["lang"]) Meta.allow_tag_with_these_attributes(:br, ["lang"]) - Meta.allow_tag_with_these_attributes(:center, []) Meta.allow_tag_with_these_attributes(:cite, ["lang"]) Meta.allow_tag_with_these_attributes(:code, ["lang"]) Meta.allow_tag_with_these_attributes(:del, ["lang"]) @@ -106,9 +105,7 @@ defmodule Pleroma.HTML.Scrubber.Default do "mfm-sparkle", "mfm-rotate", "mfm-ruby", - "mfm-unixtime", - # Exists in Akkoma but not Misskey? - "mfm-center" + "mfm-unixtime" ]) Meta.allow_tag_with_this_attribute_values(:p, "class", ["quote-inline"]) diff --git a/test/pleroma/integration/mastodon_websocket_test.exs b/test/pleroma/integration/mastodon_websocket_test.exs index de88e5002..078bb643c 100644 --- a/test/pleroma/integration/mastodon_websocket_test.exs +++ b/test/pleroma/integration/mastodon_websocket_test.exs @@ -11,7 +11,6 @@ defmodule Pleroma.Integration.MastodonWebsocketTest do alias Pleroma.Integration.WebsocketClient alias Pleroma.Web.CommonAPI - alias Pleroma.Web.MastodonAPI.StatusView alias Pleroma.Web.OAuth @moduletag needs_streamer: true, capture_log: true @@ -32,48 +31,6 @@ defmodule Pleroma.Integration.MastodonWebsocketTest do WebsocketClient.start_link(self(), path, headers) end - defp raw_websocket_handshake(qs, headers) do - uri = URI.parse(@path <> qs) - port = uri.port || 80 - path = uri.path <> if(uri.query, do: "?" <> uri.query, else: "") - - default_headers = [ - {"host", "#{uri.host}:#{port}"}, - {"upgrade", "websocket"}, - {"connection", "Upgrade"}, - {"sec-websocket-key", Base.encode64(:crypto.strong_rand_bytes(16))}, - {"sec-websocket-version", "13"} - ] - - request = [ - "GET #{path} HTTP/1.1\r\n", - Enum.map(default_headers ++ headers, fn {name, value} -> "#{name}: #{value}\r\n" end), - "\r\n" - ] - - with {:ok, socket} <- - :gen_tcp.connect(String.to_charlist(uri.host), port, [:binary, active: false], 1_000), - :ok <- :gen_tcp.send(socket, request), - {:ok, response} <- :gen_tcp.recv(socket, 0, 1_000) do - :gen_tcp.close(socket) - {:ok, parse_http_response(response)} - end - end - - defp parse_http_response(response) do - [headers | _] = String.split(response, "\r\n\r\n", parts: 2) - [status_line | header_lines] = String.split(headers, "\r\n") - [_, status | _] = String.split(status_line, " ") - - headers = - Enum.map(header_lines, fn line -> - [name, value] = String.split(line, ":", parts: 2) - {String.downcase(name), String.trim(value)} - end) - - %{status: String.to_integer(status), headers: headers} - end - defp decode_json(json) do with {:ok, %{"event" => event, "payload" => payload_text}} <- Jason.decode(json), {:ok, payload} <- Jason.decode(payload_text) do @@ -128,7 +85,9 @@ defmodule Pleroma.Integration.MastodonWebsocketTest do assert json["payload"] assert {:ok, json} = Jason.decode(json["payload"]) - view_json = atom_key_to_string(StatusView.render("show.json", activity: activity, for: nil)) + view_json = + Pleroma.Web.MastodonAPI.StatusView.render("show.json", activity: activity, for: nil) + |> atom_key_to_string() assert json == view_json end @@ -155,7 +114,10 @@ defmodule Pleroma.Integration.MastodonWebsocketTest do assert json["payload"] assert {:ok, json} = Jason.decode(json["payload"]) - view_json = atom_key_to_string(StatusView.render("show.json", activity: activity, for: nil)) + view_json = + Pleroma.Web.MastodonAPI.StatusView.render("show.json", activity: activity, for: nil) + |> Jason.encode!() + |> Jason.decode!() assert json == view_json end @@ -317,34 +279,6 @@ defmodule Pleroma.Integration.MastodonWebsocketTest do end) end - test "echoes the Sec-WebSocket-Protocol token in the handshake", %{token: token} do - assert {:ok, %{status: 101, headers: headers}} = - raw_websocket_handshake("?stream=user", [ - {"sec-websocket-protocol", token.token} - ]) - - assert {"sec-websocket-protocol", token.token} in headers - end - - test "echoes the selected Sec-WebSocket-Protocol token", %{token: token} do - assert {:ok, %{status: 101, headers: headers}} = - raw_websocket_handshake("?stream=user", [ - {"sec-websocket-protocol", "#{token.token}, phoenix"} - ]) - - assert {"sec-websocket-protocol", token.token} in headers - end - - test "does not echo an invalid Sec-WebSocket-Protocol token", %{token: token} do - assert {:ok, %{status: 401, headers: headers}} = - raw_websocket_handshake("?stream=user", [ - {"sec-websocket-protocol", "invalid"} - ]) - - refute {"sec-websocket-protocol", token.token} in headers - refute List.keymember?(headers, "sec-websocket-protocol", 0) - end - test "prefers sec-websocket-protocol token over query access_token", %{ token: token, user: user @@ -516,12 +450,12 @@ defmodule Pleroma.Integration.MastodonWebsocketTest do assert {:ok, json} = Jason.decode(json["payload"]) view_json = - atom_key_to_string( - StatusView.render("show.json", - activity: activity, - for: reading_user - ) + Pleroma.Web.MastodonAPI.StatusView.render("show.json", + activity: activity, + for: reading_user ) + |> Jason.encode!() + |> Jason.decode!() assert json == view_json end @@ -544,12 +478,12 @@ defmodule Pleroma.Integration.MastodonWebsocketTest do activity = Pleroma.Activity.normalize(activity) view_json = - atom_key_to_string( - StatusView.render("show.json", - activity: activity, - for: reading_user - ) + Pleroma.Web.MastodonAPI.StatusView.render("show.json", + activity: activity, + for: reading_user ) + |> Jason.encode!() + |> Jason.decode!() assert {:ok, %{"event" => "status.update", "payload" => ^view_json}} = decode_json(raw_json) end diff --git a/test/pleroma/web/activity_pub/activity_pub_test.exs b/test/pleroma/web/activity_pub/activity_pub_test.exs index dd7a82a55..9aafc41a5 100644 --- a/test/pleroma/web/activity_pub/activity_pub_test.exs +++ b/test/pleroma/web/activity_pub/activity_pub_test.exs @@ -415,105 +415,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do assert %{data: %{"id" => ^object_url}} = Object.get_by_ap_id(object_url) end - test "fetches user featured collection from the first collection page" do - ap_id = "https://example.com/users/lain" - featured_url = "https://example.com/users/lain/collections/featured" - first_page_url = "#{featured_url}?page=true" - - user_data = - "test/fixtures/users_mock/user.json" - |> File.read!() - |> String.replace("{{nickname}}", "lain") - |> Jason.decode!() - |> Map.put("featured", featured_url) - |> Jason.encode!() - - object_id = Ecto.UUID.generate() - object_url = "https://example.com/objects/#{object_id}" - - featured_data = - Jason.encode!(%{ - "id" => featured_url, - "type" => "OrderedCollection", - "first" => first_page_url - }) - - first_page_data = - Jason.encode!(%{ - "id" => first_page_url, - "type" => "OrderedCollectionPage", - "partOf" => featured_url, - "orderedItems" => [object_url] - }) - - object_data = - "test/fixtures/statuses/note.json" - |> File.read!() - |> String.replace("{{object_id}}", object_id) - |> String.replace("{{nickname}}", "lain") - - Tesla.Mock.mock(fn - %{ - method: :get, - url: ^ap_id - } -> - %Tesla.Env{ - status: 200, - body: user_data, - headers: [{"content-type", "application/activity+json"}] - } - - %{ - method: :get, - url: ^featured_url - } -> - %Tesla.Env{ - status: 200, - body: featured_data, - headers: [{"content-type", "application/activity+json"}] - } - - %{ - method: :get, - url: ^first_page_url - } -> - %Tesla.Env{ - status: 200, - body: first_page_data, - headers: [{"content-type", "application/activity+json"}] - } - - %{ - method: :get, - url: ^object_url - } -> - %Tesla.Env{ - status: 200, - body: object_data, - headers: [{"content-type", "application/activity+json"}] - } - end) - - refute capture_log(fn -> - {:ok, user} = ActivityPub.make_user_from_ap_id(ap_id) - - assert_enqueued( - worker: Pleroma.Workers.RemoteFetcherWorker, - args: %{ - "op" => "fetch_remote", - "id" => object_url, - "depth" => 1 - } - ) - - Pleroma.Tests.ObanHelpers.perform_all() - - assert user.featured_address == featured_url - assert Map.has_key?(user.pinned_objects, object_url) - assert %{data: %{"id" => ^object_url}} = Object.get_by_ap_id(object_url) - end) =~ "Could not parse featured collection" - end - test "fetches user birthday information from misskey" do user_id = "https://misskey.io/@mkljczk" @@ -2917,38 +2818,4 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do "first" => "https://social.example/users/alice/collections/featured?page=true" }) end - - test "fetch_and_prepare_featured_from_ap_id handles embedded first collection pages" do - featured_url = "https://social.example/users/alice/collections/featured" - first_page_url = "#{featured_url}?page=true" - object_url = "https://social.example/objects/1" - - featured_data = - Jason.encode!(%{ - "id" => featured_url, - "type" => "OrderedCollection", - "first" => %{ - "id" => first_page_url, - "type" => "OrderedCollectionPage", - "partOf" => featured_url, - "orderedItems" => [object_url] - } - }) - - Tesla.Mock.mock(fn - %{method: :get, url: ^featured_url} -> - %Tesla.Env{ - status: 200, - body: featured_data, - headers: [{"content-type", "application/activity+json"}] - } - end) - - refute capture_log(fn -> - assert {:ok, pinned_objects} = - ActivityPub.fetch_and_prepare_featured_from_ap_id(featured_url) - - assert Map.has_key?(pinned_objects, object_url) - end) =~ "Could not parse featured collection" - end end diff --git a/test/pleroma/web/rich_media/backfill_test.exs b/test/pleroma/web/rich_media/backfill_test.exs index 071f9b48a..6d221fcf5 100644 --- a/test/pleroma/web/rich_media/backfill_test.exs +++ b/test/pleroma/web/rich_media/backfill_test.exs @@ -5,22 +5,12 @@ defmodule Pleroma.Web.RichMedia.BackfillTest do use Pleroma.DataCase - alias Pleroma.Tests.ObanHelpers - alias Pleroma.Web.CommonAPI alias Pleroma.Web.RichMedia.Backfill alias Pleroma.Web.RichMedia.Card import Mox - import Pleroma.Factory - setup do - clear_config([:rich_media, :enabled], true) - - Mox.stub_with(Pleroma.UnstubbedConfigMock, Pleroma.Test.StaticConfig) - Mox.stub_with(Pleroma.CachexMock, Pleroma.NullCache) - - :ok - end + setup_all do: clear_config([:rich_media, :enabled], true) test "sets a negative cache entry for an error" do url = "https://bad.example.com/" @@ -33,139 +23,4 @@ defmodule Pleroma.Web.RichMedia.BackfillTest do Backfill.run(%{"url" => url}) end - - test "sets a warm_cache entry" do - url = "https://good.example.com/" - url_hash = Card.url_to_hash(url) - - Tesla.Mock.mock(fn %{url: ^url} -> - {:ok, - %Tesla.Env{ - status: 200, - body: "" - }} - end) - - Pleroma.CachexMock - |> expect(:put, fn :rich_media_cache, - ^url_hash, - %Pleroma.Web.RichMedia.Card{url_hash: ^url_hash} -> - {:ok, true} - end) - - Backfill.run(%{"url" => url}) - end - - test "streams out update when stream == true" do - url = "https://example.com" - user = insert(:user) - - Tesla.Mock.mock(fn %{url: ^url} -> - {:ok, - %Tesla.Env{ - status: 200, - body: "" - }} - end) - - {:ok, activity} = CommonAPI.post(user, %{status: "#cofe #{url}"}) - - Pleroma.CachexMock - |> expect(:put, fn :rich_media_cache, _, _ -> {:ok, true} end) - - Pleroma.Web.ActivityPub.ActivityPubMock - |> expect(:stream_out, fn %Pleroma.Activity{id: id} -> - assert id == activity.id - :ok - end) - - Backfill.run(%{"url" => url, "activity_id" => activity.id, "stream" => true}) - end - - test "does not stream out update when stream == false" do - url = "https://example.com" - user = insert(:user) - - Tesla.Mock.mock(fn %{url: ^url} -> - {:ok, - %Tesla.Env{ - status: 200, - body: "" - }} - end) - - {:ok, activity} = CommonAPI.post(user, %{status: "#cofe #{url}"}) - - Pleroma.CachexMock - |> expect(:put, fn :rich_media_cache, _, _ -> {:ok, true} end) - - Pleroma.Web.ActivityPub.ActivityPubMock - |> deny(:stream_out, 1) - - Backfill.run(%{"url" => url, "activity_id" => "#{activity.data["id"]}", "stream" => false}) - end - - # NOTE: Below two MastoAPI tests cover almost the same code paths. - # index.json will always prefetch rich media, while show.json will try to get the card and - # fetch it when it isn't cached (both use Card.get_by_activity in the end). - # So if index.json doesn't fetch the rich media, show.json will when it renders the post, - # hence why index.json test will only call ActivityPub.stream_out twice, - # if streaming is re-enabled for in both. - test "does not stream out in MastoAPI StatusView index" do - url = "https://example.com" - user = insert(:user) - - Tesla.Mock.mock(fn %{url: ^url} -> - {:ok, - %Tesla.Env{ - status: 200, - body: "" - }} - end) - - # CommonAPI federation processing will stream out once as a new post - Pleroma.Web.ActivityPub.ActivityPubMock - |> expect(:stream_out, 1, fn _ -> :ok end) - - {:ok, activity} = CommonAPI.post(user, %{status: "#cofe #{url}"}) - ObanHelpers.perform_all() - - # Clear cache to force backfill below - Pleroma.Activity.HTML.invalidate_cache_for(activity.id) - Pleroma.Web.RichMedia.Card.delete(url) - - Pleroma.Web.MastodonAPI.StatusView.render("index.json", %{ - activities: [activity], - as: :activity - }) - - ObanHelpers.perform_all() - end - - test "does not stream out in MastoAPI StatusView show" do - url = "https://example.com" - user = insert(:user) - - Tesla.Mock.mock(fn %{url: ^url} -> - {:ok, - %Tesla.Env{ - status: 200, - body: "" - }} - end) - - # CommonAPI federation processing will stream out once as a new post - Pleroma.Web.ActivityPub.ActivityPubMock - |> expect(:stream_out, 1, fn _ -> :ok end) - - {:ok, activity} = CommonAPI.post(user, %{status: "#cofe #{url}"}) - ObanHelpers.perform_all() - - # Clear cache to force backfill below - Pleroma.Activity.HTML.invalidate_cache_for(activity.id) - Pleroma.Web.RichMedia.Card.delete(url) - - Pleroma.Web.MastodonAPI.StatusView.render("show.json", activity: activity) - ObanHelpers.perform_all() - end end