From f90a52398ba8825d70db9f4594904eca02e248b5 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sun, 26 Jul 2026 12:16:04 +0400 Subject: [PATCH] ci(search): gate pinned ParadeDB integration --- .woodpecker/paradedb-integration.yaml | 64 ++++++++ .../search/parade_db_integration_test.exs | 145 +++++++++++++----- 2 files changed, 171 insertions(+), 38 deletions(-) create mode 100644 .woodpecker/paradedb-integration.yaml diff --git a/.woodpecker/paradedb-integration.yaml b/.woodpecker/paradedb-integration.yaml new file mode 100644 index 000000000..b91222837 --- /dev/null +++ b/.woodpecker/paradedb-integration.yaml @@ -0,0 +1,64 @@ +when: + - event: pull_request + path: + - ".woodpecker/**" + - "lib/pleroma/search/**" + - "test/pleroma/search/**" + - "mix.exs" + - "mix.lock" + evaluate: 'CI_COMMIT_SOURCE_BRANCH != "weblate"' + - event: push + branch: ${CI_REPO_DEFAULT_BRANCH} + path: + - ".woodpecker/**" + - "lib/pleroma/search/**" + - "test/pleroma/search/**" + - "mix.exs" + - "mix.lock" + +depends_on: + - lint + +labels: + platform: linux/amd64 + +variables: + script_file_entrypoint: &script_file_entrypoint + - /bin/sh + - -c + - 'printf "%s" "$CI_SCRIPT" | base64 -d > /tmp/ci-script.sh && /bin/sh -xe /tmp/ci-script.sh' + +steps: + paradedb-integration: + image: docker.io/elixir:1.19-otp-28-alpine + entrypoint: *script_file_entrypoint + environment: + MIX_ENV: test + VIX_COMPILATION_MODE: PLATFORM_PROVIDED_LIBVIPS + DB_HOST: postgres + DB_PORT: 5432 + PARADEDB_TEST_URL: postgres://postgres:postgres@paradedb:5432/paradedb + commands: + - apk add --no-cache build-base cmake exiftool ffmpeg vips-dev file-dev git openssl postgresql-client + - mix local.hex --force + - mix local.rebar --force + - mix deps.get + - timeout 60 sh -c 'until pg_isready -h postgres -U postgres -d pleroma_test; do sleep 1; done' || { echo "PostgreSQL did not become ready" >&2; exit 1; } + - timeout 60 sh -c 'until pg_isready -h paradedb -U postgres -d paradedb; do sleep 1; done' || { echo "ParadeDB did not become ready" >&2; exit 1; } + - mix ecto.create + - mix ecto.migrate + - mix test --include integration test/pleroma/search/parade_db_integration_test.exs + +services: + postgres: + image: docker.io/postgres:13-alpine + environment: + POSTGRES_DB: pleroma_test + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + paradedb: + image: docker.io/paradedb/paradedb:0.24.3-pg15@sha256:997d62f6c2605becb7c45878af3f8c8c698f355006b675b3ce212d4eb2d14f2e + environment: + POSTGRES_DB: paradedb + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres diff --git a/test/pleroma/search/parade_db_integration_test.exs b/test/pleroma/search/parade_db_integration_test.exs index 2687260cf..4336ecfc3 100644 --- a/test/pleroma/search/parade_db_integration_test.exs +++ b/test/pleroma/search/parade_db_integration_test.exs @@ -14,7 +14,10 @@ defmodule Pleroma.Search.ParadeDBIntegrationTest do alias Pleroma.Search.ParadeDB alias Pleroma.Web.CommonAPI - @image System.get_env("PARADEDB_TEST_IMAGE", "paradedb/paradedb:latest") + @image System.get_env( + "PARADEDB_TEST_IMAGE", + "paradedb/paradedb:0.24.3-pg15@sha256:997d62f6c2605becb7c45878af3f8c8c698f355006b675b3ce212d4eb2d14f2e" + ) @db_user "postgres" @db_password "postgres" @db_name "paradedb" @@ -26,6 +29,13 @@ defmodule Pleroma.Search.ParadeDBIntegrationTest do end setup_all do + case System.get_env("PARADEDB_TEST_URL") do + nil -> setup_docker_paradedb() + url -> setup_external_paradedb(url) + end + end + + defp setup_docker_paradedb do with :ok <- ensure_docker_available() do container = "pleroma-paradedb-it-#{System.unique_integer([:positive])}" @@ -58,49 +68,67 @@ defmodule Pleroma.Search.ParadeDBIntegrationTest do |> wait_for_port!(@docker_startup_timeout_ms) wait_for_pg_ready!(container, @docker_startup_timeout_ms) - wait_for_tcp_ready!(host_port, @docker_startup_timeout_ms) url = "postgres://#{@db_user}:#{@db_password}@127.0.0.1:#{host_port}/#{@db_name}" - - prev_env = System.get_env("PARADEDB_DATABASE_URL") - System.put_env("PARADEDB_DATABASE_URL", url) - - on_exit(fn -> - if is_nil(prev_env) do - System.delete_env("PARADEDB_DATABASE_URL") - else - System.put_env("PARADEDB_DATABASE_URL", prev_env) - end - end) - - {:ok, _pid} = start_supervised({Pleroma.Search.ParadeDB.Repo, pool_size: 5}) - - previous_table = Pleroma.Config.get([Pleroma.Search.ParadeDB, :table], nil) - table = "pleroma_search_documents_it_#{System.unique_integer([:positive])}" - Pleroma.Config.put([Pleroma.Search.ParadeDB, :table], table) - - on_exit(fn -> - if is_nil(previous_table) do - Pleroma.Config.delete([Pleroma.Search.ParadeDB, :table]) - else - Pleroma.Config.put([Pleroma.Search.ParadeDB, :table], previous_table) - end - end) - - {:ok, - %{ - container: container, - container_id: container_id, - url: url, - table: table - }} + setup_repo(url, %{container: container, container_id: container_id}) end end - test "indexes and searches against real ParadeDB" do + defp setup_external_paradedb(url) do + uri = URI.parse(url) + wait_for_tcp_ready!(uri.host, uri.port || 5432, @docker_startup_timeout_ms) + setup_repo(url, %{}) + end + + defp setup_repo(url, context) do + prev_env = System.get_env("PARADEDB_DATABASE_URL") + System.put_env("PARADEDB_DATABASE_URL", url) + + on_exit(fn -> + if is_nil(prev_env) do + System.delete_env("PARADEDB_DATABASE_URL") + else + System.put_env("PARADEDB_DATABASE_URL", prev_env) + end + end) + + {:ok, _pid} = start_supervised({Pleroma.Search.ParadeDB.Repo, pool_size: 5}) + wait_for_repo_ready!(@docker_startup_timeout_ms) + + previous_table = Pleroma.Config.get([Pleroma.Search.ParadeDB, :table], nil) + suffix = Ecto.UUID.generate() |> String.replace("-", "") + table = "pleroma_search_documents_it_#{suffix}" + Pleroma.Config.put([Pleroma.Search.ParadeDB, :table], table) + + on_exit(fn -> + if is_nil(previous_table) do + Pleroma.Config.delete([Pleroma.Search.ParadeDB, :table]) + else + Pleroma.Config.put([Pleroma.Search.ParadeDB, :table], previous_table) + end + end) + + {:ok, Map.merge(context, %{url: url, table: table})} + end + + test "indexes and searches against real ParadeDB", %{table: table} do + on_exit(fn -> + _ = + Ecto.Adapters.SQL.query( + Pleroma.Search.ParadeDB.Repo, + "DROP TABLE IF EXISTS #{table}" + ) + end) + :ok = ParadeDB.drop_index() assert :ok = ParadeDB.create_index() + assert {:ok, %{rows: [["0.24.3"]]}} = + Ecto.Adapters.SQL.query( + Pleroma.Search.ParadeDB.Repo, + "SELECT extversion FROM pg_extension WHERE extname = 'pg_search'" + ) + user = insert(:user) {:ok, a1} = @@ -118,6 +146,23 @@ defmodule Pleroma.Search.ParadeDBIntegrationTest do a2 = Activity.get_by_id_with_object(a2.id) private = Activity.get_by_id_with_object(private.id) + a1 = %{ + a1 + | object: %{ + a1.object + | data: + Map.merge(a1.object.data, %{ + "summary" => "paradedb subject phrase", + "attachment" => [ + %{ + "name" => "remote-image.jpg", + "summary" => "paradedb attachment description" + } + ] + }) + } + } + assert :ok = ParadeDB.add_to_index(a1) assert :ok = ParadeDB.add_to_index(a2) assert :ok = ParadeDB.add_to_index(private) @@ -130,6 +175,9 @@ defmodule Pleroma.Search.ParadeDBIntegrationTest do refute private.id in ids assert List.first(ids) == a2.id + assert Enum.any?(ParadeDB.search(nil, "subject phrase"), &(&1.id == a1.id)) + assert Enum.any?(ParadeDB.search(nil, "attachment description"), &(&1.id == a1.id)) + previous_fuzzy_distance = Pleroma.Config.get([Pleroma.Search.ParadeDB, :fuzzy_distance], 0) Pleroma.Config.put([Pleroma.Search.ParadeDB, :fuzzy_distance], 1) @@ -226,11 +274,11 @@ defmodule Pleroma.Search.ParadeDBIntegrationTest do wait.(wait) end - defp wait_for_tcp_ready!(port, timeout_ms) do + defp wait_for_tcp_ready!(host, port, timeout_ms) do deadline = System.monotonic_time(:millisecond) + timeout_ms wait = fn wait -> - case :gen_tcp.connect(~c"127.0.0.1", port, [:binary, active: false], 500) do + case :gen_tcp.connect(String.to_charlist(host), port, [:binary, active: false], 500) do {:ok, socket} -> :gen_tcp.close(socket) :ok @@ -248,6 +296,27 @@ defmodule Pleroma.Search.ParadeDBIntegrationTest do wait.(wait) end + defp wait_for_repo_ready!(timeout_ms) do + deadline = System.monotonic_time(:millisecond) + timeout_ms + + wait = fn wait -> + case Ecto.Adapters.SQL.query(Pleroma.Search.ParadeDB.Repo, "SELECT 1") do + {:ok, _} -> + :ok + + _ -> + if System.monotonic_time(:millisecond) > deadline do + raise "Timed out waiting for ParadeDB Repo" + else + :timer.sleep(250) + wait.(wait) + end + end + end + + wait.(wait) + end + defp docker(args) do System.cmd("docker", args, stderr_to_stdout: true) end