ci(search): gate pinned ParadeDB integration
This commit is contained in:
Lain Soykaf
parent
13c944a04b
commit
f90a52398b
2 changed files with 171 additions and 38 deletions
64
.woodpecker/paradedb-integration.yaml
Normal file
64
.woodpecker/paradedb-integration.yaml
Normal file
|
|
@ -0,0 +1,64 @@
|
|||
when:
|
||||
- event: pull_request
|
||||
path:
|
||||
- ".woodpecker/**"
|
||||
- "lib/pleroma/search/**"
|
||||
- "test/pleroma/search/**"
|
||||
- "mix.exs"
|
||||
- "mix.lock"
|
||||
evaluate: 'CI_COMMIT_SOURCE_BRANCH != "weblate"'
|
||||
- event: push
|
||||
branch: ${CI_REPO_DEFAULT_BRANCH}
|
||||
path:
|
||||
- ".woodpecker/**"
|
||||
- "lib/pleroma/search/**"
|
||||
- "test/pleroma/search/**"
|
||||
- "mix.exs"
|
||||
- "mix.lock"
|
||||
|
||||
depends_on:
|
||||
- lint
|
||||
|
||||
labels:
|
||||
platform: linux/amd64
|
||||
|
||||
variables:
|
||||
script_file_entrypoint: &script_file_entrypoint
|
||||
- /bin/sh
|
||||
- -c
|
||||
- 'printf "%s" "$CI_SCRIPT" | base64 -d > /tmp/ci-script.sh && /bin/sh -xe /tmp/ci-script.sh'
|
||||
|
||||
steps:
|
||||
paradedb-integration:
|
||||
image: docker.io/elixir:1.19-otp-28-alpine
|
||||
entrypoint: *script_file_entrypoint
|
||||
environment:
|
||||
MIX_ENV: test
|
||||
VIX_COMPILATION_MODE: PLATFORM_PROVIDED_LIBVIPS
|
||||
DB_HOST: postgres
|
||||
DB_PORT: 5432
|
||||
PARADEDB_TEST_URL: postgres://postgres:postgres@paradedb:5432/paradedb
|
||||
commands:
|
||||
- apk add --no-cache build-base cmake exiftool ffmpeg vips-dev file-dev git openssl postgresql-client
|
||||
- mix local.hex --force
|
||||
- mix local.rebar --force
|
||||
- mix deps.get
|
||||
- timeout 60 sh -c 'until pg_isready -h postgres -U postgres -d pleroma_test; do sleep 1; done' || { echo "PostgreSQL did not become ready" >&2; exit 1; }
|
||||
- timeout 60 sh -c 'until pg_isready -h paradedb -U postgres -d paradedb; do sleep 1; done' || { echo "ParadeDB did not become ready" >&2; exit 1; }
|
||||
- mix ecto.create
|
||||
- mix ecto.migrate
|
||||
- mix test --include integration test/pleroma/search/parade_db_integration_test.exs
|
||||
|
||||
services:
|
||||
postgres:
|
||||
image: docker.io/postgres:13-alpine
|
||||
environment:
|
||||
POSTGRES_DB: pleroma_test
|
||||
POSTGRES_USER: postgres
|
||||
POSTGRES_PASSWORD: postgres
|
||||
paradedb:
|
||||
image: docker.io/paradedb/paradedb:0.24.3-pg15@sha256:997d62f6c2605becb7c45878af3f8c8c698f355006b675b3ce212d4eb2d14f2e
|
||||
environment:
|
||||
POSTGRES_DB: paradedb
|
||||
POSTGRES_USER: postgres
|
||||
POSTGRES_PASSWORD: postgres
|
||||
|
|
@ -14,7 +14,10 @@ defmodule Pleroma.Search.ParadeDBIntegrationTest do
|
|||
alias Pleroma.Search.ParadeDB
|
||||
alias Pleroma.Web.CommonAPI
|
||||
|
||||
@image System.get_env("PARADEDB_TEST_IMAGE", "paradedb/paradedb:latest")
|
||||
@image System.get_env(
|
||||
"PARADEDB_TEST_IMAGE",
|
||||
"paradedb/paradedb:0.24.3-pg15@sha256:997d62f6c2605becb7c45878af3f8c8c698f355006b675b3ce212d4eb2d14f2e"
|
||||
)
|
||||
@db_user "postgres"
|
||||
@db_password "postgres"
|
||||
@db_name "paradedb"
|
||||
|
|
@ -26,6 +29,13 @@ defmodule Pleroma.Search.ParadeDBIntegrationTest do
|
|||
end
|
||||
|
||||
setup_all do
|
||||
case System.get_env("PARADEDB_TEST_URL") do
|
||||
nil -> setup_docker_paradedb()
|
||||
url -> setup_external_paradedb(url)
|
||||
end
|
||||
end
|
||||
|
||||
defp setup_docker_paradedb do
|
||||
with :ok <- ensure_docker_available() do
|
||||
container = "pleroma-paradedb-it-#{System.unique_integer([:positive])}"
|
||||
|
||||
|
|
@ -58,49 +68,67 @@ defmodule Pleroma.Search.ParadeDBIntegrationTest do
|
|||
|> wait_for_port!(@docker_startup_timeout_ms)
|
||||
|
||||
wait_for_pg_ready!(container, @docker_startup_timeout_ms)
|
||||
wait_for_tcp_ready!(host_port, @docker_startup_timeout_ms)
|
||||
|
||||
url = "postgres://#{@db_user}:#{@db_password}@127.0.0.1:#{host_port}/#{@db_name}"
|
||||
|
||||
prev_env = System.get_env("PARADEDB_DATABASE_URL")
|
||||
System.put_env("PARADEDB_DATABASE_URL", url)
|
||||
|
||||
on_exit(fn ->
|
||||
if is_nil(prev_env) do
|
||||
System.delete_env("PARADEDB_DATABASE_URL")
|
||||
else
|
||||
System.put_env("PARADEDB_DATABASE_URL", prev_env)
|
||||
end
|
||||
end)
|
||||
|
||||
{:ok, _pid} = start_supervised({Pleroma.Search.ParadeDB.Repo, pool_size: 5})
|
||||
|
||||
previous_table = Pleroma.Config.get([Pleroma.Search.ParadeDB, :table], nil)
|
||||
table = "pleroma_search_documents_it_#{System.unique_integer([:positive])}"
|
||||
Pleroma.Config.put([Pleroma.Search.ParadeDB, :table], table)
|
||||
|
||||
on_exit(fn ->
|
||||
if is_nil(previous_table) do
|
||||
Pleroma.Config.delete([Pleroma.Search.ParadeDB, :table])
|
||||
else
|
||||
Pleroma.Config.put([Pleroma.Search.ParadeDB, :table], previous_table)
|
||||
end
|
||||
end)
|
||||
|
||||
{:ok,
|
||||
%{
|
||||
container: container,
|
||||
container_id: container_id,
|
||||
url: url,
|
||||
table: table
|
||||
}}
|
||||
setup_repo(url, %{container: container, container_id: container_id})
|
||||
end
|
||||
end
|
||||
|
||||
test "indexes and searches against real ParadeDB" do
|
||||
defp setup_external_paradedb(url) do
|
||||
uri = URI.parse(url)
|
||||
wait_for_tcp_ready!(uri.host, uri.port || 5432, @docker_startup_timeout_ms)
|
||||
setup_repo(url, %{})
|
||||
end
|
||||
|
||||
defp setup_repo(url, context) do
|
||||
prev_env = System.get_env("PARADEDB_DATABASE_URL")
|
||||
System.put_env("PARADEDB_DATABASE_URL", url)
|
||||
|
||||
on_exit(fn ->
|
||||
if is_nil(prev_env) do
|
||||
System.delete_env("PARADEDB_DATABASE_URL")
|
||||
else
|
||||
System.put_env("PARADEDB_DATABASE_URL", prev_env)
|
||||
end
|
||||
end)
|
||||
|
||||
{:ok, _pid} = start_supervised({Pleroma.Search.ParadeDB.Repo, pool_size: 5})
|
||||
wait_for_repo_ready!(@docker_startup_timeout_ms)
|
||||
|
||||
previous_table = Pleroma.Config.get([Pleroma.Search.ParadeDB, :table], nil)
|
||||
suffix = Ecto.UUID.generate() |> String.replace("-", "")
|
||||
table = "pleroma_search_documents_it_#{suffix}"
|
||||
Pleroma.Config.put([Pleroma.Search.ParadeDB, :table], table)
|
||||
|
||||
on_exit(fn ->
|
||||
if is_nil(previous_table) do
|
||||
Pleroma.Config.delete([Pleroma.Search.ParadeDB, :table])
|
||||
else
|
||||
Pleroma.Config.put([Pleroma.Search.ParadeDB, :table], previous_table)
|
||||
end
|
||||
end)
|
||||
|
||||
{:ok, Map.merge(context, %{url: url, table: table})}
|
||||
end
|
||||
|
||||
test "indexes and searches against real ParadeDB", %{table: table} do
|
||||
on_exit(fn ->
|
||||
_ =
|
||||
Ecto.Adapters.SQL.query(
|
||||
Pleroma.Search.ParadeDB.Repo,
|
||||
"DROP TABLE IF EXISTS #{table}"
|
||||
)
|
||||
end)
|
||||
|
||||
:ok = ParadeDB.drop_index()
|
||||
assert :ok = ParadeDB.create_index()
|
||||
|
||||
assert {:ok, %{rows: [["0.24.3"]]}} =
|
||||
Ecto.Adapters.SQL.query(
|
||||
Pleroma.Search.ParadeDB.Repo,
|
||||
"SELECT extversion FROM pg_extension WHERE extname = 'pg_search'"
|
||||
)
|
||||
|
||||
user = insert(:user)
|
||||
|
||||
{:ok, a1} =
|
||||
|
|
@ -118,6 +146,23 @@ defmodule Pleroma.Search.ParadeDBIntegrationTest do
|
|||
a2 = Activity.get_by_id_with_object(a2.id)
|
||||
private = Activity.get_by_id_with_object(private.id)
|
||||
|
||||
a1 = %{
|
||||
a1
|
||||
| object: %{
|
||||
a1.object
|
||||
| data:
|
||||
Map.merge(a1.object.data, %{
|
||||
"summary" => "paradedb subject phrase",
|
||||
"attachment" => [
|
||||
%{
|
||||
"name" => "remote-image.jpg",
|
||||
"summary" => "paradedb attachment description"
|
||||
}
|
||||
]
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
assert :ok = ParadeDB.add_to_index(a1)
|
||||
assert :ok = ParadeDB.add_to_index(a2)
|
||||
assert :ok = ParadeDB.add_to_index(private)
|
||||
|
|
@ -130,6 +175,9 @@ defmodule Pleroma.Search.ParadeDBIntegrationTest do
|
|||
refute private.id in ids
|
||||
assert List.first(ids) == a2.id
|
||||
|
||||
assert Enum.any?(ParadeDB.search(nil, "subject phrase"), &(&1.id == a1.id))
|
||||
assert Enum.any?(ParadeDB.search(nil, "attachment description"), &(&1.id == a1.id))
|
||||
|
||||
previous_fuzzy_distance = Pleroma.Config.get([Pleroma.Search.ParadeDB, :fuzzy_distance], 0)
|
||||
Pleroma.Config.put([Pleroma.Search.ParadeDB, :fuzzy_distance], 1)
|
||||
|
||||
|
|
@ -226,11 +274,11 @@ defmodule Pleroma.Search.ParadeDBIntegrationTest do
|
|||
wait.(wait)
|
||||
end
|
||||
|
||||
defp wait_for_tcp_ready!(port, timeout_ms) do
|
||||
defp wait_for_tcp_ready!(host, port, timeout_ms) do
|
||||
deadline = System.monotonic_time(:millisecond) + timeout_ms
|
||||
|
||||
wait = fn wait ->
|
||||
case :gen_tcp.connect(~c"127.0.0.1", port, [:binary, active: false], 500) do
|
||||
case :gen_tcp.connect(String.to_charlist(host), port, [:binary, active: false], 500) do
|
||||
{:ok, socket} ->
|
||||
:gen_tcp.close(socket)
|
||||
:ok
|
||||
|
|
@ -248,6 +296,27 @@ defmodule Pleroma.Search.ParadeDBIntegrationTest do
|
|||
wait.(wait)
|
||||
end
|
||||
|
||||
defp wait_for_repo_ready!(timeout_ms) do
|
||||
deadline = System.monotonic_time(:millisecond) + timeout_ms
|
||||
|
||||
wait = fn wait ->
|
||||
case Ecto.Adapters.SQL.query(Pleroma.Search.ParadeDB.Repo, "SELECT 1") do
|
||||
{:ok, _} ->
|
||||
:ok
|
||||
|
||||
_ ->
|
||||
if System.monotonic_time(:millisecond) > deadline do
|
||||
raise "Timed out waiting for ParadeDB Repo"
|
||||
else
|
||||
:timer.sleep(250)
|
||||
wait.(wait)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
wait.(wait)
|
||||
end
|
||||
|
||||
defp docker(args) do
|
||||
System.cmd("docker", args, stderr_to_stdout: true)
|
||||
end
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue