Merge remote-tracking branch 'origin/develop' into pr-7444-finish
This commit is contained in:
Lain Soykaf
commit
842e6bcaff
28 changed files with 1717 additions and 93 deletions
5
test/fixtures/config/static_search.config
vendored
Normal file
5
test/fixtures/config/static_search.config
vendored
Normal file
|
|
@ -0,0 +1,5 @@
|
|||
import Config
|
||||
|
||||
config :pleroma, Pleroma.Search, module: Pleroma.Search.ParadeDB
|
||||
config :pleroma, Pleroma.Search.ParadeDB, url: "postgres://localhost/paradedb"
|
||||
config :pleroma, Pleroma.Search.ParadeDB.Repo, pool_size: 2
|
||||
|
|
@ -87,6 +87,14 @@ defmodule Mix.Tasks.Pleroma.ConfigTest do
|
|||
config = ConfigDB.get_by_params(%{group: ":pleroma", key: ":first_setting"})
|
||||
assert config.value == [key: "value", key2: [Repo]]
|
||||
end
|
||||
|
||||
test "static search settings remain in the config file" do
|
||||
MixTask.migrate_to_db("test/fixtures/config/static_search.config")
|
||||
|
||||
refute ConfigDB.get_by_group_and_key(:pleroma, Pleroma.Search)
|
||||
refute ConfigDB.get_by_group_and_key(:pleroma, Pleroma.Search.ParadeDB)
|
||||
refute ConfigDB.get_by_group_and_key(:pleroma, Pleroma.Search.ParadeDB.Repo)
|
||||
end
|
||||
end
|
||||
|
||||
describe "with deletion of temp file" do
|
||||
|
|
|
|||
|
|
@ -71,6 +71,31 @@ defmodule Pleroma.Config.TransferTaskTest do
|
|||
assert assets_env[:mascots] == [a: 1, b: 2]
|
||||
end
|
||||
|
||||
test "does not transfer static search configuration" do
|
||||
search_config = Application.get_env(:pleroma, Pleroma.Search)
|
||||
parade_config = Application.get_env(:pleroma, Pleroma.Search.ParadeDB)
|
||||
|
||||
clear_config(:database_config_blacklist, [])
|
||||
|
||||
insert(:config, key: Pleroma.Search, value: [module: Pleroma.Search.ParadeDB])
|
||||
|
||||
insert(:config,
|
||||
key: Pleroma.Search.ParadeDB,
|
||||
value: [url: "postgres://localhost/paradedb"]
|
||||
)
|
||||
|
||||
insert(:config,
|
||||
key: Pleroma.Search.ParadeDB.Repo,
|
||||
value: [pool_size: 50]
|
||||
)
|
||||
|
||||
TransferTask.start_link([])
|
||||
|
||||
assert Application.get_env(:pleroma, Pleroma.Search) == search_config
|
||||
assert Application.get_env(:pleroma, Pleroma.Search.ParadeDB) == parade_config
|
||||
assert Application.get_env(:pleroma, Pleroma.Search.ParadeDB.Repo)[:pool_size] == 2
|
||||
end
|
||||
|
||||
describe "pleroma restart" do
|
||||
setup do
|
||||
on_exit(fn ->
|
||||
|
|
|
|||
|
|
@ -36,6 +36,20 @@ defmodule Pleroma.ConfigDBTest do
|
|||
assert config[:goose][:webhook_url] == "https://gander.com/"
|
||||
end
|
||||
|
||||
test "get_all_as_keyword/1 excludes static search configuration" do
|
||||
insert(:config, key: Pleroma.Search, value: [module: Pleroma.Search.ParadeDB])
|
||||
insert(:config, key: Pleroma.Search.ParadeDB, value: [url: "postgres://example/db"])
|
||||
insert(:config, key: Pleroma.Search.ParadeDB.Repo, value: [pool_size: 50])
|
||||
insert(:config, key: :instance, value: [name: "Pleroma"])
|
||||
|
||||
config = ConfigDB.get_all_as_keyword(exclude_static: true)
|
||||
|
||||
refute config[:pleroma][Pleroma.Search]
|
||||
refute config[:pleroma][Pleroma.Search.ParadeDB]
|
||||
refute config[:pleroma][Pleroma.Search.ParadeDB.Repo]
|
||||
assert config[:pleroma][:instance] == [name: "Pleroma"]
|
||||
end
|
||||
|
||||
describe "update_or_create/1" do
|
||||
test "common" do
|
||||
config = insert(:config)
|
||||
|
|
|
|||
79
test/pleroma/search/backfill_test.exs
Normal file
79
test/pleroma/search/backfill_test.exs
Normal file
|
|
@ -0,0 +1,79 @@
|
|||
# Pleroma: A lightweight social networking server
|
||||
# Copyright © 2017-2026 Pleroma Authors <https://pleroma.social/>
|
||||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
defmodule Pleroma.Search.BackfillTest do
|
||||
use Pleroma.DataCase, async: false
|
||||
use Oban.Testing, repo: Pleroma.Repo
|
||||
|
||||
import Pleroma.Factory
|
||||
|
||||
alias Pleroma.Search.Backfill
|
||||
alias Pleroma.Workers.SearchIndexingWorker
|
||||
|
||||
test "enqueues the exact requested number across partial pages" do
|
||||
activities = insert_list(5, :note_activity) |> Enum.sort_by(& &1.id, :desc)
|
||||
|
||||
assert %{enqueued: 3, exhausted: false, next_cursor: cursor} =
|
||||
Backfill.run(limit: 3, step: 2, chunk: 2)
|
||||
|
||||
assert search_indexing_ids() == Enum.map(Enum.take(activities, 3), & &1.id)
|
||||
assert cursor == Enum.at(activities, 2).id
|
||||
end
|
||||
|
||||
test "resumes after the last processed activity without duplicates" do
|
||||
activities = insert_list(5, :note_activity) |> Enum.sort_by(& &1.id, :desc)
|
||||
|
||||
assert %{enqueued: 2, next_cursor: cursor} = Backfill.run(limit: 2, step: 1)
|
||||
|
||||
Pleroma.Repo.delete_all(Oban.Job)
|
||||
|
||||
assert %{enqueued: 2, next_cursor: next_cursor} =
|
||||
Backfill.run(before: cursor, limit: 2, step: 1)
|
||||
|
||||
assert search_indexing_ids() ==
|
||||
activities
|
||||
|> Enum.slice(2, 2)
|
||||
|> Enum.map(& &1.id)
|
||||
|
||||
assert next_cursor == Enum.at(activities, 3).id
|
||||
end
|
||||
|
||||
test "only queues Create activities" do
|
||||
create = insert(:note_activity)
|
||||
insert(:like_activity, note_activity: create)
|
||||
|
||||
assert %{enqueued: 1, exhausted: true, next_cursor: cursor} = Backfill.run(step: 2)
|
||||
assert search_indexing_ids() == [create.id]
|
||||
assert cursor == create.id
|
||||
end
|
||||
|
||||
test "rejects unknown options and malformed cursors before enqueueing" do
|
||||
assert_raise ArgumentError, fn -> Backfill.run(limt: 1) end
|
||||
assert_raise ArgumentError, fn -> Backfill.run(before: "not-an-activity-id") end
|
||||
assert_raise ArgumentError, fn -> Backfill.run(on_page: :invalid) end
|
||||
|
||||
assert all_enqueued(worker: SearchIndexingWorker) == []
|
||||
end
|
||||
|
||||
test "does not duplicate or skip the initial activities when newer rows are inserted" do
|
||||
activities = insert_list(3, :note_activity) |> Enum.sort_by(& &1.id, :desc)
|
||||
|
||||
assert %{enqueued: 3, exhausted: true} =
|
||||
Backfill.run(
|
||||
limit: :infinity,
|
||||
step: 1,
|
||||
on_page: fn %{enqueued: enqueued} ->
|
||||
if enqueued == 1, do: insert(:note_activity)
|
||||
end
|
||||
)
|
||||
|
||||
assert search_indexing_ids() == Enum.map(activities, & &1.id)
|
||||
end
|
||||
|
||||
defp search_indexing_ids do
|
||||
all_enqueued(worker: SearchIndexingWorker)
|
||||
|> Enum.sort_by(& &1.id)
|
||||
|> Enum.map(& &1.args["activity"])
|
||||
end
|
||||
end
|
||||
323
test/pleroma/search/parade_db_integration_test.exs
Normal file
323
test/pleroma/search/parade_db_integration_test.exs
Normal file
|
|
@ -0,0 +1,323 @@
|
|||
# Pleroma: A lightweight social networking server
|
||||
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
|
||||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
defmodule Pleroma.Search.ParadeDBIntegrationTest do
|
||||
use Pleroma.DataCase, async: false
|
||||
|
||||
@moduletag :integration
|
||||
@moduletag timeout: 180_000
|
||||
|
||||
import Pleroma.Factory
|
||||
|
||||
alias Pleroma.Activity
|
||||
alias Pleroma.Search.ParadeDB
|
||||
alias Pleroma.Web.CommonAPI
|
||||
|
||||
@image System.get_env(
|
||||
"PARADEDB_TEST_IMAGE",
|
||||
"paradedb/paradedb:0.24.3-pg15@sha256:997d62f6c2605becb7c45878af3f8c8c698f355006b675b3ce212d4eb2d14f2e"
|
||||
)
|
||||
@db_user "postgres"
|
||||
@db_password "postgres"
|
||||
@db_name "paradedb"
|
||||
@docker_startup_timeout_ms 60_000
|
||||
|
||||
setup do
|
||||
Mox.stub_with(Pleroma.UnstubbedConfigMock, Pleroma.Config)
|
||||
:ok
|
||||
end
|
||||
|
||||
setup_all do
|
||||
case System.get_env("PARADEDB_TEST_URL") do
|
||||
nil -> setup_docker_paradedb()
|
||||
url -> setup_external_paradedb(url)
|
||||
end
|
||||
end
|
||||
|
||||
defp setup_docker_paradedb do
|
||||
with :ok <- ensure_docker_available() do
|
||||
container = "pleroma-paradedb-it-#{System.unique_integer([:positive])}"
|
||||
|
||||
on_exit(fn ->
|
||||
_ = docker(["rm", "-f", container])
|
||||
end)
|
||||
|
||||
{id, 0} =
|
||||
docker([
|
||||
"run",
|
||||
"--rm",
|
||||
"--name",
|
||||
container,
|
||||
"-e",
|
||||
"POSTGRES_USER=#{@db_user}",
|
||||
"-e",
|
||||
"POSTGRES_PASSWORD=#{@db_password}",
|
||||
"-e",
|
||||
"POSTGRES_DB=#{@db_name}",
|
||||
"-p",
|
||||
"127.0.0.1::5432",
|
||||
"-d",
|
||||
@image
|
||||
])
|
||||
|
||||
container_id = String.trim(id)
|
||||
|
||||
host_port =
|
||||
container
|
||||
|> wait_for_port!(@docker_startup_timeout_ms)
|
||||
|
||||
wait_for_pg_ready!(container, @docker_startup_timeout_ms)
|
||||
|
||||
url = "postgres://#{@db_user}:#{@db_password}@127.0.0.1:#{host_port}/#{@db_name}"
|
||||
setup_repo(url, %{container: container, container_id: container_id})
|
||||
end
|
||||
end
|
||||
|
||||
defp setup_external_paradedb(url) do
|
||||
uri = URI.parse(url)
|
||||
wait_for_tcp_ready!(uri.host, uri.port || 5432, @docker_startup_timeout_ms)
|
||||
setup_repo(url, %{})
|
||||
end
|
||||
|
||||
defp setup_repo(url, context) do
|
||||
prev_env = System.get_env("PARADEDB_DATABASE_URL")
|
||||
System.put_env("PARADEDB_DATABASE_URL", url)
|
||||
|
||||
on_exit(fn ->
|
||||
if is_nil(prev_env) do
|
||||
System.delete_env("PARADEDB_DATABASE_URL")
|
||||
else
|
||||
System.put_env("PARADEDB_DATABASE_URL", prev_env)
|
||||
end
|
||||
end)
|
||||
|
||||
{:ok, _pid} = start_supervised({Pleroma.Search.ParadeDB.Repo, pool_size: 5})
|
||||
wait_for_repo_ready!(@docker_startup_timeout_ms)
|
||||
|
||||
previous_table = Pleroma.Config.get([Pleroma.Search.ParadeDB, :table], nil)
|
||||
suffix = Ecto.UUID.generate() |> String.replace("-", "")
|
||||
table = "pleroma_search_documents_it_#{suffix}"
|
||||
Pleroma.Config.put([Pleroma.Search.ParadeDB, :table], table)
|
||||
|
||||
on_exit(fn ->
|
||||
if is_nil(previous_table) do
|
||||
Pleroma.Config.delete([Pleroma.Search.ParadeDB, :table])
|
||||
else
|
||||
Pleroma.Config.put([Pleroma.Search.ParadeDB, :table], previous_table)
|
||||
end
|
||||
end)
|
||||
|
||||
{:ok, Map.merge(context, %{url: url, table: table})}
|
||||
end
|
||||
|
||||
test "indexes and searches against real ParadeDB", %{table: table} do
|
||||
on_exit(fn ->
|
||||
_ =
|
||||
Ecto.Adapters.SQL.query(
|
||||
Pleroma.Search.ParadeDB.Repo,
|
||||
"DROP TABLE IF EXISTS #{table}"
|
||||
)
|
||||
end)
|
||||
|
||||
:ok = ParadeDB.drop_index()
|
||||
assert :ok = ParadeDB.create_index()
|
||||
|
||||
assert {:ok, %{rows: [["0.24.3"]]}} =
|
||||
Ecto.Adapters.SQL.query(
|
||||
Pleroma.Search.ParadeDB.Repo,
|
||||
"SELECT extversion FROM pg_extension WHERE extname = 'pg_search'"
|
||||
)
|
||||
|
||||
user = insert(:user)
|
||||
|
||||
{:ok, a1} =
|
||||
CommonAPI.post(user, %{status: "paradedb integration first", visibility: "public"})
|
||||
|
||||
:timer.sleep(1100)
|
||||
|
||||
{:ok, a2} =
|
||||
CommonAPI.post(user, %{status: "paradedb integration second", visibility: "public"})
|
||||
|
||||
{:ok, private} =
|
||||
CommonAPI.post(user, %{status: "paradedb integration private", visibility: "private"})
|
||||
|
||||
a1 = Activity.get_by_id_with_object(a1.id)
|
||||
a2 = Activity.get_by_id_with_object(a2.id)
|
||||
private = Activity.get_by_id_with_object(private.id)
|
||||
|
||||
a1 = %{
|
||||
a1
|
||||
| object: %{
|
||||
a1.object
|
||||
| data:
|
||||
Map.merge(a1.object.data, %{
|
||||
"summary" => "paradedb subject phrase",
|
||||
"attachment" => [
|
||||
%{
|
||||
"name" => "remote-image.jpg",
|
||||
"summary" => "paradedb attachment description"
|
||||
}
|
||||
]
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
assert :ok = ParadeDB.add_to_index(a1)
|
||||
assert :ok = ParadeDB.add_to_index(a2)
|
||||
assert :ok = ParadeDB.add_to_index(private)
|
||||
|
||||
results = ParadeDB.search(nil, "paradedb integration", limit: 40, offset: 0)
|
||||
ids = Enum.map(results, & &1.id)
|
||||
|
||||
assert a1.id in ids
|
||||
assert a2.id in ids
|
||||
refute private.id in ids
|
||||
assert List.first(ids) == a2.id
|
||||
|
||||
assert Enum.any?(ParadeDB.search(nil, "subject phrase"), &(&1.id == a1.id))
|
||||
assert Enum.any?(ParadeDB.search(nil, "attachment description"), &(&1.id == a1.id))
|
||||
|
||||
previous_fuzzy_distance = Pleroma.Config.get([Pleroma.Search.ParadeDB, :fuzzy_distance], 0)
|
||||
Pleroma.Config.put([Pleroma.Search.ParadeDB, :fuzzy_distance], 1)
|
||||
|
||||
on_exit(fn ->
|
||||
Pleroma.Config.put([Pleroma.Search.ParadeDB, :fuzzy_distance], previous_fuzzy_distance)
|
||||
end)
|
||||
|
||||
fuzzy_results = ParadeDB.search(nil, "paradedb integraion", limit: 40, offset: 0)
|
||||
fuzzy_ids = Enum.map(fuzzy_results, & &1.id)
|
||||
|
||||
assert a1.id in fuzzy_ids
|
||||
assert a2.id in fuzzy_ids
|
||||
refute private.id in fuzzy_ids
|
||||
|
||||
# Removing an object should remove it from the ParadeDB index.
|
||||
assert :ok = ParadeDB.remove_from_index(a2.object)
|
||||
|
||||
results_after_delete = ParadeDB.search(nil, "paradedb integration", limit: 40, offset: 0)
|
||||
ids_after_delete = Enum.map(results_after_delete, & &1.id)
|
||||
|
||||
assert a1.id in ids_after_delete
|
||||
refute a2.id in ids_after_delete
|
||||
end
|
||||
|
||||
defp ensure_docker_available do
|
||||
if is_nil(System.find_executable("docker")) do
|
||||
{:skip, "Docker not installed"}
|
||||
else
|
||||
case docker(["info"]) do
|
||||
{_out, 0} -> :ok
|
||||
{out, _} -> {:skip, "Docker not available: #{String.trim(out)}"}
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
defp wait_for_port!(container, timeout_ms) do
|
||||
deadline = System.monotonic_time(:millisecond) + timeout_ms
|
||||
|
||||
wait = fn wait ->
|
||||
case docker(["port", container, "5432/tcp"]) do
|
||||
{out, 0} when byte_size(out) > 0 ->
|
||||
out
|
||||
|> String.split("\n", trim: true)
|
||||
|> List.first()
|
||||
|> parse_host_port!()
|
||||
|
||||
_ ->
|
||||
if System.monotonic_time(:millisecond) > deadline do
|
||||
raise "Timed out waiting for docker port mapping"
|
||||
else
|
||||
:timer.sleep(250)
|
||||
wait.(wait)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
wait.(wait)
|
||||
end
|
||||
|
||||
defp parse_host_port!(mapping) do
|
||||
mapping
|
||||
|> String.trim()
|
||||
|> String.split(":")
|
||||
|> List.last()
|
||||
|> String.to_integer()
|
||||
end
|
||||
|
||||
defp wait_for_pg_ready!(container, timeout_ms) do
|
||||
deadline = System.monotonic_time(:millisecond) + timeout_ms
|
||||
|
||||
wait = fn wait ->
|
||||
case docker([
|
||||
"exec",
|
||||
container,
|
||||
"pg_isready",
|
||||
"-U",
|
||||
@db_user,
|
||||
"-d",
|
||||
@db_name
|
||||
]) do
|
||||
{_out, 0} ->
|
||||
:ok
|
||||
|
||||
_ ->
|
||||
if System.monotonic_time(:millisecond) > deadline do
|
||||
raise "Timed out waiting for ParadeDB Postgres to become ready"
|
||||
else
|
||||
:timer.sleep(250)
|
||||
wait.(wait)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
wait.(wait)
|
||||
end
|
||||
|
||||
defp wait_for_tcp_ready!(host, port, timeout_ms) do
|
||||
deadline = System.monotonic_time(:millisecond) + timeout_ms
|
||||
|
||||
wait = fn wait ->
|
||||
case :gen_tcp.connect(String.to_charlist(host), port, [:binary, active: false], 500) do
|
||||
{:ok, socket} ->
|
||||
:gen_tcp.close(socket)
|
||||
:ok
|
||||
|
||||
{:error, _} ->
|
||||
if System.monotonic_time(:millisecond) > deadline do
|
||||
raise "Timed out waiting for TCP port #{port} to accept connections"
|
||||
else
|
||||
:timer.sleep(250)
|
||||
wait.(wait)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
wait.(wait)
|
||||
end
|
||||
|
||||
defp wait_for_repo_ready!(timeout_ms) do
|
||||
deadline = System.monotonic_time(:millisecond) + timeout_ms
|
||||
|
||||
wait = fn wait ->
|
||||
case Ecto.Adapters.SQL.query(Pleroma.Search.ParadeDB.Repo, "SELECT 1") do
|
||||
{:ok, _} ->
|
||||
:ok
|
||||
|
||||
_ ->
|
||||
if System.monotonic_time(:millisecond) > deadline do
|
||||
raise "Timed out waiting for ParadeDB Repo"
|
||||
else
|
||||
:timer.sleep(250)
|
||||
wait.(wait)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
wait.(wait)
|
||||
end
|
||||
|
||||
defp docker(args) do
|
||||
System.cmd("docker", args, stderr_to_stdout: true)
|
||||
end
|
||||
end
|
||||
286
test/pleroma/search/parade_db_test.exs
Normal file
286
test/pleroma/search/parade_db_test.exs
Normal file
|
|
@ -0,0 +1,286 @@
|
|||
# Pleroma: A lightweight social networking server
|
||||
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
|
||||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
defmodule Pleroma.Search.ParadeDBTest do
|
||||
use Pleroma.DataCase, async: true
|
||||
use Oban.Testing, repo: Pleroma.Repo
|
||||
|
||||
import Mox
|
||||
import Pleroma.Factory
|
||||
|
||||
alias Pleroma.Search.ParadeDB
|
||||
alias Pleroma.Search.ParadeDB.ClientMock
|
||||
alias Pleroma.UnstubbedConfigMock, as: Config
|
||||
alias Pleroma.Web.CommonAPI
|
||||
alias Pleroma.Workers.SearchIndexingWorker
|
||||
|
||||
describe "ParadeDB" do
|
||||
test "indexes a public post on creation" do
|
||||
user = insert(:user)
|
||||
|
||||
{:ok, activity} =
|
||||
CommonAPI.post(user, %{
|
||||
status: "guys i just don't wanna leave the swamp",
|
||||
visibility: "public"
|
||||
})
|
||||
|
||||
args = %{"op" => "add_to_index", "activity" => activity.id}
|
||||
|
||||
assert_enqueued(worker: SearchIndexingWorker, args: args)
|
||||
|
||||
ClientMock
|
||||
|> expect(:query, fn sql, params ->
|
||||
assert sql =~ "INSERT INTO pleroma_search_documents"
|
||||
|
||||
[activity_id, object_id, object_ap_id, actor_ap_id, content, published_at] = params
|
||||
|
||||
{:ok, dumped_activity_id} = FlakeId.Ecto.CompatType.dump(activity.id)
|
||||
|
||||
assert activity_id == dumped_activity_id
|
||||
assert is_integer(object_id)
|
||||
assert is_binary(object_ap_id)
|
||||
assert actor_ap_id == activity.data["actor"]
|
||||
assert content == "guys i just don't wanna leave the swamp"
|
||||
assert %DateTime{} = published_at
|
||||
|
||||
send(self(), :inserted)
|
||||
{:ok, %{}}
|
||||
end)
|
||||
|
||||
Config
|
||||
|> expect(:get, 3, fn
|
||||
[Pleroma.Search, :module], nil ->
|
||||
ParadeDB
|
||||
|
||||
[Pleroma.Search.ParadeDB, :client_impl], nil ->
|
||||
ClientMock
|
||||
|
||||
[Pleroma.Search.ParadeDB, :table], "pleroma_search_documents" ->
|
||||
"pleroma_search_documents"
|
||||
end)
|
||||
|
||||
assert :ok = perform_job(SearchIndexingWorker, args)
|
||||
assert_received(:inserted)
|
||||
end
|
||||
|
||||
test "doesn't index posts that are not public" do
|
||||
user = insert(:user)
|
||||
|
||||
Enum.each(["private", "direct"], fn visibility ->
|
||||
{:ok, activity} =
|
||||
CommonAPI.post(user, %{
|
||||
status: "guys i just don't wanna leave the swamp",
|
||||
visibility: visibility
|
||||
})
|
||||
|
||||
args = %{"op" => "add_to_index", "activity" => activity.id}
|
||||
refute_enqueued(worker: SearchIndexingWorker, args: args)
|
||||
end)
|
||||
end
|
||||
|
||||
test "backend skips non-public posts when called directly" do
|
||||
user = insert(:user)
|
||||
|
||||
Enum.each(["private", "direct"], fn visibility ->
|
||||
{:ok, activity} =
|
||||
CommonAPI.post(user, %{
|
||||
status: "guys i just don't wanna leave the swamp",
|
||||
visibility: visibility
|
||||
})
|
||||
|
||||
assert :ok = ParadeDB.add_to_index(activity)
|
||||
end)
|
||||
end
|
||||
|
||||
test "deletes posts from index when deleted locally" do
|
||||
user = insert(:user)
|
||||
|
||||
ClientMock
|
||||
|> expect(:query, 2, fn sql, params ->
|
||||
cond do
|
||||
String.contains?(sql, "INSERT INTO pleroma_search_documents") ->
|
||||
send(self(), :inserted)
|
||||
{:ok, %{}}
|
||||
|
||||
String.contains?(sql, "DELETE FROM pleroma_search_documents") ->
|
||||
assert [object_id] = params
|
||||
assert is_integer(object_id)
|
||||
|
||||
send(self(), :deleted)
|
||||
{:ok, %{}}
|
||||
|
||||
true ->
|
||||
flunk("Unexpected ParadeDB SQL: #{inspect(sql)}")
|
||||
end
|
||||
end)
|
||||
|
||||
Config
|
||||
|> expect(:get, 6, fn
|
||||
[Pleroma.Search, :module], nil ->
|
||||
ParadeDB
|
||||
|
||||
[Pleroma.Search.ParadeDB, :client_impl], nil ->
|
||||
ClientMock
|
||||
|
||||
[Pleroma.Search.ParadeDB, :table], "pleroma_search_documents" ->
|
||||
"pleroma_search_documents"
|
||||
end)
|
||||
|
||||
{:ok, activity} =
|
||||
CommonAPI.post(user, %{
|
||||
status: "guys i just don't wanna leave the swamp",
|
||||
visibility: "public"
|
||||
})
|
||||
|
||||
args = %{"op" => "add_to_index", "activity" => activity.id}
|
||||
assert_enqueued(worker: SearchIndexingWorker, args: args)
|
||||
assert :ok = perform_job(SearchIndexingWorker, args)
|
||||
assert_received(:inserted)
|
||||
|
||||
{:ok, _} = CommonAPI.delete(activity.id, user)
|
||||
|
||||
delete_args = %{"op" => "remove_from_index", "object" => activity.object.id}
|
||||
assert_enqueued(worker: SearchIndexingWorker, args: delete_args)
|
||||
assert :ok = perform_job(SearchIndexingWorker, delete_args)
|
||||
assert_received(:deleted)
|
||||
end
|
||||
|
||||
test "search returns activities in backend order" do
|
||||
user = insert(:user)
|
||||
|
||||
{:ok, activity1} = CommonAPI.post(user, %{status: "first swamp", visibility: "public"})
|
||||
{:ok, activity2} = CommonAPI.post(user, %{status: "second swamp", visibility: "public"})
|
||||
|
||||
activity1_id = activity1.id
|
||||
activity2_id = activity2.id
|
||||
|
||||
{:ok, dumped_activity1_id} = FlakeId.Ecto.CompatType.dump(activity1_id)
|
||||
{:ok, dumped_activity2_id} = FlakeId.Ecto.CompatType.dump(activity2_id)
|
||||
|
||||
ClientMock
|
||||
|> expect(:query, fn sql, params ->
|
||||
assert sql =~ "SELECT id FROM pleroma_search_documents"
|
||||
assert ["swamp", _limit, _offset] = params
|
||||
|
||||
{:ok, %{rows: [[dumped_activity2_id], [dumped_activity1_id]]}}
|
||||
end)
|
||||
|
||||
Config
|
||||
|> expect(:get, 3, fn
|
||||
[Pleroma.Search.ParadeDB, :client_impl], nil ->
|
||||
ClientMock
|
||||
|
||||
[Pleroma.Search.ParadeDB, :table], "pleroma_search_documents" ->
|
||||
"pleroma_search_documents"
|
||||
|
||||
[Pleroma.Search.ParadeDB, :fuzzy_distance], 0 ->
|
||||
0
|
||||
end)
|
||||
|
||||
assert [%{id: ^activity2_id}, %{id: ^activity1_id}] =
|
||||
ParadeDB.search(nil, "swamp", limit: 40, offset: 0)
|
||||
end
|
||||
|
||||
test "search excludes non-public activities from backend hits" do
|
||||
user = insert(:user)
|
||||
|
||||
{:ok, public_activity} =
|
||||
CommonAPI.post(user, %{status: "public swamp", visibility: "public"})
|
||||
|
||||
{:ok, private_activity} =
|
||||
CommonAPI.post(user, %{status: "private swamp", visibility: "private"})
|
||||
|
||||
public_activity_id = public_activity.id
|
||||
|
||||
{:ok, dumped_public_id} = FlakeId.Ecto.CompatType.dump(public_activity.id)
|
||||
{:ok, dumped_private_id} = FlakeId.Ecto.CompatType.dump(private_activity.id)
|
||||
|
||||
ClientMock
|
||||
|> expect(:query, fn sql, params ->
|
||||
assert sql =~ "SELECT id FROM pleroma_search_documents"
|
||||
assert ["swamp", _limit, _offset] = params
|
||||
|
||||
{:ok, %{rows: [[dumped_private_id], [dumped_public_id]]}}
|
||||
end)
|
||||
|
||||
Config
|
||||
|> expect(:get, 3, fn
|
||||
[Pleroma.Search.ParadeDB, :client_impl], nil ->
|
||||
ClientMock
|
||||
|
||||
[Pleroma.Search.ParadeDB, :table], "pleroma_search_documents" ->
|
||||
"pleroma_search_documents"
|
||||
|
||||
[Pleroma.Search.ParadeDB, :fuzzy_distance], 0 ->
|
||||
0
|
||||
end)
|
||||
|
||||
assert [%{id: ^public_activity_id}] = ParadeDB.search(nil, "swamp", limit: 40, offset: 0)
|
||||
end
|
||||
|
||||
test "search supports backend ids returned as uuid strings" do
|
||||
user = insert(:user)
|
||||
|
||||
{:ok, activity} = CommonAPI.post(user, %{status: "uuid swamp", visibility: "public"})
|
||||
|
||||
activity_id = activity.id
|
||||
{:ok, dumped_id} = FlakeId.Ecto.CompatType.dump(activity_id)
|
||||
{:ok, uuid_string} = Ecto.UUID.load(dumped_id)
|
||||
|
||||
ClientMock
|
||||
|> expect(:query, fn sql, params ->
|
||||
assert sql =~ "SELECT id FROM pleroma_search_documents"
|
||||
assert ["swamp", _limit, _offset] = params
|
||||
|
||||
{:ok, %{rows: [[uuid_string]]}}
|
||||
end)
|
||||
|
||||
Config
|
||||
|> expect(:get, 3, fn
|
||||
[Pleroma.Search.ParadeDB, :client_impl], nil ->
|
||||
ClientMock
|
||||
|
||||
[Pleroma.Search.ParadeDB, :table], "pleroma_search_documents" ->
|
||||
"pleroma_search_documents"
|
||||
|
||||
[Pleroma.Search.ParadeDB, :fuzzy_distance], 0 ->
|
||||
0
|
||||
end)
|
||||
|
||||
assert [%{id: ^activity_id}] = ParadeDB.search(nil, "swamp", limit: 40, offset: 0)
|
||||
end
|
||||
|
||||
test "search uses configured fuzzy distance" do
|
||||
user = insert(:user)
|
||||
|
||||
{:ok, activity} = CommonAPI.post(user, %{status: "runing shoes", visibility: "public"})
|
||||
|
||||
activity_id = activity.id
|
||||
{:ok, dumped_id} = FlakeId.Ecto.CompatType.dump(activity_id)
|
||||
|
||||
ClientMock
|
||||
|> expect(:query, fn sql, params ->
|
||||
assert sql =~ "SELECT id FROM pleroma_search_documents"
|
||||
assert sql =~ "($1::text)::pdb.fuzzy(1)"
|
||||
assert ["running shoes", _limit, _offset] = params
|
||||
|
||||
{:ok, %{rows: [[dumped_id]]}}
|
||||
end)
|
||||
|
||||
Config
|
||||
|> expect(:get, 3, fn
|
||||
[Pleroma.Search.ParadeDB, :client_impl], nil ->
|
||||
ClientMock
|
||||
|
||||
[Pleroma.Search.ParadeDB, :table], "pleroma_search_documents" ->
|
||||
"pleroma_search_documents"
|
||||
|
||||
[Pleroma.Search.ParadeDB, :fuzzy_distance], 0 ->
|
||||
1
|
||||
end)
|
||||
|
||||
assert [%{id: ^activity_id}] = ParadeDB.search(nil, "running shoes", limit: 40, offset: 0)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
@ -3,26 +3,32 @@
|
|||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
defmodule Pleroma.SearchTest do
|
||||
require Pleroma.Constants
|
||||
|
||||
use Pleroma.DataCase, async: true
|
||||
use Oban.Testing, repo: Pleroma.Repo
|
||||
|
||||
import Pleroma.Factory
|
||||
|
||||
alias Pleroma.Object
|
||||
alias Pleroma.Search
|
||||
alias Pleroma.Web.CommonAPI
|
||||
alias Pleroma.Workers.SearchIndexingWorker
|
||||
|
||||
test "indexes posts that are public" do
|
||||
test "indexes posts that are public or unlisted" do
|
||||
user = insert(:user)
|
||||
|
||||
{:ok, activity} =
|
||||
CommonAPI.post(user, %{
|
||||
status: "Well this is a story all about how my life got flipped turned upside down",
|
||||
visibility: "public"
|
||||
})
|
||||
Enum.each(["public", "unlisted"], fn visibility ->
|
||||
{:ok, activity} =
|
||||
CommonAPI.post(user, %{
|
||||
status: "Well this is a story all about how my life got flipped turned upside down",
|
||||
visibility: visibility
|
||||
})
|
||||
|
||||
args = %{"op" => "add_to_index", "activity" => activity.id}
|
||||
args = %{"op" => "add_to_index", "activity" => activity.id}
|
||||
|
||||
assert_enqueued(worker: SearchIndexingWorker, args: args)
|
||||
assert_enqueued(worker: SearchIndexingWorker, args: args)
|
||||
end)
|
||||
end
|
||||
|
||||
test "doesn't index posts that are not public" do
|
||||
|
|
@ -66,4 +72,83 @@ defmodule Pleroma.SearchTest do
|
|||
|
||||
refute_enqueued(worker: SearchIndexingWorker, args: args)
|
||||
end
|
||||
|
||||
test "object_to_search_data includes content warnings and attachment descriptions" do
|
||||
object = %Object{
|
||||
id: 1,
|
||||
data: %{
|
||||
"id" => "https://example.com/objects/1",
|
||||
"type" => "Note",
|
||||
"content" => "body text",
|
||||
"summary" => "subject text",
|
||||
"attachment" => [
|
||||
%{"type" => "Document", "name" => "image description"},
|
||||
%{
|
||||
"type" => "Document",
|
||||
"name" => "remote-image.jpg",
|
||||
"summary" => "remote image description"
|
||||
},
|
||||
%{"type" => "Document", "name" => ""}
|
||||
],
|
||||
"published" => DateTime.utc_now() |> DateTime.to_iso8601()
|
||||
}
|
||||
}
|
||||
|
||||
assert %{content: content} = Search.object_to_search_data(object)
|
||||
assert content =~ "body text"
|
||||
assert content =~ "subject text"
|
||||
assert content =~ "image description"
|
||||
assert content =~ "remote image description"
|
||||
refute content =~ "remote-image.jpg"
|
||||
end
|
||||
|
||||
test "object_to_search_data ignores objects with invalid published dates" do
|
||||
for published <- [nil, 123, "not a date"] do
|
||||
object = %Object{
|
||||
id: 1,
|
||||
data: %{
|
||||
"id" => "https://example.com/objects/1",
|
||||
"type" => "Note",
|
||||
"content" => "body text",
|
||||
"published" => published
|
||||
}
|
||||
}
|
||||
|
||||
refute Search.object_to_search_data(object)
|
||||
end
|
||||
end
|
||||
|
||||
test "indexable? accepts searchable content warnings and attachment descriptions" do
|
||||
public = Pleroma.Constants.as_public()
|
||||
published = DateTime.utc_now() |> DateTime.to_iso8601()
|
||||
|
||||
for data <- [
|
||||
%{"content" => ".", "summary" => "subject text", "attachment" => []},
|
||||
%{
|
||||
"content" => ".",
|
||||
"summary" => "",
|
||||
"attachment" => [%{"type" => "Document", "name" => "image description"}]
|
||||
}
|
||||
] do
|
||||
object = %Object{
|
||||
id: 1,
|
||||
data:
|
||||
Map.merge(data, %{
|
||||
"id" => "https://example.com/objects/1",
|
||||
"type" => "Note",
|
||||
"to" => [public],
|
||||
"published" => published
|
||||
})
|
||||
}
|
||||
|
||||
activity = %Pleroma.Activity{
|
||||
id: Ecto.UUID.generate(),
|
||||
data: %{"type" => "Create"},
|
||||
object: object,
|
||||
recipients: [public]
|
||||
}
|
||||
|
||||
assert Search.indexable?(activity)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
|
|||
|
|
@ -121,6 +121,19 @@ defmodule Pleroma.Web.AdminAPI.ConfigControllerTest do
|
|||
end)
|
||||
end
|
||||
|
||||
test "effective configuration omits static search settings", %{conn: conn} do
|
||||
insert(:config, key: Pleroma.Search, value: [module: Pleroma.Search.ParadeDB])
|
||||
insert(:config, key: Pleroma.Search.ParadeDB, value: [url: "postgres://example/db"])
|
||||
|
||||
%{"configs" => configs} =
|
||||
conn
|
||||
|> get("/api/pleroma/admin/config")
|
||||
|> json_response_and_validate_schema(200)
|
||||
|
||||
refute Enum.any?(configs, &(&1["key"] == "Pleroma.Search"))
|
||||
refute Enum.any?(configs, &(&1["key"] == "Pleroma.Search.ParadeDB"))
|
||||
end
|
||||
|
||||
test "subkeys with full update right merge", %{conn: conn} do
|
||||
insert(:config,
|
||||
key: ":emoji",
|
||||
|
|
@ -1552,12 +1565,15 @@ defmodule Pleroma.Web.AdminAPI.ConfigControllerTest do
|
|||
assert web_endpoint["children"]
|
||||
end
|
||||
|
||||
test "all keys from description are whitelisted", %{conn: conn} do
|
||||
test "descriptions exclude blacklisted keys", %{conn: conn} do
|
||||
conn = get(conn, "/api/pleroma/admin/config/descriptions")
|
||||
|
||||
assert response = json_response_and_validate_schema(conn, 200)
|
||||
|
||||
assert length(response) == length(Pleroma.Docs.JSON.compiled_descriptions())
|
||||
refute Enum.any?(response, &(&1["key"] == "Pleroma.Search"))
|
||||
refute Enum.any?(response, &(&1["key"] == "Pleroma.Search.ParadeDB"))
|
||||
|
||||
assert length(response) == length(Pleroma.Docs.JSON.compiled_descriptions()) - 2
|
||||
end
|
||||
|
||||
test "filters by database configuration blacklist and whitelist", %{conn: conn} do
|
||||
|
|
|
|||
|
|
@ -46,3 +46,5 @@ Mox.defmock(Pleroma.SignatureMock, for: Pleroma.Signature.API)
|
|||
Mox.defmock(Pleroma.Web.ActivityPub.TransmogrifierMock,
|
||||
for: Pleroma.Web.ActivityPub.Transmogrifier.API
|
||||
)
|
||||
|
||||
Mox.defmock(Pleroma.Search.ParadeDB.ClientMock, for: Pleroma.Search.ParadeDB.Client)
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@
|
|||
|
||||
ExUnit.configure(capture_log: true, max_cases: System.schedulers_online())
|
||||
|
||||
ExUnit.start(exclude: [:federated])
|
||||
ExUnit.start(exclude: [:federated, :integration])
|
||||
|
||||
if match?({:unix, :darwin}, :os.type()) do
|
||||
excluded = ExUnit.configuration() |> Keyword.get(:exclude, [])
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue