diff --git a/config/config.exs b/config/config.exs index 7e374b22b..01dd09a18 100644 --- a/config/config.exs +++ b/config/config.exs @@ -940,7 +940,8 @@ config :pleroma, Pleroma.Search.Meilisearch, config :pleroma, Pleroma.Search.ParadeDB, url: nil, - table: "pleroma_search_documents" + table: "pleroma_search_documents", + fuzzy_distance: 0 config :pleroma, Pleroma.Search.ParadeDB.Repo, pool_size: 10, diff --git a/config/description.exs b/config/description.exs index de454dfb2..9a00afdcd 100644 --- a/config/description.exs +++ b/config/description.exs @@ -3560,6 +3560,12 @@ config :pleroma, :config_description, [ type: :string, description: "Table name used for search documents.", suggestion: ["pleroma_search_documents"] + }, + %{ + key: :fuzzy_distance, + type: :integer, + description: "Edit-distance used for fuzzy token matching (0 disables fuzziness, max 2).", + suggestion: [0, 1, 2] } ] }, diff --git a/docs/configuration/search.md b/docs/configuration/search.md index 780904d10..c0d3f8462 100644 --- a/docs/configuration/search.md +++ b/docs/configuration/search.md @@ -160,7 +160,8 @@ To use it, set the search module to `Pleroma.Search.ParadeDB` and configure the > > config :pleroma, Pleroma.Search.ParadeDB, > url: System.get_env("PARADEDB_DATABASE_URL") || "postgres://postgres:postgres@127.0.0.1:5432/paradedb", -> table: "pleroma_search_documents" +> table: "pleroma_search_documents", +> fuzzy_distance: 0 Then, create the table and BM25 index once: @@ -178,4 +179,6 @@ After that, run an initial indexing pass to backfill existing posts: Note: Like the other external search backends, only public/unlisted Notes are indexed. +Set `fuzzy_distance` to `1` or `2` to allow typo-tolerant matching (`2` is the maximum ParadeDB supports). + Indexing runs via the Oban `search_indexing` queue, so ensure that queue is enabled (not paused) in your Oban configuration. diff --git a/lib/pleroma/search/paradedb.ex b/lib/pleroma/search/paradedb.ex index 01cbc323f..57e5599b5 100644 --- a/lib/pleroma/search/paradedb.ex +++ b/lib/pleroma/search/paradedb.ex @@ -35,6 +35,22 @@ defmodule Pleroma.Search.ParadeDB do Config.get([Pleroma.Search.ParadeDB, :table], @default_table) end + defp fuzzy_distance do + case Config.get([Pleroma.Search.ParadeDB, :fuzzy_distance], 0) do + value when is_integer(value) and value in 0..2 -> + value + + value when is_binary(value) -> + case Integer.parse(value) do + {parsed, ""} when parsed in 0..2 -> parsed + _ -> 0 + end + + _ -> + 0 + end + end + @impl true def create_index do with {:ok, _} <- client_impl().query("CREATE EXTENSION IF NOT EXISTS pg_search", []), @@ -190,7 +206,9 @@ defmodule Pleroma.Search.ParadeDB do end defp search_ids(query, limit, offset, author) do - base_sql = "SELECT id FROM #{table()} WHERE content &&& $1" + base_sql = + "SELECT id FROM #{table()} WHERE content &&& #{search_query_param_sql(fuzzy_distance())}" + params = [query] {base_sql, params} = @@ -216,6 +234,9 @@ defmodule Pleroma.Search.ParadeDB do end end + defp search_query_param_sql(0), do: "$1" + defp search_query_param_sql(distance), do: "$1::pdb.fuzzy(#{distance})" + defp create_table_sql(table) do """ CREATE TABLE IF NOT EXISTS #{table} ( diff --git a/test/pleroma/search/paradedb_integration_test.exs b/test/pleroma/search/paradedb_integration_test.exs index 42d12c4d6..743dcbaff 100644 --- a/test/pleroma/search/paradedb_integration_test.exs +++ b/test/pleroma/search/paradedb_integration_test.exs @@ -73,7 +73,7 @@ defmodule Pleroma.Search.ParadeDBIntegrationTest do end end) - {:ok, _pid} = start_supervised({Pleroma.Search.ParadeDB.Repo, pool_size: 1}) + {:ok, _pid} = start_supervised({Pleroma.Search.ParadeDB.Repo, pool_size: 5}) previous_table = Pleroma.Config.get([Pleroma.Search.ParadeDB, :table], nil) table = "pleroma_search_documents_it_#{System.unique_integer([:positive])}" diff --git a/test/pleroma/search/paradedb_test.exs b/test/pleroma/search/paradedb_test.exs index c9bb216df..c4fb57cb3 100644 --- a/test/pleroma/search/paradedb_test.exs +++ b/test/pleroma/search/paradedb_test.exs @@ -161,12 +161,15 @@ defmodule Pleroma.Search.ParadeDBTest do end) Config - |> expect(:get, 2, fn + |> expect(:get, 3, fn [Pleroma.Search.ParadeDB, :client_impl], nil -> ClientMock [Pleroma.Search.ParadeDB, :table], "pleroma_search_documents" -> "pleroma_search_documents" + + [Pleroma.Search.ParadeDB, :fuzzy_distance], 0 -> + 0 end) assert [%{id: ^activity2_id}, %{id: ^activity1_id}] = @@ -196,12 +199,15 @@ defmodule Pleroma.Search.ParadeDBTest do end) Config - |> expect(:get, 2, fn + |> expect(:get, 3, fn [Pleroma.Search.ParadeDB, :client_impl], nil -> ClientMock [Pleroma.Search.ParadeDB, :table], "pleroma_search_documents" -> "pleroma_search_documents" + + [Pleroma.Search.ParadeDB, :fuzzy_distance], 0 -> + 0 end) assert [%{id: ^public_activity_id}] = ParadeDB.search(nil, "swamp", limit: 40, offset: 0) @@ -225,15 +231,50 @@ defmodule Pleroma.Search.ParadeDBTest do end) Config - |> expect(:get, 2, fn + |> expect(:get, 3, fn [Pleroma.Search.ParadeDB, :client_impl], nil -> ClientMock [Pleroma.Search.ParadeDB, :table], "pleroma_search_documents" -> "pleroma_search_documents" + + [Pleroma.Search.ParadeDB, :fuzzy_distance], 0 -> + 0 end) assert [%{id: ^activity_id}] = ParadeDB.search(nil, "swamp", limit: 40, offset: 0) end + + test "search uses configured fuzzy distance" do + user = insert(:user) + + {:ok, activity} = CommonAPI.post(user, %{status: "runing shoes", visibility: "public"}) + + activity_id = activity.id + {:ok, dumped_id} = FlakeId.Ecto.CompatType.dump(activity_id) + + ClientMock + |> expect(:query, fn sql, params -> + assert sql =~ "SELECT id FROM pleroma_search_documents" + assert sql =~ "$1::pdb.fuzzy(1)" + assert ["running shoes", _limit, _offset] = params + + {:ok, %{rows: [[dumped_id]]}} + end) + + Config + |> expect(:get, 3, fn + [Pleroma.Search.ParadeDB, :client_impl], nil -> + ClientMock + + [Pleroma.Search.ParadeDB, :table], "pleroma_search_documents" -> + "pleroma_search_documents" + + [Pleroma.Search.ParadeDB, :fuzzy_distance], 0 -> + 1 + end) + + assert [%{id: ^activity_id}] = ParadeDB.search(nil, "running shoes", limit: 40, offset: 0) + end end end