From fa8541309c7fa0e45dd63ee71d054f8ab1da0369 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Thu, 29 Oct 2026 11:26:02 +0400 Subject: [PATCH] fix(search): set gin_fuzzy_search_limit only for search queries The limit was a connection parameter, so it applied to every GIN index scan, not only full-text search: recipient lookups and Oban job queries could return incomplete, random results. Set it with SET LOCAL around the search query only, configurable under Pleroma.Search.DatabaseSearch. The default stays 500 for search: measured on a 74M-object database, higher limits or none make searches for common words exceed the query timeout. Remote fetches for URL queries run outside the transaction. Based on Akkoma 811f0a468. Co-authored-by: Oneric --- changelog.d/gin-fuzzy-search-limit.fix | 1 + config/config.exs | 4 +- config/description.exs | 15 +++++ docs/configuration/search.md | 8 +++ lib/pleroma/search/database_search.ex | 60 ++++++++++++++------ test/pleroma/search/database_search_test.exs | 27 +++++++++ 6 files changed, 98 insertions(+), 17 deletions(-) create mode 100644 changelog.d/gin-fuzzy-search-limit.fix diff --git a/changelog.d/gin-fuzzy-search-limit.fix b/changelog.d/gin-fuzzy-search-limit.fix new file mode 100644 index 000000000..276c3daa9 --- /dev/null +++ b/changelog.d/gin-fuzzy-search-limit.fix @@ -0,0 +1 @@ +gin_fuzzy_search_limit is now only set for search queries (configurable under Pleroma.Search.DatabaseSearch), so other GIN index scans such as recipient lookups and Oban job queries are exact again diff --git a/config/config.exs b/config/config.exs index 1b8baed1f..32f5b6c20 100644 --- a/config/config.exs +++ b/config/config.exs @@ -843,7 +843,7 @@ config :pleroma, :modules, runtime_dir: "instance/modules" config :pleroma, configurable_from_database: false config :pleroma, Pleroma.Repo, - parameters: [gin_fuzzy_search_limit: "500", jit: "off"], + parameters: [jit: "off"], prepare: :unnamed config :pleroma, :connections_pool, @@ -948,6 +948,8 @@ config :pleroma, Pleroma.Web.WebFinger, domain: nil, update_nickname_on_user_fet config :pleroma, Pleroma.Search, module: Pleroma.Search.DatabaseSearch +config :pleroma, Pleroma.Search.DatabaseSearch, gin_fuzzy_search_limit: 500 + config :pleroma, Pleroma.Search.Meilisearch, url: "http://127.0.0.1:7700/", private_key: nil, diff --git a/config/description.exs b/config/description.exs index f435628aa..67ae4a585 100644 --- a/config/description.exs +++ b/config/description.exs @@ -3565,6 +3565,21 @@ config :pleroma, :config_description, [ } ] }, + %{ + group: :pleroma, + key: Pleroma.Search.DatabaseSearch, + type: :group, + description: "Built-in database search settings.", + children: [ + %{ + key: :gin_fuzzy_search_limit, + type: :integer, + description: + "Limits how many matches PostgreSQL considers in GIN full-text search queries (see `gin_fuzzy_search_limit` in the PostgreSQL docs). Only set for search queries. Keeps search fast on large databases, but results become incomplete and change between requests. Set to `nil` for exact results, which is fine on small instances. Default: 500.", + suggestions: [500, nil] + } + ] + }, %{ group: :pleroma, key: Pleroma.Search.Meilisearch, diff --git a/docs/configuration/search.md b/docs/configuration/search.md index 5820cdbd9..c37629184 100644 --- a/docs/configuration/search.md +++ b/docs/configuration/search.md @@ -10,6 +10,14 @@ To use built-in search that has no external dependencies, set the search module While it has no external dependencies, it has problems with performance and relevancy. +With a GIN index (the default), search queries limit how many matches PostgreSQL considers (`gin_fuzzy_search_limit`, see [PostgreSQL's docs](https://www.postgresql.org/docs/current/gin.html#GIN-TIPS)). This keeps searches for common words fast on large databases, but the results are incomplete and can change between requests. On smaller instances you can turn it off for exact results: + +> config :pleroma, Pleroma.Search.DatabaseSearch, gin_fuzzy_search_limit: nil + +Without a limit, searches for common words can take longer than the database timeout on large databases, and then return nothing. + +Earlier versions of Pleroma set this limit for every database connection, which also made other GIN index scans (for example recipient lookups) incomplete. It is now only set for search queries. + ## QdrantSearch This uses the vector search engine [Qdrant](https://qdrant.tech) to search the posts in a vector space. This needs a way to generate embeddings and uses the [OpenAI API](https://platform.openai.com/docs/guides/embeddings/what-are-embeddings). This is implemented by several project besides OpenAI itself, including the python-based fastembed-server found in `supplemental/search/fastembed-api`. diff --git a/lib/pleroma/search/database_search.ex b/lib/pleroma/search/database_search.ex index e88d632cb..c66723d1d 100644 --- a/lib/pleroma/search/database_search.ex +++ b/lib/pleroma/search/database_search.ex @@ -7,6 +7,7 @@ defmodule Pleroma.Search.DatabaseSearch do alias Pleroma.Config alias Pleroma.Object.Fetcher alias Pleroma.Pagination + alias Pleroma.Repo alias Pleroma.User alias Pleroma.Web.ActivityPub.Visibility @@ -23,22 +24,49 @@ defmodule Pleroma.Search.DatabaseSearch do offset = Keyword.get(options, :offset, 0) author = Keyword.get(options, :author) - try do - Activity - |> Activity.with_preloaded_object() - |> Activity.restrict_deactivated_users() - |> restrict_public(user) - |> query_with(index_type, search_query) - |> maybe_restrict_local(user) - |> maybe_restrict_author(author) - |> maybe_restrict_blocked(user) - |> Pagination.fetch_paginated( - %{"offset" => offset, "limit" => limit, "skip_order" => index_type == :rum}, - :offset - ) - |> maybe_fetch(user, search_query) - rescue - _ -> maybe_fetch([], user, search_query) + activities = + try do + with_fuzzy_limit(fn -> + Activity + |> Activity.with_preloaded_object() + |> Activity.restrict_deactivated_users() + |> restrict_public(user) + |> query_with(index_type, search_query) + |> maybe_restrict_local(user) + |> maybe_restrict_author(author) + |> maybe_restrict_blocked(user) + |> Pagination.fetch_paginated( + %{"offset" => offset, "limit" => limit, "skip_order" => index_type == :rum}, + :offset + ) + end) + rescue + _ -> [] + end + + # Outside the query transaction, as it may fetch over the network. + maybe_fetch(activities, user, search_query) + end + + @doc """ + Runs `fun` with `gin_fuzzy_search_limit` set, if configured under + `Pleroma.Search.DatabaseSearch`. The limit makes GIN scans faster but their + results incomplete and random, so it is only set for search queries. + """ + def with_fuzzy_limit(fun) do + case Config.get([__MODULE__, :gin_fuzzy_search_limit]) do + limit when is_integer(limit) and limit > 0 -> + {:ok, result} = + Repo.transaction(fn -> + # SET LOCAL takes no parameters; limit is an integer. + Repo.query!("SET LOCAL gin_fuzzy_search_limit TO #{limit}") + fun.() + end) + + result + + _ -> + fun.() end end diff --git a/test/pleroma/search/database_search_test.exs b/test/pleroma/search/database_search_test.exs index d8dd09063..cce810e99 100644 --- a/test/pleroma/search/database_search_test.exs +++ b/test/pleroma/search/database_search_test.exs @@ -44,4 +44,31 @@ defmodule Pleroma.Search.DatabaseSearchTest do assert result.id == other_post.id end + + describe "gin_fuzzy_search_limit" do + defp fuzzy_limit do + %{rows: [[value]]} = Pleroma.Repo.query!("SHOW gin_fuzzy_search_limit") + value + end + + test "is not set on normal database connections" do + assert fuzzy_limit() == "0" + end + + test "is only set for the search query when configured" do + clear_config([Search, :gin_fuzzy_search_limit], nil) + assert Search.with_fuzzy_limit(&fuzzy_limit/0) == "0" + + clear_config([Search, :gin_fuzzy_search_limit], 10_000) + assert Search.with_fuzzy_limit(&fuzzy_limit/0) == "10000" + end + + test "search still works with a limit" do + clear_config([Search, :gin_fuzzy_search_limit], 10_000) + {:ok, post} = CommonAPI.post(insert(:user), %{status: "it's wednesday my dudes"}) + + assert [%{id: id}] = Search.search(nil, "wednesday") + assert id == post.id + end + end end