Merge pull request 'Search: set gin_fuzzy_search_limit only for search queries' (#8038) from lambadalambda/pleroma:fix/gin-fuzzy-search-limit into develop

changelog.d/gin-fuzzy-search-limit.fix

@@ -0,0 +1 @@
+gin_fuzzy_search_limit is now only set for search queries (configurable under Pleroma.Search.DatabaseSearch), so other GIN index scans such as recipient lookups and Oban job queries are exact again

config/config.exs

@@ -843,7 +843,7 @@ config :pleroma, :modules, runtime_dir: "instance/modules"
 config :pleroma, configurable_from_database: false
 
 config :pleroma, Pleroma.Repo,
-  parameters: [gin_fuzzy_search_limit: "500", jit: "off"],
+  parameters: [jit: "off"],
   prepare: :unnamed
 
 config :pleroma, :connections_pool,
@@ -948,6 +948,8 @@ config :pleroma, Pleroma.Web.WebFinger, domain: nil, update_nickname_on_user_fet
 
 config :pleroma, Pleroma.Search, module: Pleroma.Search.DatabaseSearch
 
+config :pleroma, Pleroma.Search.DatabaseSearch, gin_fuzzy_search_limit: 500
+
 config :pleroma, Pleroma.Search.Meilisearch,
   url: "http://127.0.0.1:7700/",
   private_key: nil,

config/description.exs

@@ -3565,6 +3565,21 @@ config :pleroma, :config_description, [
       }
     ]
   },
+  %{
+    group: :pleroma,
+    key: Pleroma.Search.DatabaseSearch,
+    type: :group,
+    description: "Built-in database search settings.",
+    children: [
+      %{
+        key: :gin_fuzzy_search_limit,
+        type: :integer,
+        description:
+          "Limits how many matches PostgreSQL considers in GIN full-text search queries (see `gin_fuzzy_search_limit` in the PostgreSQL docs). Only set for search queries. Keeps search fast on large databases, but results become incomplete and change between requests. Set to `nil` for exact results, which is fine on small instances. Default: 500.",
+        suggestions: [500, nil]
+      }
+    ]
+  },
   %{
     group: :pleroma,
     key: Pleroma.Search.Meilisearch,

docs/configuration/search.md

@@ -10,6 +10,14 @@ To use built-in search that has no external dependencies, set the search module
 
 While it has no external dependencies, it has problems with performance and relevancy.
 
+With a GIN index (the default), search queries limit how many matches PostgreSQL considers (`gin_fuzzy_search_limit`, see [PostgreSQL's docs](https://www.postgresql.org/docs/current/gin.html#GIN-TIPS)). This keeps searches for common words fast on large databases, but the results are incomplete and can change between requests. On smaller instances you can turn it off for exact results:
+
+> config :pleroma, Pleroma.Search.DatabaseSearch, gin_fuzzy_search_limit: nil
+
+Without a limit, searches for common words can take longer than the database timeout on large databases, and then return nothing.
+
+Earlier versions of Pleroma set this limit for every database connection, which also made other GIN index scans (for example recipient lookups) incomplete. It is now only set for search queries.
+
 ## QdrantSearch
 
 This uses the vector search engine [Qdrant](https://qdrant.tech) to search the posts in a vector space. This needs a way to generate embeddings and uses the [OpenAI API](https://platform.openai.com/docs/guides/embeddings/what-are-embeddings). This is implemented by several project besides OpenAI itself, including the python-based fastembed-server found in `supplemental/search/fastembed-api`.

lib/pleroma/search/database_search.ex

@@ -7,6 +7,7 @@ defmodule Pleroma.Search.DatabaseSearch do
   alias Pleroma.Config
   alias Pleroma.Object.Fetcher
   alias Pleroma.Pagination
+  alias Pleroma.Repo
   alias Pleroma.User
   alias Pleroma.Web.ActivityPub.Visibility
 
@@ -23,22 +24,49 @@ defmodule Pleroma.Search.DatabaseSearch do
     offset = Keyword.get(options, :offset, 0)
     author = Keyword.get(options, :author)
 
-    try do
-      Activity
-      |> Activity.with_preloaded_object()
-      |> Activity.restrict_deactivated_users()
-      |> restrict_public(user)
-      |> query_with(index_type, search_query)
-      |> maybe_restrict_local(user)
-      |> maybe_restrict_author(author)
-      |> maybe_restrict_blocked(user)
-      |> Pagination.fetch_paginated(
-        %{"offset" => offset, "limit" => limit, "skip_order" => index_type == :rum},
-        :offset
-      )
-      |> maybe_fetch(user, search_query)
-    rescue
-      _ -> maybe_fetch([], user, search_query)
+    activities =
+      try do
+        with_fuzzy_limit(fn ->
+          Activity
+          |> Activity.with_preloaded_object()
+          |> Activity.restrict_deactivated_users()
+          |> restrict_public(user)
+          |> query_with(index_type, search_query)
+          |> maybe_restrict_local(user)
+          |> maybe_restrict_author(author)
+          |> maybe_restrict_blocked(user)
+          |> Pagination.fetch_paginated(
+            %{"offset" => offset, "limit" => limit, "skip_order" => index_type == :rum},
+            :offset
+          )
+        end)
+      rescue
+        _ -> []
+      end
+
+    # Outside the query transaction, as it may fetch over the network.
+    maybe_fetch(activities, user, search_query)
+  end
+
+  @doc """
+  Runs `fun` with `gin_fuzzy_search_limit` set, if configured under
+  `Pleroma.Search.DatabaseSearch`. The limit makes GIN scans faster but their
+  results incomplete and random, so it is only set for search queries.
+  """
+  def with_fuzzy_limit(fun) do
+    case Config.get([__MODULE__, :gin_fuzzy_search_limit]) do
+      limit when is_integer(limit) and limit > 0 ->
+        {:ok, result} =
+          Repo.transaction(fn ->
+            # SET LOCAL takes no parameters; limit is an integer.
+            Repo.query!("SET LOCAL gin_fuzzy_search_limit TO #{limit}")
+            fun.()
+          end)
+
+        result
+
+      _ ->
+        fun.()
     end
   end
 

test/pleroma/search/database_search_test.exs

@@ -44,4 +44,31 @@ defmodule Pleroma.Search.DatabaseSearchTest do
 
     assert result.id == other_post.id
   end
+
+  describe "gin_fuzzy_search_limit" do
+    defp fuzzy_limit do
+      %{rows: [[value]]} = Pleroma.Repo.query!("SHOW gin_fuzzy_search_limit")
+      value
+    end
+
+    test "is not set on normal database connections" do
+      assert fuzzy_limit() == "0"
+    end
+
+    test "is only set for the search query when configured" do
+      clear_config([Search, :gin_fuzzy_search_limit], nil)
+      assert Search.with_fuzzy_limit(&fuzzy_limit/0) == "0"
+
+      clear_config([Search, :gin_fuzzy_search_limit], 10_000)
+      assert Search.with_fuzzy_limit(&fuzzy_limit/0) == "10000"
+    end
+
+    test "search still works with a limit" do
+      clear_config([Search, :gin_fuzzy_search_limit], 10_000)
+      {:ok, post} = CommonAPI.post(insert(:user), %{status: "it's wednesday my dudes"})
+
+      assert [%{id: id}] = Search.search(nil, "wednesday")
+      assert id == post.id
+    end
+  end
 end