Fix ParadeDB search indexing

changelog.d/paradedb-search.add

@@ -1 +1 @@
-Add ParadeDB (pg_search) as an optional external search backend.
+Add ParadeDB (pg_search) as an optional external search backend, indexing status content, content warnings, and attachment descriptions.

config/config.exs

@@ -946,7 +946,7 @@ config :pleroma, Pleroma.Search.ParadeDB,
   fuzzy_distance: 0
 
 config :pleroma, Pleroma.Search.ParadeDB.Repo,
-  pool_size: 10,
+  pool_size: 2,
   prepare: :unnamed
 
 config :pleroma, Pleroma.Application,

docs/configuration/search.md

@@ -85,7 +85,7 @@ You will see a "Default Admin API Key", this is the key you actually put into yo
 
 ### Initial indexing
 
-After setting up the configuration, you'll want to index all of your already existing posts. Only public posts are indexed.  You'll only
+After setting up the configuration, you'll want to index all of your already existing posts. Only public/unlisted posts are indexed.  You'll only
 have to do it one time, but it might take a while, depending on the amount of posts your instance has seen. This is also a fairly RAM
 consuming process for `meilisearch`, and it will take a lot of RAM when running if you have a lot of posts (seems to be around 5G for ~1.2
 million posts while idle and up to 7G while indexing initially, but your experience may be different).
@@ -153,8 +153,10 @@ external search backend by pointing Pleroma at a separate Postgres instance with
 
 Pleroma will maintain a small `pleroma_search_documents` table in that database (via the existing search indexing queue) and run search
 queries against it.
+The indexed text includes status content, content warnings, and attachment descriptions.
 
 To use it, set the search module to `Pleroma.Search.ParadeDB` and configure the ParadeDB database URL:
+These settings must be in static/runtime config (or `PARADEDB_DATABASE_URL`) so the dedicated ParadeDB Repo is started with the application.
 
 > config :pleroma, Pleroma.Search, module: Pleroma.Search.ParadeDB
 >
@@ -162,6 +164,10 @@ To use it, set the search module to `Pleroma.Search.ParadeDB` and configure the
 >   url: System.get_env("PARADEDB_DATABASE_URL") || "postgres://postgres:postgres@127.0.0.1:5432/paradedb",
 >   table: "pleroma_search_documents",
 >   fuzzy_distance: 0
+>
+> config :pleroma, Pleroma.Search.ParadeDB.Repo,
+>   pool_size: 2,
+>   prepare: :unnamed
 
 Then, create the table and BM25 index once:
 

lib/pleroma/search.ex

@@ -5,13 +5,12 @@ defmodule Pleroma.Search do
   alias Pleroma.Workers.SearchIndexingWorker
 
   @spec add_to_index(Activity.t()) :: {:ok, Oban.Job.t() | :noop} | {:error, Oban.Job.changeset()}
-  def add_to_index(%Activity{id: activity_id, object: %Object{} = object} = activity) do
+  def add_to_index(%Activity{id: activity_id, object: %Object{}} = activity) do
-    with {_, true} <- {:indexable, indexable?(activity)},
+    if indexable?(activity) do
-         {_, "public"} <- {:visibility, Visibility.get_visibility(object)} do
       SearchIndexingWorker.new(%{"op" => "add_to_index", "activity" => activity_id})
       |> Oban.insert()
     else
-      _ -> {:ok, :noop}
+      {:ok, :noop}
     end
   end
 
@@ -41,40 +40,59 @@ defmodule Pleroma.Search do
   def object_to_search_data(%Object{} = object) do
     data = object.data
 
-    content_str =
-      case data["content"] do
-        [nil | rest] -> to_string(rest)
-        str -> str
-      end
-
     content =
-      with {:ok, scrubbed} <-
+      data
-             FastSanitize.Sanitizer.scrub(content_str, Pleroma.HTML.Scrubber.SearchIndexing),
+      |> search_texts()
-           trimmed <- String.trim(scrubbed) do
+      |> Enum.map(&sanitize_search_text/1)
-        trimmed
+      |> Enum.reject(&(&1 in ["", "."]))
-      end
+      |> Enum.join(" ")
-
-    # Make sure we have a non-empty string
-    if content != "" do
-      {:ok, published, _} = DateTime.from_iso8601(data["published"])
 
+    with true <- content != "",
+         published when is_binary(published) <- data["published"],
+         {:ok, published, _} <- DateTime.from_iso8601(published) do
       %{
         id: object.id,
         content: content,
         ap: data["id"],
         published: published |> DateTime.to_unix()
       }
+    else
+      _ -> nil
     end
   end
 
-  defp indexable?(%Activity{
+  def indexable?(%Activity{
         data: %{"type" => "Create"},
-         object: %Object{
+        object: %Object{data: %{"published" => published, "type" => "Note"}} = object
-           data: %{"content" => content, "published" => published, "type" => "Note"}
-         }
       })
-       when not is_nil(content) and content not in ["", "."] and not is_nil(published),
+      when not is_nil(published) do
-       do: true
+    Visibility.get_visibility(object) in ["public", "unlisted"] and
-
+      not is_nil(object_to_search_data(object))
-  defp indexable?(_), do: false
+  end
+
+  def indexable?(_), do: false
+
+  defp search_texts(data) do
+    [data["content"], data["summary"] | attachment_names(data["attachment"])]
+  end
+
+  defp attachment_names(attachments) when is_list(attachments) do
+    Enum.map(attachments, fn
+      %{"name" => name} -> name
+      _ -> nil
+    end)
+  end
+
+  defp attachment_names(_), do: []
+
+  defp sanitize_search_text([nil | rest]), do: sanitize_search_text(to_string(rest))
+
+  defp sanitize_search_text(text) when is_binary(text) do
+    case FastSanitize.Sanitizer.scrub(text, Pleroma.HTML.Scrubber.SearchIndexing) do
+      {:ok, scrubbed} -> String.trim(scrubbed)
+      _ -> ""
+    end
+  end
+
+  defp sanitize_search_text(_), do: ""
 end

lib/pleroma/search/parade_db.ex

@@ -66,9 +66,10 @@ defmodule Pleroma.Search.ParadeDB do
 
   @impl true
   def drop_index do
-    with {:ok, _} <- client_impl().query("DROP TABLE IF EXISTS #{table()}", []) do
+    case client_impl().query("DROP TABLE IF EXISTS #{table()}", []) do
+      {:ok, _} ->
         :ok
-    else
+
       {:error, error} ->
         Logger.error("ParadeDB drop_index failed: #{inspect(error)}")
         {:error, error}
@@ -80,9 +81,9 @@ defmodule Pleroma.Search.ParadeDB do
 
   @impl true
   def add_to_index(activity) do
-    maybe_search_data = Search.object_to_search_data(activity.object)
+    with true <- Search.indexable?(activity),
-
+         maybe_search_data when not is_nil(maybe_search_data) <-
-    if activity.data["type"] == "Create" and maybe_search_data do
+           Search.object_to_search_data(activity.object) do
       dumped_activity_id = dump_activity_id(activity.id)
       actor_ap_id = activity.data["actor"]
       published_at = DateTime.from_unix!(maybe_search_data.published)
@@ -122,7 +123,7 @@ defmodule Pleroma.Search.ParadeDB do
         end
       end
     else
-      :ok
+      _ -> :ok
     end
   end
 

lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex

@@ -24,7 +24,11 @@ defmodule Pleroma.Web.ActivityPub.MRF.AntiFollowbotPolicy do
   defp score_displayname("fedibot"), do: 1.0
   defp score_displayname(_), do: 0.0
 
-  defp determine_if_followbot(%User{nickname: nickname, name: displayname, actor_type: actor_type}) do
+  defp determine_if_followbot(%User{
+         nickname: nickname,
+         name: displayname,
+         actor_type: actor_type
+       }) do
     # nickname will be a binary string except when following a relay
     nick_score =
       if is_binary(nickname) do

lib/pleroma/web/activity_pub/mrf/simple_policy.ex

@@ -108,7 +108,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do
   end
 
   defp intersection(list1, list2) do
-    list1 -- list1 -- list2
+    list1 -- (list1 -- list2)
   end
 
   defp check_followers_only(%{host: actor_host} = _actor_info, activity) do

lib/pleroma/web/mastodon_api/views/status_view.ex

@@ -680,7 +680,9 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
     %{ancestors: ancestors, descendants: descendants} =
       activities
       |> Enum.reverse()
-      |> Enum.group_by(fn %{id: id} -> if id < activity.id, do: :ancestors, else: :descendants end)
+      |> Enum.group_by(fn %{id: id} ->
+        if id < activity.id, do: :ancestors, else: :descendants
+      end)
       |> Map.put_new(:ancestors, [])
       |> Map.put_new(:descendants, [])
 

lib/pleroma/workers/mailer_worker.ex

@@ -6,7 +6,9 @@ defmodule Pleroma.Workers.MailerWorker do
   use Oban.Worker, queue: :background
 
   @impl true
-  def perform(%Job{args: %{"op" => "email", "encoded_email" => encoded_email, "config" => config}}) do
+  def perform(%Job{
+        args: %{"op" => "email", "encoded_email" => encoded_email, "config" => config}
+      }) do
     encoded_email
     |> Base.decode64!()
     |> :erlang.binary_to_term()

test/pleroma/search/meilisearch_test.exs

@@ -99,7 +99,7 @@ defmodule Pleroma.Search.MeilisearchTest do
 
         %{method: :delete, url: "http://127.0.0.1:7700/indexes/objects/documents/" <> id} ->
           send(self(), "called_delete")
-          assert String.length(id) > 1
+          assert id != ""
           json(%{})
       end)
 

test/pleroma/search/parade_db_test.exs

@@ -75,15 +75,21 @@ defmodule Pleroma.Search.ParadeDBTest do
           })
 
         args = %{"op" => "add_to_index", "activity" => activity.id}
-        assert_enqueued(worker: SearchIndexingWorker, args: args)
+        refute_enqueued(worker: SearchIndexingWorker, args: args)
-
-        Config
-        |> expect(:get, fn
-          [Pleroma.Search, :module], nil ->
-            ParadeDB
       end)
+    end
 
-        assert :ok = perform_job(SearchIndexingWorker, args)
+    test "backend skips non-public posts when called directly" do
+      user = insert(:user)
+
+      Enum.each(["private", "direct"], fn visibility ->
+        {:ok, activity} =
+          CommonAPI.post(user, %{
+            status: "guys i just don't wanna leave the swamp",
+            visibility: visibility
+          })
+
+        assert :ok = ParadeDB.add_to_index(activity)
       end)
     end
 

test/pleroma/search_test.exs

@@ -3,26 +3,32 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.SearchTest do
+  require Pleroma.Constants
+
   use Pleroma.DataCase, async: true
   use Oban.Testing, repo: Pleroma.Repo
 
   import Pleroma.Factory
 
+  alias Pleroma.Object
+  alias Pleroma.Search
   alias Pleroma.Web.CommonAPI
   alias Pleroma.Workers.SearchIndexingWorker
 
-  test "indexes posts that are public" do
+  test "indexes posts that are public or unlisted" do
     user = insert(:user)
 
+    Enum.each(["public", "unlisted"], fn visibility ->
       {:ok, activity} =
         CommonAPI.post(user, %{
           status: "Well this is a story all about how my life got flipped turned upside down",
-        visibility: "public"
+          visibility: visibility
         })
 
       args = %{"op" => "add_to_index", "activity" => activity.id}
 
       assert_enqueued(worker: SearchIndexingWorker, args: args)
+    end)
   end
 
   test "doesn't index posts that are not public" do
@@ -66,4 +72,76 @@ defmodule Pleroma.SearchTest do
 
     refute_enqueued(worker: SearchIndexingWorker, args: args)
   end
+
+  test "object_to_search_data includes content warnings and attachment descriptions" do
+    object = %Object{
+      id: 1,
+      data: %{
+        "id" => "https://example.com/objects/1",
+        "type" => "Note",
+        "content" => "body text",
+        "summary" => "subject text",
+        "attachment" => [
+          %{"type" => "Document", "name" => "image description"},
+          %{"type" => "Document", "name" => ""}
+        ],
+        "published" => DateTime.utc_now() |> DateTime.to_iso8601()
+      }
+    }
+
+    assert %{content: content} = Search.object_to_search_data(object)
+    assert content =~ "body text"
+    assert content =~ "subject text"
+    assert content =~ "image description"
+  end
+
+  test "object_to_search_data ignores objects with invalid published dates" do
+    for published <- [nil, 123, "not a date"] do
+      object = %Object{
+        id: 1,
+        data: %{
+          "id" => "https://example.com/objects/1",
+          "type" => "Note",
+          "content" => "body text",
+          "published" => published
+        }
+      }
+
+      refute Search.object_to_search_data(object)
+    end
+  end
+
+  test "indexable? accepts searchable content warnings and attachment descriptions" do
+    public = Pleroma.Constants.as_public()
+    published = DateTime.utc_now() |> DateTime.to_iso8601()
+
+    for data <- [
+          %{"content" => ".", "summary" => "subject text", "attachment" => []},
+          %{
+            "content" => ".",
+            "summary" => "",
+            "attachment" => [%{"type" => "Document", "name" => "image description"}]
+          }
+        ] do
+      object = %Object{
+        id: 1,
+        data:
+          Map.merge(data, %{
+            "id" => "https://example.com/objects/1",
+            "type" => "Note",
+            "to" => [public],
+            "published" => published
+          })
+      }
+
+      activity = %Pleroma.Activity{
+        id: Ecto.UUID.generate(),
+        data: %{"type" => "Create"},
+        object: object,
+        recipients: [public]
+      }
+
+      assert Search.indexable?(activity)
+    end
+  end
 end