diff --git a/changelog.d/paradedb-search.add b/changelog.d/paradedb-search.add index 00a6e2361..f72ebca14 100644 --- a/changelog.d/paradedb-search.add +++ b/changelog.d/paradedb-search.add @@ -1 +1 @@ -Add ParadeDB (pg_search) as an optional external search backend. +Add ParadeDB (pg_search) as an optional external search backend, indexing status content, content warnings, and attachment descriptions. diff --git a/config/config.exs b/config/config.exs index 2e405413c..3c308f093 100644 --- a/config/config.exs +++ b/config/config.exs @@ -946,7 +946,7 @@ config :pleroma, Pleroma.Search.ParadeDB, fuzzy_distance: 0 config :pleroma, Pleroma.Search.ParadeDB.Repo, - pool_size: 10, + pool_size: 2, prepare: :unnamed config :pleroma, Pleroma.Application, diff --git a/docs/configuration/search.md b/docs/configuration/search.md index c0d3f8462..1d5056ddf 100644 --- a/docs/configuration/search.md +++ b/docs/configuration/search.md @@ -85,7 +85,7 @@ You will see a "Default Admin API Key", this is the key you actually put into yo ### Initial indexing -After setting up the configuration, you'll want to index all of your already existing posts. Only public posts are indexed. You'll only +After setting up the configuration, you'll want to index all of your already existing posts. Only public/unlisted posts are indexed. You'll only have to do it one time, but it might take a while, depending on the amount of posts your instance has seen. This is also a fairly RAM consuming process for `meilisearch`, and it will take a lot of RAM when running if you have a lot of posts (seems to be around 5G for ~1.2 million posts while idle and up to 7G while indexing initially, but your experience may be different). @@ -153,8 +153,10 @@ external search backend by pointing Pleroma at a separate Postgres instance with Pleroma will maintain a small `pleroma_search_documents` table in that database (via the existing search indexing queue) and run search queries against it. +The indexed text includes status content, content warnings, and attachment descriptions. To use it, set the search module to `Pleroma.Search.ParadeDB` and configure the ParadeDB database URL: +These settings must be in static/runtime config (or `PARADEDB_DATABASE_URL`) so the dedicated ParadeDB Repo is started with the application. > config :pleroma, Pleroma.Search, module: Pleroma.Search.ParadeDB > @@ -162,6 +164,10 @@ To use it, set the search module to `Pleroma.Search.ParadeDB` and configure the > url: System.get_env("PARADEDB_DATABASE_URL") || "postgres://postgres:postgres@127.0.0.1:5432/paradedb", > table: "pleroma_search_documents", > fuzzy_distance: 0 +> +> config :pleroma, Pleroma.Search.ParadeDB.Repo, +> pool_size: 2, +> prepare: :unnamed Then, create the table and BM25 index once: diff --git a/lib/pleroma/search.ex b/lib/pleroma/search.ex index 9cd2768c4..cd4907026 100644 --- a/lib/pleroma/search.ex +++ b/lib/pleroma/search.ex @@ -5,13 +5,12 @@ defmodule Pleroma.Search do alias Pleroma.Workers.SearchIndexingWorker @spec add_to_index(Activity.t()) :: {:ok, Oban.Job.t() | :noop} | {:error, Oban.Job.changeset()} - def add_to_index(%Activity{id: activity_id, object: %Object{} = object} = activity) do - with {_, true} <- {:indexable, indexable?(activity)}, - {_, "public"} <- {:visibility, Visibility.get_visibility(object)} do + def add_to_index(%Activity{id: activity_id, object: %Object{}} = activity) do + if indexable?(activity) do SearchIndexingWorker.new(%{"op" => "add_to_index", "activity" => activity_id}) |> Oban.insert() else - _ -> {:ok, :noop} + {:ok, :noop} end end @@ -41,40 +40,59 @@ defmodule Pleroma.Search do def object_to_search_data(%Object{} = object) do data = object.data - content_str = - case data["content"] do - [nil | rest] -> to_string(rest) - str -> str - end - content = - with {:ok, scrubbed} <- - FastSanitize.Sanitizer.scrub(content_str, Pleroma.HTML.Scrubber.SearchIndexing), - trimmed <- String.trim(scrubbed) do - trimmed - end - - # Make sure we have a non-empty string - if content != "" do - {:ok, published, _} = DateTime.from_iso8601(data["published"]) + data + |> search_texts() + |> Enum.map(&sanitize_search_text/1) + |> Enum.reject(&(&1 in ["", "."])) + |> Enum.join(" ") + with true <- content != "", + published when is_binary(published) <- data["published"], + {:ok, published, _} <- DateTime.from_iso8601(published) do %{ id: object.id, content: content, ap: data["id"], published: published |> DateTime.to_unix() } + else + _ -> nil end end - defp indexable?(%Activity{ - data: %{"type" => "Create"}, - object: %Object{ - data: %{"content" => content, "published" => published, "type" => "Note"} - } - }) - when not is_nil(content) and content not in ["", "."] and not is_nil(published), - do: true + def indexable?(%Activity{ + data: %{"type" => "Create"}, + object: %Object{data: %{"published" => published, "type" => "Note"}} = object + }) + when not is_nil(published) do + Visibility.get_visibility(object) in ["public", "unlisted"] and + not is_nil(object_to_search_data(object)) + end - defp indexable?(_), do: false + def indexable?(_), do: false + + defp search_texts(data) do + [data["content"], data["summary"] | attachment_names(data["attachment"])] + end + + defp attachment_names(attachments) when is_list(attachments) do + Enum.map(attachments, fn + %{"name" => name} -> name + _ -> nil + end) + end + + defp attachment_names(_), do: [] + + defp sanitize_search_text([nil | rest]), do: sanitize_search_text(to_string(rest)) + + defp sanitize_search_text(text) when is_binary(text) do + case FastSanitize.Sanitizer.scrub(text, Pleroma.HTML.Scrubber.SearchIndexing) do + {:ok, scrubbed} -> String.trim(scrubbed) + _ -> "" + end + end + + defp sanitize_search_text(_), do: "" end diff --git a/lib/pleroma/search/paradedb.ex b/lib/pleroma/search/parade_db.ex similarity index 96% rename from lib/pleroma/search/paradedb.ex rename to lib/pleroma/search/parade_db.ex index 20be5b6f9..ee00e74af 100644 --- a/lib/pleroma/search/paradedb.ex +++ b/lib/pleroma/search/parade_db.ex @@ -66,9 +66,10 @@ defmodule Pleroma.Search.ParadeDB do @impl true def drop_index do - with {:ok, _} <- client_impl().query("DROP TABLE IF EXISTS #{table()}", []) do - :ok - else + case client_impl().query("DROP TABLE IF EXISTS #{table()}", []) do + {:ok, _} -> + :ok + {:error, error} -> Logger.error("ParadeDB drop_index failed: #{inspect(error)}") {:error, error} @@ -80,9 +81,9 @@ defmodule Pleroma.Search.ParadeDB do @impl true def add_to_index(activity) do - maybe_search_data = Search.object_to_search_data(activity.object) - - if activity.data["type"] == "Create" and maybe_search_data do + with true <- Search.indexable?(activity), + maybe_search_data when not is_nil(maybe_search_data) <- + Search.object_to_search_data(activity.object) do dumped_activity_id = dump_activity_id(activity.id) actor_ap_id = activity.data["actor"] published_at = DateTime.from_unix!(maybe_search_data.published) @@ -122,7 +123,7 @@ defmodule Pleroma.Search.ParadeDB do end end else - :ok + _ -> :ok end end diff --git a/lib/pleroma/search/paradedb/client.ex b/lib/pleroma/search/parade_db/client.ex similarity index 100% rename from lib/pleroma/search/paradedb/client.ex rename to lib/pleroma/search/parade_db/client.ex diff --git a/lib/pleroma/search/paradedb/repo.ex b/lib/pleroma/search/parade_db/repo.ex similarity index 100% rename from lib/pleroma/search/paradedb/repo.ex rename to lib/pleroma/search/parade_db/repo.ex diff --git a/lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex b/lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex index 8ea61aec2..d0320ce49 100644 --- a/lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex @@ -24,7 +24,11 @@ defmodule Pleroma.Web.ActivityPub.MRF.AntiFollowbotPolicy do defp score_displayname("fedibot"), do: 1.0 defp score_displayname(_), do: 0.0 - defp determine_if_followbot(%User{nickname: nickname, name: displayname, actor_type: actor_type}) do + defp determine_if_followbot(%User{ + nickname: nickname, + name: displayname, + actor_type: actor_type + }) do # nickname will be a binary string except when following a relay nick_score = if is_binary(nickname) do diff --git a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex index a97e8db7b..93f0616ec 100644 --- a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex @@ -108,7 +108,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do end defp intersection(list1, list2) do - list1 -- list1 -- list2 + list1 -- (list1 -- list2) end defp check_followers_only(%{host: actor_host} = _actor_info, activity) do diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index 6949c5f2d..5b18261a7 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -680,7 +680,9 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do %{ancestors: ancestors, descendants: descendants} = activities |> Enum.reverse() - |> Enum.group_by(fn %{id: id} -> if id < activity.id, do: :ancestors, else: :descendants end) + |> Enum.group_by(fn %{id: id} -> + if id < activity.id, do: :ancestors, else: :descendants + end) |> Map.put_new(:ancestors, []) |> Map.put_new(:descendants, []) diff --git a/lib/pleroma/workers/mailer_worker.ex b/lib/pleroma/workers/mailer_worker.ex index b0259b191..9ef6a80c9 100644 --- a/lib/pleroma/workers/mailer_worker.ex +++ b/lib/pleroma/workers/mailer_worker.ex @@ -6,7 +6,9 @@ defmodule Pleroma.Workers.MailerWorker do use Oban.Worker, queue: :background @impl true - def perform(%Job{args: %{"op" => "email", "encoded_email" => encoded_email, "config" => config}}) do + def perform(%Job{ + args: %{"op" => "email", "encoded_email" => encoded_email, "config" => config} + }) do encoded_email |> Base.decode64!() |> :erlang.binary_to_term() diff --git a/test/pleroma/search/meilisearch_test.exs b/test/pleroma/search/meilisearch_test.exs index ff32491c5..6e4a42c95 100644 --- a/test/pleroma/search/meilisearch_test.exs +++ b/test/pleroma/search/meilisearch_test.exs @@ -99,7 +99,7 @@ defmodule Pleroma.Search.MeilisearchTest do %{method: :delete, url: "http://127.0.0.1:7700/indexes/objects/documents/" <> id} -> send(self(), "called_delete") - assert String.length(id) > 1 + assert id != "" json(%{}) end) diff --git a/test/pleroma/search/paradedb_integration_test.exs b/test/pleroma/search/parade_db_integration_test.exs similarity index 100% rename from test/pleroma/search/paradedb_integration_test.exs rename to test/pleroma/search/parade_db_integration_test.exs diff --git a/test/pleroma/search/paradedb_test.exs b/test/pleroma/search/parade_db_test.exs similarity index 94% rename from test/pleroma/search/paradedb_test.exs rename to test/pleroma/search/parade_db_test.exs index 3a3b8298f..3133b6b38 100644 --- a/test/pleroma/search/paradedb_test.exs +++ b/test/pleroma/search/parade_db_test.exs @@ -75,15 +75,21 @@ defmodule Pleroma.Search.ParadeDBTest do }) args = %{"op" => "add_to_index", "activity" => activity.id} - assert_enqueued(worker: SearchIndexingWorker, args: args) + refute_enqueued(worker: SearchIndexingWorker, args: args) + end) + end - Config - |> expect(:get, fn - [Pleroma.Search, :module], nil -> - ParadeDB - end) + test "backend skips non-public posts when called directly" do + user = insert(:user) - assert :ok = perform_job(SearchIndexingWorker, args) + Enum.each(["private", "direct"], fn visibility -> + {:ok, activity} = + CommonAPI.post(user, %{ + status: "guys i just don't wanna leave the swamp", + visibility: visibility + }) + + assert :ok = ParadeDB.add_to_index(activity) end) end diff --git a/test/pleroma/search_test.exs b/test/pleroma/search_test.exs index d777bcda2..00d8b10ed 100644 --- a/test/pleroma/search_test.exs +++ b/test/pleroma/search_test.exs @@ -3,26 +3,32 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.SearchTest do + require Pleroma.Constants + use Pleroma.DataCase, async: true use Oban.Testing, repo: Pleroma.Repo import Pleroma.Factory + alias Pleroma.Object + alias Pleroma.Search alias Pleroma.Web.CommonAPI alias Pleroma.Workers.SearchIndexingWorker - test "indexes posts that are public" do + test "indexes posts that are public or unlisted" do user = insert(:user) - {:ok, activity} = - CommonAPI.post(user, %{ - status: "Well this is a story all about how my life got flipped turned upside down", - visibility: "public" - }) + Enum.each(["public", "unlisted"], fn visibility -> + {:ok, activity} = + CommonAPI.post(user, %{ + status: "Well this is a story all about how my life got flipped turned upside down", + visibility: visibility + }) - args = %{"op" => "add_to_index", "activity" => activity.id} + args = %{"op" => "add_to_index", "activity" => activity.id} - assert_enqueued(worker: SearchIndexingWorker, args: args) + assert_enqueued(worker: SearchIndexingWorker, args: args) + end) end test "doesn't index posts that are not public" do @@ -66,4 +72,76 @@ defmodule Pleroma.SearchTest do refute_enqueued(worker: SearchIndexingWorker, args: args) end + + test "object_to_search_data includes content warnings and attachment descriptions" do + object = %Object{ + id: 1, + data: %{ + "id" => "https://example.com/objects/1", + "type" => "Note", + "content" => "body text", + "summary" => "subject text", + "attachment" => [ + %{"type" => "Document", "name" => "image description"}, + %{"type" => "Document", "name" => ""} + ], + "published" => DateTime.utc_now() |> DateTime.to_iso8601() + } + } + + assert %{content: content} = Search.object_to_search_data(object) + assert content =~ "body text" + assert content =~ "subject text" + assert content =~ "image description" + end + + test "object_to_search_data ignores objects with invalid published dates" do + for published <- [nil, 123, "not a date"] do + object = %Object{ + id: 1, + data: %{ + "id" => "https://example.com/objects/1", + "type" => "Note", + "content" => "body text", + "published" => published + } + } + + refute Search.object_to_search_data(object) + end + end + + test "indexable? accepts searchable content warnings and attachment descriptions" do + public = Pleroma.Constants.as_public() + published = DateTime.utc_now() |> DateTime.to_iso8601() + + for data <- [ + %{"content" => ".", "summary" => "subject text", "attachment" => []}, + %{ + "content" => ".", + "summary" => "", + "attachment" => [%{"type" => "Document", "name" => "image description"}] + } + ] do + object = %Object{ + id: 1, + data: + Map.merge(data, %{ + "id" => "https://example.com/objects/1", + "type" => "Note", + "to" => [public], + "published" => published + }) + } + + activity = %Pleroma.Activity{ + id: Ecto.UUID.generate(), + data: %{"type" => "Create"}, + object: object, + recipients: [public] + } + + assert Search.indexable?(activity) + end + end end