Extend Pleroma.Pagination to support offset-based pagination, use async/await to execute status and account search in parallel
This commit is contained in:
Eugenij
kaniini
parent
4d382d1347
commit
4198c3ac39
8 changed files with 193 additions and 62 deletions
|
|
@ -344,5 +344,5 @@ defmodule Pleroma.Activity do
|
|||
)
|
||||
end
|
||||
|
||||
defdelegate search(user, query), to: Pleroma.Activity.Search
|
||||
defdelegate search(user, query, options \\ []), to: Pleroma.Activity.Search
|
||||
end
|
||||
|
|
|
|||
|
|
@ -5,14 +5,17 @@
|
|||
defmodule Pleroma.Activity.Search do
|
||||
alias Pleroma.Activity
|
||||
alias Pleroma.Object.Fetcher
|
||||
alias Pleroma.Repo
|
||||
alias Pleroma.Pagination
|
||||
alias Pleroma.User
|
||||
alias Pleroma.Web.ActivityPub.Visibility
|
||||
|
||||
import Ecto.Query
|
||||
|
||||
def search(user, search_query) do
|
||||
def search(user, search_query, options \\ []) do
|
||||
index_type = if Pleroma.Config.get([:database, :rum_enabled]), do: :rum, else: :gin
|
||||
limit = Enum.min([Keyword.get(options, :limit), 40])
|
||||
offset = Keyword.get(options, :offset, 0)
|
||||
author = Keyword.get(options, :author)
|
||||
|
||||
Activity
|
||||
|> Activity.with_preloaded_object()
|
||||
|
|
@ -20,15 +23,23 @@ defmodule Pleroma.Activity.Search do
|
|||
|> restrict_public()
|
||||
|> query_with(index_type, search_query)
|
||||
|> maybe_restrict_local(user)
|
||||
|> Repo.all()
|
||||
|> maybe_restrict_author(author)
|
||||
|> Pagination.fetch_paginated(%{"offset" => offset, "limit" => limit}, :offset)
|
||||
|> maybe_fetch(user, search_query)
|
||||
end
|
||||
|
||||
def maybe_restrict_author(query, %User{} = author) do
|
||||
from([a, o] in query,
|
||||
where: a.actor == ^author.ap_id
|
||||
)
|
||||
end
|
||||
|
||||
def maybe_restrict_author(query, _), do: query
|
||||
|
||||
defp restrict_public(q) do
|
||||
from([a, o] in q,
|
||||
where: fragment("?->>'type' = 'Create'", a.data),
|
||||
where: "https://www.w3.org/ns/activitystreams#Public" in a.recipients,
|
||||
limit: 40
|
||||
where: "https://www.w3.org/ns/activitystreams#Public" in a.recipients
|
||||
)
|
||||
end
|
||||
|
||||
|
|
|
|||
|
|
@ -14,16 +14,28 @@ defmodule Pleroma.Pagination do
|
|||
|
||||
@default_limit 20
|
||||
|
||||
def fetch_paginated(query, params) do
|
||||
def fetch_paginated(query, params, type \\ :keyset)
|
||||
|
||||
def fetch_paginated(query, params, :keyset) do
|
||||
options = cast_params(params)
|
||||
|
||||
query
|
||||
|> paginate(options)
|
||||
|> paginate(options, :keyset)
|
||||
|> Repo.all()
|
||||
|> enforce_order(options)
|
||||
end
|
||||
|
||||
def paginate(query, options) do
|
||||
def fetch_paginated(query, params, :offset) do
|
||||
options = cast_params(params)
|
||||
|
||||
query
|
||||
|> paginate(options, :offset)
|
||||
|> Repo.all()
|
||||
end
|
||||
|
||||
def paginate(query, options, method \\ :keyset)
|
||||
|
||||
def paginate(query, options, :keyset) do
|
||||
query
|
||||
|> restrict(:min_id, options)
|
||||
|> restrict(:since_id, options)
|
||||
|
|
@ -32,11 +44,18 @@ defmodule Pleroma.Pagination do
|
|||
|> restrict(:limit, options)
|
||||
end
|
||||
|
||||
def paginate(query, options, :offset) do
|
||||
query
|
||||
|> restrict(:offset, options)
|
||||
|> restrict(:limit, options)
|
||||
end
|
||||
|
||||
defp cast_params(params) do
|
||||
param_types = %{
|
||||
min_id: :string,
|
||||
since_id: :string,
|
||||
max_id: :string,
|
||||
offset: :integer,
|
||||
limit: :integer
|
||||
}
|
||||
|
||||
|
|
@ -70,6 +89,10 @@ defmodule Pleroma.Pagination do
|
|||
order_by(query, [u], fragment("? desc nulls last", u.id))
|
||||
end
|
||||
|
||||
defp restrict(query, :offset, %{offset: offset}) do
|
||||
offset(query, ^offset)
|
||||
end
|
||||
|
||||
defp restrict(query, :limit, options) do
|
||||
limit = Map.get(options, :limit, @default_limit)
|
||||
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@
|
|||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
defmodule Pleroma.User.Search do
|
||||
alias Pleroma.Pagination
|
||||
alias Pleroma.Repo
|
||||
alias Pleroma.User
|
||||
import Ecto.Query
|
||||
|
|
@ -32,8 +33,7 @@ defmodule Pleroma.User.Search do
|
|||
|
||||
query_string
|
||||
|> search_query(for_user, following)
|
||||
|> paginate(result_limit, offset)
|
||||
|> Repo.all()
|
||||
|> Pagination.fetch_paginated(%{"offset" => offset, "limit" => result_limit}, :offset)
|
||||
end)
|
||||
|
||||
results
|
||||
|
|
@ -87,10 +87,6 @@ defmodule Pleroma.User.Search do
|
|||
|
||||
defp filter_blocked_domains(query, _), do: query
|
||||
|
||||
defp paginate(query, limit, offset) do
|
||||
from(q in query, limit: ^limit, offset: ^offset)
|
||||
end
|
||||
|
||||
defp union_subqueries({fts_subquery, trigram_subquery}) do
|
||||
from(s in trigram_subquery, union_all: ^fts_subquery)
|
||||
end
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ defmodule Pleroma.Web.MastodonAPI.SearchController do
|
|||
|
||||
alias Pleroma.Activity
|
||||
alias Pleroma.Plugs.RateLimiter
|
||||
alias Pleroma.Repo
|
||||
alias Pleroma.User
|
||||
alias Pleroma.Web
|
||||
alias Pleroma.Web.ControllerHelper
|
||||
|
|
@ -16,43 +17,6 @@ defmodule Pleroma.Web.MastodonAPI.SearchController do
|
|||
require Logger
|
||||
plug(RateLimiter, :search when action in [:search, :search2, :account_search])
|
||||
|
||||
def search2(%{assigns: %{user: user}} = conn, %{"q" => query} = params) do
|
||||
accounts = with_fallback(fn -> User.search(query, search_options(params, user)) end, [])
|
||||
statuses = with_fallback(fn -> Activity.search(user, query) end, [])
|
||||
|
||||
tags_path = Web.base_url() <> "/tag/"
|
||||
|
||||
tags =
|
||||
query
|
||||
|> prepare_tags
|
||||
|> Enum.map(fn tag -> %{name: tag, url: tags_path <> tag} end)
|
||||
|
||||
res = %{
|
||||
"accounts" => AccountView.render("accounts.json", users: accounts, for: user, as: :user),
|
||||
"statuses" =>
|
||||
StatusView.render("index.json", activities: statuses, for: user, as: :activity),
|
||||
"hashtags" => tags
|
||||
}
|
||||
|
||||
json(conn, res)
|
||||
end
|
||||
|
||||
def search(%{assigns: %{user: user}} = conn, %{"q" => query} = params) do
|
||||
accounts = with_fallback(fn -> User.search(query, search_options(params, user)) end)
|
||||
statuses = with_fallback(fn -> Activity.search(user, query) end)
|
||||
|
||||
tags = prepare_tags(query)
|
||||
|
||||
res = %{
|
||||
"accounts" => AccountView.render("accounts.json", users: accounts, for: user, as: :user),
|
||||
"statuses" =>
|
||||
StatusView.render("index.json", activities: statuses, for: user, as: :activity),
|
||||
"hashtags" => tags
|
||||
}
|
||||
|
||||
json(conn, res)
|
||||
end
|
||||
|
||||
def account_search(%{assigns: %{user: user}} = conn, %{"q" => query} = params) do
|
||||
accounts = User.search(query, search_options(params, user))
|
||||
res = AccountView.render("accounts.json", users: accounts, for: user, as: :user)
|
||||
|
|
@ -60,12 +24,36 @@ defmodule Pleroma.Web.MastodonAPI.SearchController do
|
|||
json(conn, res)
|
||||
end
|
||||
|
||||
defp prepare_tags(query) do
|
||||
query
|
||||
|> String.split()
|
||||
|> Enum.uniq()
|
||||
|> Enum.filter(fn tag -> String.starts_with?(tag, "#") end)
|
||||
|> Enum.map(fn tag -> String.slice(tag, 1..-1) end)
|
||||
def search2(conn, params), do: do_search(:v2, conn, params)
|
||||
def search(conn, params), do: do_search(:v1, conn, params)
|
||||
|
||||
defp do_search(version, %{assigns: %{user: user}} = conn, %{"q" => query} = params) do
|
||||
options = search_options(params, user)
|
||||
timeout = Keyword.get(Repo.config(), :timeout, 15_000)
|
||||
default_values = %{"statuses" => [], "accounts" => [], "hashtags" => []}
|
||||
|
||||
result =
|
||||
default_values
|
||||
|> Enum.map(fn {resource, default_value} ->
|
||||
if params["type"] == nil or params["type"] == resource do
|
||||
{resource, fn -> resource_search(version, resource, query, options) end}
|
||||
else
|
||||
{resource, fn -> default_value end}
|
||||
end
|
||||
end)
|
||||
|> Task.async_stream(fn {resource, f} -> {resource, with_fallback(f)} end,
|
||||
timeout: timeout,
|
||||
on_timeout: :kill_task
|
||||
)
|
||||
|> Enum.reduce(default_values, fn
|
||||
{:ok, {resource, result}}, acc ->
|
||||
Map.put(acc, resource, result)
|
||||
|
||||
_error, acc ->
|
||||
acc
|
||||
end)
|
||||
|
||||
json(conn, result)
|
||||
end
|
||||
|
||||
defp search_options(params, user) do
|
||||
|
|
@ -74,8 +62,45 @@ defmodule Pleroma.Web.MastodonAPI.SearchController do
|
|||
following: params["following"] == "true",
|
||||
limit: ControllerHelper.fetch_integer_param(params, "limit"),
|
||||
offset: ControllerHelper.fetch_integer_param(params, "offset"),
|
||||
type: params["type"],
|
||||
author: get_author(params),
|
||||
for_user: user
|
||||
]
|
||||
|> Enum.filter(&elem(&1, 1))
|
||||
end
|
||||
|
||||
defp resource_search(_, "accounts", query, options) do
|
||||
accounts = with_fallback(fn -> User.search(query, options) end)
|
||||
AccountView.render("accounts.json", users: accounts, for: options[:for_user], as: :user)
|
||||
end
|
||||
|
||||
defp resource_search(_, "statuses", query, options) do
|
||||
statuses = with_fallback(fn -> Activity.search(options[:for_user], query, options) end)
|
||||
StatusView.render("index.json", activities: statuses, for: options[:for_user], as: :activity)
|
||||
end
|
||||
|
||||
defp resource_search(:v2, "hashtags", query, _options) do
|
||||
tags_path = Web.base_url() <> "/tag/"
|
||||
|
||||
query
|
||||
|> prepare_tags()
|
||||
|> Enum.map(fn tag ->
|
||||
tag = String.trim_leading(tag, "#")
|
||||
%{name: tag, url: tags_path <> tag}
|
||||
end)
|
||||
end
|
||||
|
||||
defp resource_search(:v1, "hashtags", query, _options) do
|
||||
query
|
||||
|> prepare_tags()
|
||||
|> Enum.map(fn tag -> String.trim_leading(tag, "#") end)
|
||||
end
|
||||
|
||||
defp prepare_tags(query) do
|
||||
query
|
||||
|> String.split()
|
||||
|> Enum.uniq()
|
||||
|> Enum.filter(fn tag -> String.starts_with?(tag, "#") end)
|
||||
end
|
||||
|
||||
defp with_fallback(f, fallback \\ []) do
|
||||
|
|
@ -87,4 +112,9 @@ defmodule Pleroma.Web.MastodonAPI.SearchController do
|
|||
fallback
|
||||
end
|
||||
end
|
||||
|
||||
defp get_author(%{"account_id" => account_id}) when is_binary(account_id),
|
||||
do: User.get_cached_by_id(account_id)
|
||||
|
||||
defp get_author(_params), do: nil
|
||||
end
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue