Merge branch 'features/validators-note' into 'develop'

Pipeline Ingestion: Note

Closes #290

See merge request pleroma/pleroma!2984
This commit is contained in:
Haelwenn 2021-06-01 01:51:38 +00:00
commit c4b4258374
31 changed files with 389 additions and 318 deletions

View file

@ -13,21 +13,33 @@ defmodule Pleroma.EctoType.ActivityPub.ObjectValidators.Recipients do
cast([object])
end
def cast(data) when is_list(data) do
data
|> Enum.reduce_while({:ok, []}, fn element, {:ok, list} ->
case ObjectID.cast(element) do
{:ok, id} ->
{:cont, {:ok, [id | list]}}
_ ->
{:halt, :error}
end
end)
def cast(object) when is_map(object) do
case ObjectID.cast(object) do
{:ok, data} -> {:ok, [data]}
_ -> :error
end
end
def cast(_) do
:error
def cast(data) when is_list(data) do
data =
data
|> Enum.reduce_while([], fn element, list ->
case ObjectID.cast(element) do
{:ok, id} ->
{:cont, [id | list]}
_ ->
{:cont, list}
end
end)
|> Enum.sort()
|> Enum.uniq()
{:ok, data}
end
def cast(data) do
{:error, data}
end
def dump(data) do

View file

@ -12,4 +12,10 @@ defmodule Pleroma.Maps do
_ -> map
end
end
def safe_put_in(data, keys, value) when is_map(data) and is_list(keys) do
Kernel.put_in(data, keys, value)
rescue
_ -> data
end
end

View file

@ -4,6 +4,7 @@
defmodule Pleroma.Object.Fetcher do
alias Pleroma.HTTP
alias Pleroma.Maps
alias Pleroma.Object
alias Pleroma.Object.Containment
alias Pleroma.Repo
@ -101,6 +102,9 @@ defmodule Pleroma.Object.Fetcher do
{:transmogrifier, {:error, {:reject, e}}} ->
{:reject, e}
{:transmogrifier, {:reject, e}} ->
{:reject, e}
{:transmogrifier, _} = e ->
{:error, e}
@ -124,12 +128,14 @@ defmodule Pleroma.Object.Fetcher do
defp prepare_activity_params(data) do
%{
"type" => "Create",
"to" => data["to"] || [],
"cc" => data["cc"] || [],
# Should we seriously keep this attributedTo thing?
"actor" => data["actor"] || data["attributedTo"],
"object" => data
}
|> Maps.put_if_present("to", data["to"])
|> Maps.put_if_present("cc", data["cc"])
|> Maps.put_if_present("bto", data["bto"])
|> Maps.put_if_present("bcc", data["bcc"])
end
def fetch_object_from_id!(id, options \\ []) do

View file

@ -88,7 +88,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
defp increase_replies_count_if_reply(_create_data), do: :noop
@object_types ~w[ChatMessage Question Answer Audio Video Event Article]
@object_types ~w[ChatMessage Question Answer Audio Video Event Article Note]
@impl true
def persist(%{"type" => type} = object, meta) when type in @object_types do
with {:ok, object} <- Object.create(object) do

View file

@ -102,7 +102,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do
%{"type" => "Create", "object" => %{"type" => objtype} = object} = create_activity,
meta
)
when objtype in ~w[Question Answer Audio Video Event Article] do
when objtype in ~w[Question Answer Audio Video Event Article Note] do
with {:ok, object_data} <- cast_and_apply(object),
meta = Keyword.put(meta, :object_data, object_data |> stringify_keys),
{:ok, create_activity} <-
@ -115,7 +115,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do
end
def validate(%{"type" => type} = object, meta)
when type in ~w[Event Question Audio Video Article] do
when type in ~w[Event Question Audio Video Article Note] do
validator =
case type do
"Event" -> EventValidator
@ -123,6 +123,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do
"Audio" -> AudioVideoValidator
"Video" -> AudioVideoValidator
"Article" -> ArticleNoteValidator
"Note" -> ArticleNoteValidator
end
with {:ok, object} <-
@ -194,7 +195,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do
EventValidator.cast_and_apply(object)
end
def cast_and_apply(%{"type" => "Article"} = object) do
def cast_and_apply(%{"type" => type} = object) when type in ~w[Article Note] do
ArticleNoteValidator.cast_and_apply(object)
end

View file

@ -6,6 +6,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AnswerValidator do
use Ecto.Schema
alias Pleroma.EctoType.ActivityPub.ObjectValidators
alias Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes
alias Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations
import Ecto.Changeset
@ -23,6 +24,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AnswerValidator do
field(:name, :string)
field(:inReplyTo, ObjectValidators.ObjectID)
field(:attributedTo, ObjectValidators.ObjectID)
field(:context, :string)
# TODO: Remove actor on objects
field(:actor, ObjectValidators.ObjectID)
@ -46,6 +48,11 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AnswerValidator do
end
def changeset(struct, data) do
data =
data
|> CommonFixes.fix_actor()
|> CommonFixes.fix_object_defaults()
struct
|> cast(data, __schema__(:fields))
end

View file

@ -50,6 +50,8 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNoteValidator do
field(:likes, {:array, ObjectValidators.ObjectID}, default: [])
field(:announcements, {:array, ObjectValidators.ObjectID}, default: [])
field(:replies, {:array, ObjectValidators.ObjectID}, default: [])
end
def cast_and_apply(data) do
@ -65,25 +67,39 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNoteValidator do
end
def cast_data(data) do
data = fix(data)
%__MODULE__{}
|> changeset(data)
end
defp fix_url(%{"url" => url} = data) when is_map(url) do
Map.put(data, "url", url["href"])
end
defp fix_url(%{"url" => url} = data) when is_bitstring(url), do: data
defp fix_url(%{"url" => url} = data) when is_map(url), do: Map.put(data, "url", url["href"])
defp fix_url(data), do: data
defp fix_tag(%{"tag" => tag} = data) when is_list(tag), do: data
defp fix_tag(%{"tag" => tag} = data) when is_map(tag), do: Map.put(data, "tag", [tag])
defp fix_tag(data), do: Map.drop(data, ["tag"])
defp fix_replies(%{"replies" => %{"first" => %{"items" => replies}}} = data)
when is_list(replies),
do: Map.put(data, "replies", replies)
defp fix_replies(%{"replies" => %{"items" => replies}} = data) when is_list(replies),
do: Map.put(data, "replies", replies)
defp fix_replies(%{"replies" => replies} = data) when is_bitstring(replies),
do: Map.drop(data, ["replies"])
defp fix_replies(data), do: data
defp fix(data) do
data
|> CommonFixes.fix_defaults()
|> CommonFixes.fix_attribution()
|> CommonFixes.fix_actor()
|> CommonFixes.fix_object_defaults()
|> fix_url()
|> fix_tag()
|> fix_replies()
|> Transmogrifier.fix_emoji()
|> Transmogrifier.fix_content_map()
end
def changeset(struct, data) do

View file

@ -119,9 +119,8 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AudioVideoValidator do
defp fix(data) do
data
|> CommonFixes.fix_defaults()
|> CommonFixes.fix_attribution()
|> CommonFixes.fix_actor()
|> CommonFixes.fix_object_defaults()
|> Transmogrifier.fix_emoji()
|> fix_url()
|> fix_content()

View file

@ -3,26 +3,55 @@
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes do
alias Pleroma.EctoType.ActivityPub.ObjectValidators
alias Pleroma.Object.Containment
alias Pleroma.User
alias Pleroma.Web.ActivityPub.Transmogrifier
alias Pleroma.Web.ActivityPub.Utils
# based on Pleroma.Web.ActivityPub.Utils.lazy_put_objects_defaults
def fix_defaults(data) do
def cast_and_filter_recipients(message, field, follower_collection, field_fallback \\ []) do
{:ok, data} = ObjectValidators.Recipients.cast(message[field] || field_fallback)
data =
Enum.reject(data, fn x ->
String.ends_with?(x, "/followers") and x != follower_collection
end)
Map.put(message, field, data)
end
def fix_object_defaults(data) do
%{data: %{"id" => context}, id: context_id} =
Utils.create_context(data["context"] || data["conversation"])
%User{follower_address: follower_collection} = User.get_cached_by_ap_id(data["attributedTo"])
data
|> Map.put("context", context)
|> Map.put("context_id", context_id)
|> cast_and_filter_recipients("to", follower_collection)
|> cast_and_filter_recipients("cc", follower_collection)
|> cast_and_filter_recipients("bto", follower_collection)
|> cast_and_filter_recipients("bcc", follower_collection)
|> Transmogrifier.fix_implicit_addressing(follower_collection)
end
def fix_attribution(data) do
data
|> Map.put_new("actor", data["attributedTo"])
def fix_activity_addressing(activity, _meta) do
%User{follower_address: follower_collection} = User.get_cached_by_ap_id(activity["actor"])
activity
|> cast_and_filter_recipients("to", follower_collection)
|> cast_and_filter_recipients("cc", follower_collection)
|> cast_and_filter_recipients("bto", follower_collection)
|> cast_and_filter_recipients("bcc", follower_collection)
|> Transmogrifier.fix_implicit_addressing(follower_collection)
end
def fix_actor(data) do
actor = Containment.get_actor(data)
actor =
data
|> Map.put_new("actor", data["attributedTo"])
|> Containment.get_actor()
data
|> Map.put("actor", actor)

View file

@ -15,6 +15,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations do
fields
|> Enum.map(fn field -> get_field(cng, field) end)
|> Enum.any?(fn
nil -> false
[] -> false
_ -> true
end)

View file

@ -10,8 +10,10 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateGenericValidator do
alias Pleroma.EctoType.ActivityPub.ObjectValidators
alias Pleroma.Object
alias Pleroma.User
alias Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes
alias Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations
alias Pleroma.Web.ActivityPub.Transmogrifier
import Ecto.Changeset
@ -23,6 +25,8 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateGenericValidator do
field(:type, :string)
field(:to, ObjectValidators.Recipients, default: [])
field(:cc, ObjectValidators.Recipients, default: [])
field(:bto, ObjectValidators.Recipients, default: [])
field(:bcc, ObjectValidators.Recipients, default: [])
field(:object, ObjectValidators.ObjectID)
field(:expires_at, ObjectValidators.DateTime)
@ -54,39 +58,37 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateGenericValidator do
|> cast(data, __schema__(:fields))
end
defp fix_context(data, meta) do
if object = meta[:object_data] do
Map.put_new(data, "context", object["context"])
else
data
end
end
# CommonFixes.fix_activity_addressing adapted for Create specific behavior
defp fix_addressing(data, object) do
%User{follower_address: follower_collection} = User.get_cached_by_ap_id(data["actor"])
data
|> CommonFixes.cast_and_filter_recipients("to", follower_collection, object["to"])
|> CommonFixes.cast_and_filter_recipients("cc", follower_collection, object["cc"])
|> CommonFixes.cast_and_filter_recipients("bto", follower_collection, object["bto"])
|> CommonFixes.cast_and_filter_recipients("bcc", follower_collection, object["bcc"])
|> Transmogrifier.fix_implicit_addressing(follower_collection)
end
def fix(data, meta) do
object = meta[:object_data]
defp fix_addressing(data, meta) do
if object = meta[:object_data] do
data
|> Map.put_new("to", object["to"] || [])
|> Map.put_new("cc", object["cc"] || [])
else
data
end
end
defp fix(data, meta) do
data
|> fix_context(meta)
|> fix_addressing(meta)
|> CommonFixes.fix_actor()
|> Map.put_new("context", object["context"])
|> fix_addressing(object)
end
defp validate_data(cng, meta) do
object = meta[:object_data]
cng
|> validate_required([:actor, :type, :object])
|> validate_required([:actor, :type, :object, :to, :cc])
|> validate_inclusion(:type, ["Create"])
|> CommonValidations.validate_actor_presence()
|> CommonValidations.validate_any_presence([:to, :cc])
|> validate_actors_match(meta)
|> validate_context_match(meta)
|> validate_actors_match(object)
|> validate_context_match(object)
|> validate_addressing_match(object)
|> validate_object_nonexistence()
|> validate_object_containment()
end
@ -118,8 +120,8 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateGenericValidator do
end)
end
def validate_actors_match(cng, meta) do
attributed_to = meta[:object_data]["attributedTo"] || meta[:object_data]["actor"]
def validate_actors_match(cng, object) do
attributed_to = object["attributedTo"] || object["actor"]
cng
|> validate_change(:actor, fn :actor, actor ->
@ -131,7 +133,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateGenericValidator do
end)
end
def validate_context_match(cng, %{object_data: %{"context" => object_context}}) do
def validate_context_match(cng, %{"context" => object_context}) do
cng
|> validate_change(:context, fn :context, context ->
if context == object_context do
@ -142,5 +144,18 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateGenericValidator do
end)
end
def validate_context_match(cng, _), do: cng
def validate_addressing_match(cng, object) do
[:to, :cc, :bcc, :bto]
|> Enum.reduce(cng, fn field, cng ->
object_data = object[to_string(field)]
validate_change(cng, field, fn field, data ->
if data == object_data do
[]
else
[{field, "field doesn't match with object (#{inspect(object_data)})"}]
end
end)
end)
end
end

View file

@ -1,29 +0,0 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateNoteValidator do
use Ecto.Schema
alias Pleroma.EctoType.ActivityPub.ObjectValidators
alias Pleroma.Web.ActivityPub.ObjectValidators.NoteValidator
import Ecto.Changeset
@primary_key false
embedded_schema do
field(:id, ObjectValidators.ObjectID, primary_key: true)
field(:actor, ObjectValidators.ObjectID)
field(:type, :string)
field(:to, ObjectValidators.Recipients, default: [])
field(:cc, ObjectValidators.Recipients, default: [])
field(:bto, ObjectValidators.Recipients, default: [])
field(:bcc, ObjectValidators.Recipients, default: [])
embeds_one(:object, NoteValidator)
end
def cast_data(data) do
cast(%__MODULE__{}, data, __schema__(:fields))
end
end

View file

@ -72,8 +72,8 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.EventValidator do
defp fix(data) do
data
|> CommonFixes.fix_defaults()
|> CommonFixes.fix_attribution()
|> CommonFixes.fix_actor()
|> CommonFixes.fix_object_defaults()
|> Transmogrifier.fix_emoji()
end

View file

@ -83,8 +83,8 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.QuestionValidator do
defp fix(data) do
data
|> CommonFixes.fix_defaults()
|> CommonFixes.fix_attribution()
|> CommonFixes.fix_actor()
|> CommonFixes.fix_object_defaults()
|> Transmogrifier.fix_emoji()
|> fix_closed()
end

View file

@ -203,6 +203,19 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do
Object.increase_replies_count(in_reply_to)
end
reply_depth = (meta[:depth] || 0) + 1
# FIXME: Force inReplyTo to replies
if Pleroma.Web.Federator.allowed_thread_distance?(reply_depth) and
object.data["replies"] != nil do
for reply_id <- object.data["replies"] do
Pleroma.Workers.RemoteFetcherWorker.enqueue("fetch_remote", %{
"id" => reply_id,
"depth" => reply_depth
})
end
end
ConcurrentLimiter.limit(Pleroma.Web.RichMedia.Helpers, fn ->
Task.start(fn -> Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity) end)
end)
@ -423,7 +436,7 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do
end
def handle_object_creation(%{"type" => objtype} = object, meta)
when objtype in ~w[Audio Video Question Event Article] do
when objtype in ~w[Audio Video Question Event Article Note] do
with {:ok, object, meta} <- Pipeline.common_pipeline(object, meta) do
{:ok, object, meta}
end

View file

@ -43,7 +43,6 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
|> fix_content_map()
|> fix_addressing()
|> fix_summary()
|> fix_type(options)
end
def fix_summary(%{"summary" => nil} = object) do
@ -72,17 +71,21 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
end
end
def fix_explicit_addressing(
%{"to" => to, "cc" => cc} = object,
explicit_mentions,
follower_collection
) do
explicit_to = Enum.filter(to, fn x -> x in explicit_mentions end)
# if directMessage flag is set to true, leave the addressing alone
def fix_explicit_addressing(%{"directMessage" => true} = object, _follower_collection),
do: object
def fix_explicit_addressing(%{"to" => to, "cc" => cc} = object, follower_collection) do
explicit_mentions =
Utils.determine_explicit_mentions(object) ++
[Pleroma.Constants.as_public(), follower_collection]
explicit_to = Enum.filter(to, fn x -> x in explicit_mentions end)
explicit_cc = Enum.filter(to, fn x -> x not in explicit_mentions end)
final_cc =
(cc ++ explicit_cc)
|> Enum.filter(& &1)
|> Enum.reject(fn x -> String.ends_with?(x, "/followers") and x != follower_collection end)
|> Enum.uniq()
@ -91,29 +94,6 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
|> Map.put("cc", final_cc)
end
def fix_explicit_addressing(object, _explicit_mentions, _followers_collection), do: object
# if directMessage flag is set to true, leave the addressing alone
def fix_explicit_addressing(%{"directMessage" => true} = object), do: object
def fix_explicit_addressing(object) do
explicit_mentions = Utils.determine_explicit_mentions(object)
%User{follower_address: follower_collection} =
object
|> Containment.get_actor()
|> User.get_cached_by_ap_id()
explicit_mentions =
explicit_mentions ++
[
Pleroma.Constants.as_public(),
follower_collection
]
fix_explicit_addressing(object, explicit_mentions, follower_collection)
end
# if as:Public is addressed, then make sure the followers collection is also addressed
# so that the activities will be delivered to local users.
def fix_implicit_addressing(%{"to" => to, "cc" => cc} = object, followers_collection) do
@ -137,19 +117,19 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
end
end
def fix_implicit_addressing(object, _), do: object
def fix_addressing(object) do
{:ok, %User{} = user} = User.get_or_fetch_by_ap_id(object["actor"])
followers_collection = User.ap_followers(user)
{:ok, %User{follower_address: follower_collection}} =
object
|> Containment.get_actor()
|> User.get_or_fetch_by_ap_id()
object
|> fix_addressing_list("to")
|> fix_addressing_list("cc")
|> fix_addressing_list("bto")
|> fix_addressing_list("bcc")
|> fix_explicit_addressing()
|> fix_implicit_addressing(followers_collection)
|> fix_explicit_addressing(follower_collection)
|> fix_implicit_addressing(follower_collection)
end
def fix_actor(%{"attributedTo" => actor} = object) do
@ -342,19 +322,18 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
def fix_content_map(object), do: object
def fix_type(object, options \\ [])
defp fix_type(%{"type" => "Note", "inReplyTo" => reply_id, "name" => _} = object, options)
when is_binary(reply_id) do
options = Keyword.put(options, :fetch, true)
def fix_type(%{"inReplyTo" => reply_id, "name" => _} = object, options)
when is_binary(reply_id) do
with true <- Federator.allowed_thread_distance?(options[:depth]),
{:ok, %{data: %{"type" => "Question"} = _} = _} <- get_obj_helper(reply_id, options) do
with %Object{data: %{"type" => "Question"}} <- Object.normalize(reply_id, options) do
Map.put(object, "type", "Answer")
else
_ -> object
end
end
def fix_type(object, _), do: object
defp fix_type(object, _options), do: object
# Reduce the object list to find the reported user.
defp get_reported(objects) do
@ -425,10 +404,9 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
# - tags
# - emoji
def handle_incoming(
%{"type" => "Create", "object" => %{"type" => objtype} = object} = data,
%{"type" => "Create", "object" => %{"type" => "Page"} = object} = data,
options
)
when objtype in ~w{Note Page} do
) do
actor = Containment.get_actor(data)
with nil <- Activity.get_create_by_object_ap_id(object["id"]),
@ -520,14 +498,23 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
def handle_incoming(
%{"type" => "Create", "object" => %{"type" => objtype, "id" => obj_id}} = data,
_options
options
)
when objtype in ~w{Question Answer ChatMessage Audio Video Event Article} do
data = Map.put(data, "object", strip_internal_fields(data["object"]))
when objtype in ~w{Question Answer ChatMessage Audio Video Event Article Note} do
fetch_options = Keyword.put(options, :depth, (options[:depth] || 0) + 1)
object =
data["object"]
|> strip_internal_fields()
|> fix_type(fetch_options)
|> fix_in_reply_to(fetch_options)
data = Map.put(data, "object", object)
options = Keyword.put(options, :local, false)
with {:ok, %User{}} <- ObjectValidator.fetch_actor(data),
nil <- Activity.get_create_by_object_ap_id(obj_id),
{:ok, activity, _} <- Pipeline.common_pipeline(data, local: false) do
{:ok, activity, _} <- Pipeline.common_pipeline(data, options) do
{:ok, activity}
else
%Activity{} = activity -> {:ok, activity}

View file

@ -96,8 +96,11 @@ defmodule Pleroma.Web.ActivityPub.Utils do
!label_in_collection?(ap_id, params["cc"])
if need_splice? do
cc_list = extract_list(params["cc"])
Map.put(params, "cc", [ap_id | cc_list])
cc = [ap_id | extract_list(params["cc"])]
params
|> Map.put("cc", cc)
|> Maps.safe_put_in(["object", "cc"], cc)
else
params
end

View file

@ -96,6 +96,11 @@ defmodule Pleroma.Web.Federator do
Logger.debug("Unhandled actor #{actor}, #{inspect(e)}")
{:error, e}
{:error, {:validate_object, _}} = e ->
Logger.error("Incoming AP doc validation error: #{inspect(e)}")
Logger.debug(Jason.encode!(params, pretty: true))
e
e ->
# Just drop those for now
Logger.debug(fn -> "Unhandled activity\n" <> Jason.encode!(params, pretty: true) end)