Merge branch 'feature/database-compaction' into 'develop'

database compaction

See merge request pleroma/pleroma!473
This commit is contained in:
kaniini 2019-04-19 16:55:33 +00:00
commit 9da8b287f8
37 changed files with 764 additions and 579 deletions

View file

@ -84,17 +84,21 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do
{:ok, status_two} = CommonAPI.post(user, %{"status" => ". #essais"})
{:ok, status_three} = CommonAPI.post(user, %{"status" => ". #test #reject"})
fetch_one = ActivityPub.fetch_activities([], %{"tag" => "test"})
fetch_two = ActivityPub.fetch_activities([], %{"tag" => ["test", "essais"]})
fetch_one = ActivityPub.fetch_activities([], %{"type" => "Create", "tag" => "test"})
fetch_two =
ActivityPub.fetch_activities([], %{"type" => "Create", "tag" => ["test", "essais"]})
fetch_three =
ActivityPub.fetch_activities([], %{
"type" => "Create",
"tag" => ["test", "essais"],
"tag_reject" => ["reject"]
})
fetch_four =
ActivityPub.fetch_activities([], %{
"type" => "Create",
"tag" => ["test"],
"tag_all" => ["test", "reject"]
})
@ -192,8 +196,10 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do
}
{:ok, %Activity{} = activity} = ActivityPub.insert(data)
assert is_binary(activity.data["object"]["id"])
assert %Object{} = Object.get_by_ap_id(activity.data["object"]["id"])
object = Object.normalize(activity.data["object"])
assert is_binary(object.data["id"])
assert %Object{} = Object.get_by_ap_id(activity.data["object"])
end
end
@ -206,7 +212,11 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do
to: ["user1", "user1", "user2"],
actor: user,
context: "",
object: %{}
object: %{
"to" => ["user1", "user1", "user2"],
"type" => "Note",
"content" => "testing"
}
})
assert activity.data["to"] == ["user1", "user2"]
@ -244,25 +254,21 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do
# public
{:ok, _} = CommonAPI.post(user2, Map.put(reply_data, "visibility", "public"))
assert %{data: data, object: object} = Activity.get_by_ap_id_with_object(ap_id)
assert data["object"]["repliesCount"] == 1
assert object.data["repliesCount"] == 1
# unlisted
{:ok, _} = CommonAPI.post(user2, Map.put(reply_data, "visibility", "unlisted"))
assert %{data: data, object: object} = Activity.get_by_ap_id_with_object(ap_id)
assert data["object"]["repliesCount"] == 2
assert object.data["repliesCount"] == 2
# private
{:ok, _} = CommonAPI.post(user2, Map.put(reply_data, "visibility", "private"))
assert %{data: data, object: object} = Activity.get_by_ap_id_with_object(ap_id)
assert data["object"]["repliesCount"] == 2
assert object.data["repliesCount"] == 2
# direct
{:ok, _} = CommonAPI.post(user2, Map.put(reply_data, "visibility", "direct"))
assert %{data: data, object: object} = Activity.get_by_ap_id_with_object(ap_id)
assert data["object"]["repliesCount"] == 2
assert object.data["repliesCount"] == 2
end
end
@ -680,40 +686,13 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do
end
end
describe "fetching an object" do
test "it fetches an object" do
{:ok, object} =
ActivityPub.fetch_object_from_id("http://mastodon.example.org/@admin/99541947525187367")
describe "fetch the latest Follow" do
test "fetches the latest Follow activity" do
%Activity{data: %{"type" => "Follow"}} = activity = insert(:follow_activity)
follower = Repo.get_by(User, ap_id: activity.data["actor"])
followed = Repo.get_by(User, ap_id: activity.data["object"])
assert activity = Activity.get_create_by_object_ap_id(object.data["id"])
assert activity.data["id"]
{:ok, object_again} =
ActivityPub.fetch_object_from_id("http://mastodon.example.org/@admin/99541947525187367")
assert [attachment] = object.data["attachment"]
assert is_list(attachment["url"])
assert object == object_again
end
test "it works with objects only available via Ostatus" do
{:ok, object} = ActivityPub.fetch_object_from_id("https://shitposter.club/notice/2827873")
assert activity = Activity.get_create_by_object_ap_id(object.data["id"])
assert activity.data["id"]
{:ok, object_again} =
ActivityPub.fetch_object_from_id("https://shitposter.club/notice/2827873")
assert object == object_again
end
test "it correctly stitches up conversations between ostatus and ap" do
last = "https://mstdn.io/users/mayuutann/statuses/99568293732299394"
{:ok, object} = ActivityPub.fetch_object_from_id(last)
object = Object.get_by_ap_id(object.data["inReplyTo"])
assert object
assert activity == Utils.fetch_latest_follow(follower, followed)
end
end
@ -804,10 +783,10 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do
{:ok, a4} =
CommonAPI.post(User.get_by_id(user.id), %{"status" => "yeah", "visibility" => "direct"})
{:ok, _} = a1.data["object"]["id"] |> Object.get_by_ap_id() |> ActivityPub.delete()
{:ok, _} = a2.data["object"]["id"] |> Object.get_by_ap_id() |> ActivityPub.delete()
{:ok, _} = a3.data["object"]["id"] |> Object.get_by_ap_id() |> ActivityPub.delete()
{:ok, _} = a4.data["object"]["id"] |> Object.get_by_ap_id() |> ActivityPub.delete()
{:ok, _} = Object.normalize(a1) |> ActivityPub.delete()
{:ok, _} = Object.normalize(a2) |> ActivityPub.delete()
{:ok, _} = Object.normalize(a3) |> ActivityPub.delete()
{:ok, _} = Object.normalize(a4) |> ActivityPub.delete()
user = User.get_by_id(user.id)
assert user.info.note_count == 10
@ -849,22 +828,18 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do
_ = CommonAPI.delete(direct_reply.id, user2)
assert %{data: data, object: object} = Activity.get_by_ap_id_with_object(ap_id)
assert data["object"]["repliesCount"] == 2
assert object.data["repliesCount"] == 2
_ = CommonAPI.delete(private_reply.id, user2)
assert %{data: data, object: object} = Activity.get_by_ap_id_with_object(ap_id)
assert data["object"]["repliesCount"] == 2
assert object.data["repliesCount"] == 2
_ = CommonAPI.delete(public_reply.id, user2)
assert %{data: data, object: object} = Activity.get_by_ap_id_with_object(ap_id)
assert data["object"]["repliesCount"] == 1
assert object.data["repliesCount"] == 1
_ = CommonAPI.delete(unlisted_reply.id, user2)
assert %{data: data, object: object} = Activity.get_by_ap_id_with_object(ap_id)
assert data["object"]["repliesCount"] == 0
assert object.data["repliesCount"] == 0
end
end
@ -906,7 +881,10 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do
activities = ActivityPub.fetch_activities([user1.ap_id | user1.following])
private_activity_1 = Activity.get_by_ap_id_with_object(private_activity_1.data["id"])
assert [public_activity, private_activity_1, private_activity_3] == activities
assert [public_activity, private_activity_1, private_activity_3] ==
activities
assert length(activities) == 3
activities = ActivityPub.contain_timeline(activities, user1)
@ -916,15 +894,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do
end
end
test "it can fetch plume articles" do
{:ok, object} =
ActivityPub.fetch_object_from_id(
"https://baptiste.gelez.xyz/~/PlumeDevelopment/this-month-in-plume-june-2018/"
)
assert object
end
describe "update" do
test "it creates an update activity with the new user data" do
user = insert(:user)
@ -946,15 +915,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do
end
end
test "it can fetch peertube videos" do
{:ok, object} =
ActivityPub.fetch_object_from_id(
"https://peertube.moe/videos/watch/df5f464b-be8d-46fb-ad81-2d4c2d1630e3"
)
assert object
end
test "returned pinned statuses" do
Pleroma.Config.put([:instance, :max_pinned_statuses], 3)
user = insert(:user)

View file

@ -6,6 +6,7 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do
use Pleroma.DataCase
alias Pleroma.Activity
alias Pleroma.Object
alias Pleroma.Object.Fetcher
alias Pleroma.Repo
alias Pleroma.User
alias Pleroma.Web.ActivityPub.ActivityPub
@ -50,14 +51,14 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do
|> Map.put("object", object)
{:ok, returned_activity} = Transmogrifier.handle_incoming(data)
returned_object = Object.normalize(returned_activity.data["object"])
assert activity =
Activity.get_create_by_object_ap_id(
"tag:shitposter.club,2017-05-05:noticeId=2827873:objectType=comment"
)
assert returned_activity.data["object"]["inReplyToAtomUri"] ==
"https://shitposter.club/notice/2827873"
assert returned_object.data["inReplyToAtomUri"] == "https://shitposter.club/notice/2827873"
end
test "it works for incoming notices" do
@ -80,7 +81,7 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do
assert data["actor"] == "http://mastodon.example.org/users/admin"
object = data["object"]
object = Object.normalize(data["object"]).data
assert object["id"] == "http://mastodon.example.org/users/admin/statuses/99512778738411822"
assert object["to"] == ["https://www.w3.org/ns/activitystreams#Public"]
@ -107,7 +108,9 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do
data = File.read!("test/fixtures/mastodon-post-activity-hashtag.json") |> Poison.decode!()
{:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(data)
assert Enum.at(data["object"]["tag"], 2) == "moo"
object = Object.normalize(data["object"])
assert Enum.at(object.data["tag"], 2) == "moo"
end
test "it works for incoming notices with contentMap" do
@ -115,8 +118,9 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do
File.read!("test/fixtures/mastodon-post-activity-contentmap.json") |> Poison.decode!()
{:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(data)
object = Object.normalize(data["object"])
assert data["object"]["content"] ==
assert object.data["content"] ==
"<p><span class=\"h-card\"><a href=\"http://localtesting.pleroma.lol/users/lain\" class=\"u-url mention\">@<span>lain</span></a></span></p>"
end
@ -124,8 +128,9 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do
data = File.read!("test/fixtures/kroeg-post-activity.json") |> Poison.decode!()
{:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(data)
object = Object.normalize(data["object"])
assert data["object"]["content"] ==
assert object.data["content"] ==
"<p>henlo from my Psion netBook</p><p>message sent from my Psion netBook</p>"
end
@ -141,24 +146,27 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do
data = File.read!("test/fixtures/kroeg-array-less-emoji.json") |> Poison.decode!()
{:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(data)
object = Object.normalize(data["object"])
assert data["object"]["emoji"] == %{
assert object.data["emoji"] == %{
"icon_e_smile" => "https://puckipedia.com/forum/images/smilies/icon_e_smile.png"
}
data = File.read!("test/fixtures/kroeg-array-less-hashtag.json") |> Poison.decode!()
{:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(data)
object = Object.normalize(data["object"])
assert "test" in data["object"]["tag"]
assert "test" in object.data["tag"]
end
test "it works for incoming notices with url not being a string (prismo)" do
data = File.read!("test/fixtures/prismo-url-map.json") |> Poison.decode!()
{:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(data)
object = Object.normalize(data["object"])
assert data["object"]["url"] == "https://prismo.news/posts/83"
assert object.data["url"] == "https://prismo.news/posts/83"
end
test "it cleans up incoming notices which are not really DMs" do
@ -180,15 +188,15 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do
data = Map.put(data, "object", object)
{:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(data)
{:ok, %Activity{data: data, local: false} = activity} = Transmogrifier.handle_incoming(data)
assert data["to"] == []
assert data["cc"] == to
object = data["object"]
object_data = Object.normalize(activity).data
assert object["to"] == []
assert object["cc"] == to
assert object_data["to"] == []
assert object_data["cc"] == to
end
test "it works for incoming follow requests" do
@ -231,14 +239,14 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do
data =
File.read!("test/fixtures/mastodon-like.json")
|> Poison.decode!()
|> Map.put("object", activity.data["object"]["id"])
|> Map.put("object", activity.data["object"])
{:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(data)
assert data["actor"] == "http://mastodon.example.org/users/admin"
assert data["type"] == "Like"
assert data["id"] == "http://mastodon.example.org/users/admin#likes/2"
assert data["object"] == activity.data["object"]["id"]
assert data["object"] == activity.data["object"]
end
test "it returns an error for incoming unlikes wihout a like activity" do
@ -248,7 +256,7 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do
data =
File.read!("test/fixtures/mastodon-undo-like.json")
|> Poison.decode!()
|> Map.put("object", activity.data["object"]["id"])
|> Map.put("object", activity.data["object"])
assert Transmogrifier.handle_incoming(data) == :error
end
@ -260,7 +268,7 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do
like_data =
File.read!("test/fixtures/mastodon-like.json")
|> Poison.decode!()
|> Map.put("object", activity.data["object"]["id"])
|> Map.put("object", activity.data["object"])
{:ok, %Activity{data: like_data, local: false}} = Transmogrifier.handle_incoming(like_data)
@ -302,7 +310,7 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do
data =
File.read!("test/fixtures/mastodon-announce.json")
|> Poison.decode!()
|> Map.put("object", activity.data["object"]["id"])
|> Map.put("object", activity.data["object"])
{:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(data)
@ -312,7 +320,7 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do
assert data["id"] ==
"http://mastodon.example.org/users/admin/statuses/99542391527669785/activity"
assert data["object"] == activity.data["object"]["id"]
assert data["object"] == activity.data["object"]
assert Activity.get_create_by_object_ap_id(data["object"]).id == activity.id
end
@ -324,7 +332,7 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do
data =
File.read!("test/fixtures/mastodon-announce.json")
|> Poison.decode!()
|> Map.put("object", activity.data["object"]["id"])
|> Map.put("object", Object.normalize(activity).data["id"])
|> Map.put("to", ["http://mastodon.example.org/users/admin/followers"])
|> Map.put("cc", [])
@ -450,7 +458,7 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do
object =
data["object"]
|> Map.put("id", activity.data["object"]["id"])
|> Map.put("id", activity.data["object"])
data =
data
@ -471,7 +479,7 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do
object =
data["object"]
|> Map.put("id", activity.data["object"]["id"])
|> Map.put("id", activity.data["object"])
data =
data
@ -489,7 +497,7 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do
announce_data =
File.read!("test/fixtures/mastodon-announce.json")
|> Poison.decode!()
|> Map.put("object", activity.data["object"]["id"])
|> Map.put("object", activity.data["object"])
{:ok, %Activity{data: announce_data, local: false}} =
Transmogrifier.handle_incoming(announce_data)
@ -504,7 +512,7 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do
assert data["type"] == "Undo"
assert data["object"]["type"] == "Announce"
assert data["object"]["object"] == activity.data["object"]["id"]
assert data["object"]["object"] == activity.data["object"]
assert data["object"]["id"] ==
"http://mastodon.example.org/users/admin/statuses/99542391527669785/activity"
@ -783,7 +791,7 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do
test "it remaps video URLs as attachments if necessary" do
{:ok, object} =
ActivityPub.fetch_object_from_id(
Fetcher.fetch_object_from_id(
"https://peertube.moe/videos/watch/df5f464b-be8d-46fb-ad81-2d4c2d1630e3"
)
@ -1088,10 +1096,6 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do
end
describe "actor origin containment" do
test "it rejects objects with a bogus origin" do
{:error, _} = ActivityPub.fetch_object_from_id("https://info.pleroma.site/activity.json")
end
test "it rejects activities which reference objects with bogus origins" do
data = %{
"@context" => "https://www.w3.org/ns/activitystreams",
@ -1105,10 +1109,6 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do
:error = Transmogrifier.handle_incoming(data)
end
test "it rejects objects when attributedTo is wrong (variant 1)" do
{:error, _} = ActivityPub.fetch_object_from_id("https://info.pleroma.site/activity2.json")
end
test "it rejects activities which reference objects that have an incorrect attribution (variant 1)" do
data = %{
"@context" => "https://www.w3.org/ns/activitystreams",
@ -1122,10 +1122,6 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do
:error = Transmogrifier.handle_incoming(data)
end
test "it rejects objects when attributedTo is wrong (variant 2)" do
{:error, _} = ActivityPub.fetch_object_from_id("https://info.pleroma.site/activity3.json")
end
test "it rejects activities which reference objects that have an incorrect attribution (variant 2)" do
data = %{
"@context" => "https://www.w3.org/ns/activitystreams",
@ -1140,62 +1136,6 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do
end
end
describe "general origin containment" do
test "contain_origin_from_id() catches obvious spoofing attempts" do
data = %{
"id" => "http://example.com/~alyssa/activities/1234.json"
}
:error =
Transmogrifier.contain_origin_from_id(
"http://example.org/~alyssa/activities/1234.json",
data
)
end
test "contain_origin_from_id() allows alternate IDs within the same origin domain" do
data = %{
"id" => "http://example.com/~alyssa/activities/1234.json"
}
:ok =
Transmogrifier.contain_origin_from_id(
"http://example.com/~alyssa/activities/1234",
data
)
end
test "contain_origin_from_id() allows matching IDs" do
data = %{
"id" => "http://example.com/~alyssa/activities/1234.json"
}
:ok =
Transmogrifier.contain_origin_from_id(
"http://example.com/~alyssa/activities/1234.json",
data
)
end
test "users cannot be collided through fake direction spoofing attempts" do
insert(:user, %{
nickname: "rye@niu.moe",
local: false,
ap_id: "https://niu.moe/users/rye",
follower_address: User.ap_followers(%User{nickname: "rye@niu.moe"})
})
{:error, _} = User.get_or_fetch_by_ap_id("https://n1u.moe/users/rye")
end
test "all objects with fake directions are rejected by the object fetcher" do
{:error, _} =
ActivityPub.fetch_and_contain_remote_object_from_id(
"https://info.pleroma.site/activity4.json"
)
end
end
describe "reserialization" do
test "successfully reserializes a message with inReplyTo == nil" do
user = insert(:user)