From d6d5f46baea90e9b8a305a010457ac6d404d8829 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 11 Apr 2019 16:02:38 +0000 Subject: [PATCH 001/193] Update OAuth web template --- lib/pleroma/web/templates/layout/app.html.eex | 45 ++++++++++++--- .../web/templates/o_auth/o_auth/show.html.eex | 57 ++++++++++++++----- 2 files changed, 79 insertions(+), 23 deletions(-) diff --git a/lib/pleroma/web/templates/layout/app.html.eex b/lib/pleroma/web/templates/layout/app.html.eex index 8333bc921..7d2d609d1 100644 --- a/lib/pleroma/web/templates/layout/app.html.eex +++ b/lib/pleroma/web/templates/layout/app.html.eex @@ -63,13 +63,14 @@ .scopes-input { display: flex; + flex-direction: column; margin-top: 1em; text-align: left; color: #89898a; } .scopes-input label:first-child { - flex-basis: 40%; + height: 2em; } .scopes { @@ -80,13 +81,22 @@ } .scope { - flex-basis: 100%; display: flex; + flex-basis: 100%; height: 2em; align-items: center; } + .scope:before { + color: #b9b9ba; + content: "✔"; + margin-left: 1em; + margin-right: 1em; + } + [type="checkbox"] + label { + display: none; + cursor: pointer; margin: 0.5em; } @@ -95,10 +105,12 @@ } [type="checkbox"] + label:before { + cursor: pointer; display: inline-block; color: white; background-color: #121a24; border: 4px solid #121a24; + box-shadow: 0px 0px 1px 0 #d8a070; box-sizing: border-box; width: 1.2em; height: 1.2em; @@ -128,7 +140,8 @@ border-radius: 4px; border: none; padding: 10px; - margin-top: 30px; + margin-top: 20px; + margin-bottom: 20px; text-transform: uppercase; font-size: 16px; box-shadow: 0px 0px 2px 0px black, @@ -147,8 +160,9 @@ box-sizing: border-box; width: 100%; background-color: #931014; + border: 1px solid #a06060; + color: #902020; border-radius: 4px; - border: none; padding: 10px; margin-top: 20px; font-weight: 500; @@ -171,12 +185,27 @@ margin-top: 0 } - .scopes-input { - flex-direction: column; + .scope { + flex-basis: 0%; } - .scope { - flex-basis: 50%; + .scope:before { + content: ""; + margin-left: 0em; + margin-right: 1em; + } + + .scope:first-child:before { + margin-left: 1em; + content: "✔"; + } + + .scope:after { + content: ","; + } + + .scope:last-child:after { + content: ""; } } diff --git a/lib/pleroma/web/templates/o_auth/o_auth/show.html.eex b/lib/pleroma/web/templates/o_auth/o_auth/show.html.eex index 87278e636..c7b4ef792 100644 --- a/lib/pleroma/web/templates/o_auth/o_auth/show.html.eex +++ b/lib/pleroma/web/templates/o_auth/o_auth/show.html.eex @@ -6,26 +6,53 @@ <% end %>

OAuth Authorization

- <%= form_for @conn, o_auth_path(@conn, :authorize), [as: "authorization"], fn f -> %> -
- <%= label f, :name, "Name or email" %> - <%= text_input f, :name %> -
-
- <%= label f, :password, "Password" %> - <%= password_input f, :password %> -
-<%= render @view_module, "_scopes.html", Map.merge(assigns, %{form: f, scope_param: "authorization[scope][]"}) %> +<%= if @params["registration"] in ["true", true] do %> +

This is the first time you visit! Please enter your Pleroma handle.

+

Choose carefully! You won't be able to change this later. You will be able to change your display name, though.

+

Please only use lowercase letters and no special characters

+
+ <%= label f, :nickname, "Pleroma Handle" %> + <%= text_input f, :nickname, placeholder: "lain" %> +
+ <%= hidden_input f, :name, value: @params["name"] %> + <%= hidden_input f, :password, value: @params["password"] %> +
+ +<% else %> +
+ <%= label f, :name, "Username" %> + <%= text_input f, :name %> +
+
+ <%= label f, :password, "Password" %> + <%= password_input f, :password %> +
+ <%= submit "Log In" %> +
+ <%= label f, :scope, "The following permissions will be granted" %> +
+ <%= for scope <- @available_scopes do %> + <%# Note: using hidden input with `unchecked_value` in order to distinguish user's empty selection from `scope` param being omitted %> + <%= if scope in @scopes do %> +
+ <%= checkbox f, :"scope_#{scope}", value: scope in @scopes && scope, checked_value: scope, unchecked_value: "", name: "authorization[scope][]" %> + <%= label f, :"scope_#{scope}", String.capitalize(scope) %> + <%= if scope in @scopes && scope do %> + <%= String.capitalize(scope) %> + <% end %> +
+ <% else %> + <%= checkbox f, :"scope_#{scope}", value: scope in @scopes && scope, checked_value: scope, unchecked_value: "", name: "authorization[scope][]" %> + <% end %> + <% end %> +
+
+<% end %> <%= hidden_input f, :client_id, value: @client_id %> <%= hidden_input f, :response_type, value: @response_type %> <%= hidden_input f, :redirect_uri, value: @redirect_uri %> <%= hidden_input f, :state, value: @state %> -<%= submit "Authorize" %> -<% end %> - -<%= if Pleroma.Config.oauth_consumer_enabled?() do %> - <%= render @view_module, Pleroma.Web.Auth.Authenticator.oauth_consumer_template(), assigns %> <% end %> From 9a98f48ec3f438e543a5a621624401bb22a0d44a Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 1 May 2019 12:29:48 -0500 Subject: [PATCH 002/193] Remove incorrect statement about valid characters --- lib/pleroma/web/templates/o_auth/o_auth/show.html.eex | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/pleroma/web/templates/o_auth/o_auth/show.html.eex b/lib/pleroma/web/templates/o_auth/o_auth/show.html.eex index c7b4ef792..45f2b5cc0 100644 --- a/lib/pleroma/web/templates/o_auth/o_auth/show.html.eex +++ b/lib/pleroma/web/templates/o_auth/o_auth/show.html.eex @@ -11,7 +11,6 @@ <%= if @params["registration"] in ["true", true] do %>

This is the first time you visit! Please enter your Pleroma handle.

Choose carefully! You won't be able to change this later. You will be able to change your display name, though.

-

Please only use lowercase letters and no special characters

<%= label f, :nickname, "Pleroma Handle" %> <%= text_input f, :nickname, placeholder: "lain" %> From 62e42b03abd2cede85e85f62c35f62a8c42e8ea1 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Wed, 15 May 2019 20:10:16 +0300 Subject: [PATCH 003/193] Handle incoming Question objects --- .../web/activity_pub/transmogrifier.ex | 2 +- lib/pleroma/web/activity_pub/utils.ex | 2 +- test/fixtures/httpoison_mock/rinpatch.json | 64 ++++++++++++ test/fixtures/mastodon-question-activity.json | 99 +++++++++++++++++++ test/support/http_request_mock.ex | 8 ++ test/web/activity_pub/transmogrifier_test.exs | 17 ++++ 6 files changed, 190 insertions(+), 2 deletions(-) create mode 100644 test/fixtures/httpoison_mock/rinpatch.json create mode 100644 test/fixtures/mastodon-question-activity.json diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 508f3532f..c2596cfec 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -399,7 +399,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do # - tags # - emoji def handle_incoming(%{"type" => "Create", "object" => %{"type" => objtype} = object} = data) - when objtype in ["Article", "Note", "Video", "Page"] do + when objtype in ["Article", "Note", "Video", "Page", "Question"] do actor = Containment.get_actor(data) data = diff --git a/lib/pleroma/web/activity_pub/utils.ex b/lib/pleroma/web/activity_pub/utils.ex index 581b9d1ab..de91fa03f 100644 --- a/lib/pleroma/web/activity_pub/utils.ex +++ b/lib/pleroma/web/activity_pub/utils.ex @@ -19,7 +19,7 @@ defmodule Pleroma.Web.ActivityPub.Utils do require Logger - @supported_object_types ["Article", "Note", "Video", "Page"] + @supported_object_types ["Article", "Note", "Video", "Page", "Question"] # Some implementations send the actor URI as the actor field, others send the entire actor object, # so figure out what the actor's URI is based on what we have. diff --git a/test/fixtures/httpoison_mock/rinpatch.json b/test/fixtures/httpoison_mock/rinpatch.json new file mode 100644 index 000000000..59311ecb6 --- /dev/null +++ b/test/fixtures/httpoison_mock/rinpatch.json @@ -0,0 +1,64 @@ +{ + "@context": [ + "https://www.w3.org/ns/activitystreams", + "https://w3id.org/security/v1", + { + "manuallyApprovesFollowers": "as:manuallyApprovesFollowers", + "toot": "http://joinmastodon.org/ns#", + "featured": { + "@id": "toot:featured", + "@type": "@id" + }, + "alsoKnownAs": { + "@id": "as:alsoKnownAs", + "@type": "@id" + }, + "movedTo": { + "@id": "as:movedTo", + "@type": "@id" + }, + "schema": "http://schema.org#", + "PropertyValue": "schema:PropertyValue", + "value": "schema:value", + "Hashtag": "as:Hashtag", + "Emoji": "toot:Emoji", + "IdentityProof": "toot:IdentityProof", + "focalPoint": { + "@container": "@list", + "@id": "toot:focalPoint" + } + } + ], + "id": "https://mastodon.sdf.org/users/rinpatch", + "type": "Person", + "following": "https://mastodon.sdf.org/users/rinpatch/following", + "followers": "https://mastodon.sdf.org/users/rinpatch/followers", + "inbox": "https://mastodon.sdf.org/users/rinpatch/inbox", + "outbox": "https://mastodon.sdf.org/users/rinpatch/outbox", + "featured": "https://mastodon.sdf.org/users/rinpatch/collections/featured", + "preferredUsername": "rinpatch", + "name": "rinpatch", + "summary": "

umu

", + "url": "https://mastodon.sdf.org/@rinpatch", + "manuallyApprovesFollowers": false, + "publicKey": { + "id": "https://mastodon.sdf.org/users/rinpatch#main-key", + "owner": "https://mastodon.sdf.org/users/rinpatch", + "publicKeyPem": "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA1vbhYKDopb5xzfJB2TZY\n0ZvgxqdAhbSKKkQC5Q2b0ofhvueDy2AuZTnVk1/BbHNlqKlwhJUSpA6LiTZVvtcc\nMn6cmSaJJEg30gRF5GARP8FMcuq8e2jmceiW99NnUX17MQXsddSf2JFUwD0rUE8H\nBsgD7UzE9+zlA/PJOTBO7fvBEz9PTQ3r4sRMTJVFvKz2MU/U+aRNTuexRKMMPnUw\nfp6VWh1F44VWJEQOs4tOEjGiQiMQh5OfBk1w2haT3vrDbQvq23tNpUP1cRomLUtx\nEBcGKi5DMMBzE1RTVT1YUykR/zLWlA+JSmw7P6cWtsHYZovs8dgn8Po3X//6N+ng\nTQIDAQAB\n-----END PUBLIC KEY-----\n" + }, + "tag": [], + "attachment": [], + "endpoints": { + "sharedInbox": "https://mastodon.sdf.org/inbox" + }, + "icon": { + "type": "Image", + "mediaType": "image/jpeg", + "url": "https://mastodon.sdf.org/system/accounts/avatars/000/067/580/original/bf05521bf711b7a0.jpg?1533238802" + }, + "image": { + "type": "Image", + "mediaType": "image/gif", + "url": "https://mastodon.sdf.org/system/accounts/headers/000/067/580/original/a99b987e798f7063.gif?1533278217" + } +} diff --git a/test/fixtures/mastodon-question-activity.json b/test/fixtures/mastodon-question-activity.json new file mode 100644 index 000000000..ac329c7d5 --- /dev/null +++ b/test/fixtures/mastodon-question-activity.json @@ -0,0 +1,99 @@ +{ + "@context": [ + "https://www.w3.org/ns/activitystreams", + { + "ostatus": "http://ostatus.org#", + "atomUri": "ostatus:atomUri", + "inReplyToAtomUri": "ostatus:inReplyToAtomUri", + "conversation": "ostatus:conversation", + "sensitive": "as:sensitive", + "Hashtag": "as:Hashtag", + "toot": "http://joinmastodon.org/ns#", + "Emoji": "toot:Emoji", + "focalPoint": { + "@container": "@list", + "@id": "toot:focalPoint" + } + } + ], + "id": "https://mastodon.sdf.org/users/rinpatch/statuses/102070944809637304/activity", + "type": "Create", + "actor": "https://mastodon.sdf.org/users/rinpatch", + "published": "2019-05-10T09:03:36Z", + "to": [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "cc": [ + "https://mastodon.sdf.org/users/rinpatch/followers" + ], + "object": { + "id": "https://mastodon.sdf.org/users/rinpatch/statuses/102070944809637304", + "type": "Question", + "summary": null, + "inReplyTo": null, + "published": "2019-05-10T09:03:36Z", + "url": "https://mastodon.sdf.org/@rinpatch/102070944809637304", + "attributedTo": "https://mastodon.sdf.org/users/rinpatch", + "to": [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "cc": [ + "https://mastodon.sdf.org/users/rinpatch/followers" + ], + "sensitive": false, + "atomUri": "https://mastodon.sdf.org/users/rinpatch/statuses/102070944809637304", + "inReplyToAtomUri": null, + "conversation": "tag:mastodon.sdf.org,2019-05-10:objectId=15095122:objectType=Conversation", + "content": "

Why is Tenshi eating a corndog so cute?

", + "contentMap": { + "en": "

Why is Tenshi eating a corndog so cute?

" + }, + "endTime": "2019-05-11T09:03:36Z", + "closed": "2019-05-11T09:03:36Z", + "attachment": [], + "tag": [], + "replies": { + "id": "https://mastodon.sdf.org/users/rinpatch/statuses/102070944809637304/replies", + "type": "Collection", + "first": { + "type": "CollectionPage", + "partOf": "https://mastodon.sdf.org/users/rinpatch/statuses/102070944809637304/replies", + "items": [] + } + }, + "oneOf": [ + { + "type": "Note", + "name": "Dunno", + "replies": { + "type": "Collection", + "totalItems": 0 + } + }, + { + "type": "Note", + "name": "Everyone knows that!", + "replies": { + "type": "Collection", + "totalItems": 1 + } + }, + { + "type": "Note", + "name": "25 char limit is dumb", + "replies": { + "type": "Collection", + "totalItems": 0 + } + }, + { + "type": "Note", + "name": "I can't even fit a funny", + "replies": { + "type": "Collection", + "totalItems": 1 + } + } + ] + } +} diff --git a/test/support/http_request_mock.ex b/test/support/http_request_mock.ex index 5b355bfe6..3064c032b 100644 --- a/test/support/http_request_mock.ex +++ b/test/support/http_request_mock.ex @@ -52,6 +52,14 @@ defmodule HttpRequestMock do }} end + def get("https://mastodon.sdf.org/users/rinpatch", _, _, _) do + {:ok, + %Tesla.Env{ + status: 200, + body: File.read!("test/fixtures/httpoison_mock/rinpatch.json") + }} + end + def get( "https://mastodon.social/.well-known/webfinger?resource=https://mastodon.social/users/emelie", _, diff --git a/test/web/activity_pub/transmogrifier_test.exs b/test/web/activity_pub/transmogrifier_test.exs index c24b50f8c..727abbd17 100644 --- a/test/web/activity_pub/transmogrifier_test.exs +++ b/test/web/activity_pub/transmogrifier_test.exs @@ -113,6 +113,23 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do assert Enum.at(object.data["tag"], 2) == "moo" end + test "it works for incoming questions" do + data = File.read!("test/fixtures/mastodon-question-activity.json") |> Poison.decode!() + + {:ok, %Activity{local: false} = activity} = Transmogrifier.handle_incoming(data) + + object = Object.normalize(activity) + + assert Enum.all?(object.data["oneOf"], fn choice -> + choice["name"] in [ + "Dunno", + "Everyone knows that!", + "25 char limit is dumb", + "I can't even fit a funny" + ] + end) + end + test "it works for incoming notices with contentMap" do data = File.read!("test/fixtures/mastodon-post-activity-contentmap.json") |> Poison.decode!() From 642a67dd4492f31b5b9fe457e34c1589c9d70c3f Mon Sep 17 00:00:00 2001 From: rinpatch Date: Fri, 17 May 2019 11:44:47 +0300 Subject: [PATCH 004/193] Render polls in statuses --- .../web/mastodon_api/views/status_view.ex | 52 +++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index bd2372944..6337c50e8 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -232,6 +232,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do spoiler_text: summary_html, visibility: get_visibility(object), media_attachments: attachments, + poll: render("poll.json", %{object: object, for: opts[:for]}), mentions: mentions, tags: build_tags(tags), application: %{ @@ -321,6 +322,57 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do } end + # TODO: Add tests for this view + def render("poll.json", %{object: object} = opts) do + {multiple, options} = + case object.data do + %{"anyOf" => options} when is_list(options) -> {true, options} + %{"oneOf" => options} when is_list(options) -> {false, options} + _ -> {nil, nil} + end + + if options do + end_time = + (object.data["closed"] || object.data["endTime"]) + |> NaiveDateTime.from_iso8601!() + + votes_count = object.data["votes_count"] || 0 + + expired = + end_time + |> NaiveDateTime.compare(NaiveDateTime.utc_now()) + |> case do + :lt -> true + _ -> false + end + + options = + Enum.map(options, fn %{"name" => name} = option -> + name = + HTML.filter_tags( + name, + User.html_filter_policy(opts[:for]) + ) + + %{title: name, votes_count: option["replies"]["votes_count"] || 0} + end) + + %{ + # Mastodon uses separate ids for polls, but an object can't have more than one poll embedded so object id is fine + id: object.id, + expires_at: Utils.to_masto_date(end_time), + expired: expired, + multiple: multiple, + votes_count: votes_count, + options: options, + voted: false, + emojis: build_emojis(object.data["emoji"]) + } + else + nil + end + end + def get_reply_to(activity, %{replied_to_activities: replied_to_activities}) do object = Object.normalize(activity) From f959bf7aa6b878ee5b669c4caabd5cdc4cc2dc9e Mon Sep 17 00:00:00 2001 From: lain Date: Fri, 17 May 2019 18:21:11 +0200 Subject: [PATCH 005/193] MongooseIM: Add basic integration endpoints. --- .../web/mongooseim/mongoose_im_controller.ex | 41 +++++++++++++ lib/pleroma/web/router.ex | 5 ++ .../mongoose_im_controller_test.exs | 59 +++++++++++++++++++ 3 files changed, 105 insertions(+) create mode 100644 lib/pleroma/web/mongooseim/mongoose_im_controller.ex create mode 100644 test/web/mongooseim/mongoose_im_controller_test.exs diff --git a/lib/pleroma/web/mongooseim/mongoose_im_controller.ex b/lib/pleroma/web/mongooseim/mongoose_im_controller.ex new file mode 100644 index 000000000..f8c634653 --- /dev/null +++ b/lib/pleroma/web/mongooseim/mongoose_im_controller.ex @@ -0,0 +1,41 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.MongooseIM.MongooseIMController do + use Pleroma.Web, :controller + alias Comeonin.Pbkdf2 + alias Pleroma.User + alias Pleroma.Repo + + def user_exists(conn, %{"user" => username}) do + with %User{} <- Repo.get_by(User, nickname: username, local: true) do + conn + |> json(true) + else + _ -> + conn + |> put_status(:not_found) + |> json(false) + end + end + + def check_password(conn, %{"user" => username, "pass" => password}) do + with %User{password_hash: password_hash} <- + Repo.get_by(User, nickname: username, local: true), + true <- Pbkdf2.checkpw(password, password_hash) do + conn + |> json(true) + else + false -> + conn + |> put_status(403) + |> json(false) + + _ -> + conn + |> put_status(:not_found) + |> json(false) + end + end +end diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 6a4e4a1d4..552778c92 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -704,6 +704,11 @@ defmodule Pleroma.Web.Router do end end + scope "/", Pleroma.Web.MongooseIM do + get("/user_exists", MongooseIMController, :user_exists) + get("/check_password", MongooseIMController, :check_password) + end + scope "/", Fallback do get("/registration/:token", RedirectController, :registration_page) get("/:maybe_nickname_or_id", RedirectController, :redirector_with_meta) diff --git a/test/web/mongooseim/mongoose_im_controller_test.exs b/test/web/mongooseim/mongoose_im_controller_test.exs new file mode 100644 index 000000000..eb83999bb --- /dev/null +++ b/test/web/mongooseim/mongoose_im_controller_test.exs @@ -0,0 +1,59 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.MongooseIMController do + use Pleroma.Web.ConnCase + import Pleroma.Factory + + test "/user_exists", %{conn: conn} do + _user = insert(:user, nickname: "lain") + _remote_user = insert(:user, nickname: "alice", local: false) + + res = + conn + |> get(mongoose_im_path(conn, :user_exists), user: "lain") + |> json_response(200) + + assert res == true + + res = + conn + |> get(mongoose_im_path(conn, :user_exists), user: "alice") + |> json_response(404) + + assert res == false + + res = + conn + |> get(mongoose_im_path(conn, :user_exists), user: "bob") + |> json_response(404) + + assert res == false + end + + test "/check_password", %{conn: conn} do + user = insert(:user, password_hash: Comeonin.Pbkdf2.hashpwsalt("cool")) + + res = + conn + |> get(mongoose_im_path(conn, :check_password), user: user.nickname, pass: "cool") + |> json_response(200) + + assert res == true + + res = + conn + |> get(mongoose_im_path(conn, :check_password), user: user.nickname, pass: "uncool") + |> json_response(403) + + assert res == false + + res = + conn + |> get(mongoose_im_path(conn, :check_password), user: "nobody", pass: "cool") + |> json_response(404) + + assert res == false + end +end From 075eecec907b0a623a90eed44a0378a6812d8037 Mon Sep 17 00:00:00 2001 From: lain Date: Fri, 17 May 2019 18:32:30 +0200 Subject: [PATCH 006/193] Linting. --- lib/pleroma/web/mongooseim/mongoose_im_controller.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/mongooseim/mongoose_im_controller.ex b/lib/pleroma/web/mongooseim/mongoose_im_controller.ex index f8c634653..489d5d3a5 100644 --- a/lib/pleroma/web/mongooseim/mongoose_im_controller.ex +++ b/lib/pleroma/web/mongooseim/mongoose_im_controller.ex @@ -5,8 +5,8 @@ defmodule Pleroma.Web.MongooseIM.MongooseIMController do use Pleroma.Web, :controller alias Comeonin.Pbkdf2 - alias Pleroma.User alias Pleroma.Repo + alias Pleroma.User def user_exists(conn, %{"user" => username}) do with %User{} <- Repo.get_by(User, nickname: username, local: true) do From fd920c897339b9cedea042dd6698d14380cedae7 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Sat, 18 May 2019 13:29:28 +0300 Subject: [PATCH 007/193] Mastodon API: Add support for posting polls --- lib/pleroma/web/common_api/common_api.ex | 4 +- lib/pleroma/web/common_api/utils.ex | 64 ++++++++++++++++--- .../mastodon_api_controller_test.exs | 22 +++++++ test/web/mastodon_api/status_view_test.exs | 1 + 4 files changed, 82 insertions(+), 9 deletions(-) diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex index b53869c75..335ae70b0 100644 --- a/lib/pleroma/web/common_api/common_api.ex +++ b/lib/pleroma/web/common_api/common_api.ex @@ -149,6 +149,7 @@ defmodule Pleroma.Web.CommonAPI do data, visibility ), + {poll, mentions, tags} <- make_poll_data(data, mentions, tags), {to, cc} <- to_for_user_and_mentions(user, mentions, in_reply_to, visibility), context <- make_context(in_reply_to), cw <- data["spoiler_text"] || "", @@ -164,7 +165,8 @@ defmodule Pleroma.Web.CommonAPI do in_reply_to, tags, cw, - cc + cc, + poll ), object <- Map.put( diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex index 1dfe50b40..13cdffbbd 100644 --- a/lib/pleroma/web/common_api/utils.ex +++ b/lib/pleroma/web/common_api/utils.ex @@ -102,6 +102,48 @@ defmodule Pleroma.Web.CommonAPI.Utils do end end + def make_poll_data( + %{"poll" => %{"options" => options, "expires_in" => expires_in}} = data, + mentions, + tags + ) + when is_list(options) and is_integer(expires_in) do + content_type = get_content_type(data["content_type"]) + # XXX: There is probably a more performant/cleaner way to do this + {poll, {mentions, tags}} = + Enum.map_reduce(options, {mentions, tags}, fn option, {mentions, tags} -> + # TODO: Custom emoji + {option, mentions_merge, tags_merge} = format_input(option, content_type) + mentions = mentions ++ mentions_merge + tags = tags ++ tags_merge + + {%{ + "name" => option, + "type" => "Note", + "replies" => %{"type" => "Collection", "totalItems" => 0} + }, {mentions, tags}} + end) + + end_time = + NaiveDateTime.utc_now() + |> NaiveDateTime.add(expires_in) + |> NaiveDateTime.to_iso8601() + + poll = + if Pleroma.Web.ControllerHelper.truthy_param?(data["poll"]["multiple"]) do + %{"type" => "Question", "anyOf" => poll, "closed" => end_time} + else + %{"type" => "Question", "oneOf" => poll, "closed" => end_time} + end + + {poll, mentions, tags} + end + + def make_poll_data(data, mentions, tags) do + IO.inspect(data, label: "data") + {%{}, mentions, tags} + end + def make_content_html( status, attachments, @@ -223,8 +265,11 @@ defmodule Pleroma.Web.CommonAPI.Utils do in_reply_to, tags, cw \\ nil, - cc \\ [] + cc \\ [], + merge \\ %{} ) do + IO.inspect(merge, label: "merge") + object = %{ "type" => "Note", "to" => to, @@ -237,14 +282,17 @@ defmodule Pleroma.Web.CommonAPI.Utils do "tag" => tags |> Enum.map(fn {_, tag} -> tag end) |> Enum.uniq() } - if in_reply_to do - in_reply_to_object = Object.normalize(in_reply_to) + object = + if in_reply_to do + in_reply_to_object = Object.normalize(in_reply_to) - object - |> Map.put("inReplyTo", in_reply_to_object.data["id"]) - else - object - end + object + |> Map.put("inReplyTo", in_reply_to_object.data["id"]) + else + object + end + + Map.merge(object, merge) end def format_naive_asctime(date) do diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index 505e45010..ce581c092 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -132,6 +132,28 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do refute id == third_id end + test "posting a poll", %{conn: conn} do + user = insert(:user) + time = NaiveDateTime.utc_now() + + conn = + conn + |> assign(:user, user) + |> post("/api/v1/statuses", %{ + "status" => "Who is the best girl?", + "poll" => %{"options" => ["Rei", "Asuka", "Misato"], "expires_in" => 420} + }) + + response = json_response(conn, 200) + + assert Enum.all?(response["poll"]["options"], fn %{"title" => title} -> + title in ["Rei", "Asuka", "Misato"] + end) + + assert NaiveDateTime.diff(NaiveDateTime.from_iso8601!(response["poll"]["expires_at"]), time) in 420..430 + refute response["poll"]["expred"] + end + test "posting a sensitive status", %{conn: conn} do user = insert(:user) diff --git a/test/web/mastodon_api/status_view_test.exs b/test/web/mastodon_api/status_view_test.exs index d7c800e83..9f2ebda4e 100644 --- a/test/web/mastodon_api/status_view_test.exs +++ b/test/web/mastodon_api/status_view_test.exs @@ -103,6 +103,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusViewTest do muted: false, pinned: false, sensitive: false, + poll: nil, spoiler_text: HtmlSanitizeEx.basic_html(note.data["object"]["summary"]), visibility: "public", media_attachments: [], From 1d90f9b96999f8bad3fa3e3ec58bf50c8666be4f Mon Sep 17 00:00:00 2001 From: rinpatch Date: Sun, 19 May 2019 17:06:44 +0300 Subject: [PATCH 008/193] Remove tags/mentions/rich text from poll options because Mastodon and add custom emoji --- lib/pleroma/web/common_api/common_api.ex | 4 +-- lib/pleroma/web/common_api/utils.ex | 25 ++++++------------- .../web/mastodon_api/views/status_view.ex | 14 +++++------ 3 files changed, 15 insertions(+), 28 deletions(-) diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex index bc8f80389..374967a1b 100644 --- a/lib/pleroma/web/common_api/common_api.ex +++ b/lib/pleroma/web/common_api/common_api.ex @@ -154,7 +154,7 @@ defmodule Pleroma.Web.CommonAPI do data, visibility ), - {poll, mentions, tags} <- make_poll_data(data, mentions, tags), + {poll, poll_emoji} <- make_poll_data(data), {to, cc} <- to_for_user_and_mentions(user, mentions, in_reply_to, visibility), context <- make_context(in_reply_to), cw <- data["spoiler_text"] || "", @@ -179,7 +179,7 @@ defmodule Pleroma.Web.CommonAPI do Map.put( object, "emoji", - Formatter.get_emoji_map(full_payload) + Map.merge(Formatter.get_emoji_map(full_payload), poll_emoji) ) do res = ActivityPub.create( diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex index 2ea997789..cd8483c11 100644 --- a/lib/pleroma/web/common_api/utils.ex +++ b/lib/pleroma/web/common_api/utils.ex @@ -102,26 +102,15 @@ defmodule Pleroma.Web.CommonAPI.Utils do end end - def make_poll_data( - %{"poll" => %{"options" => options, "expires_in" => expires_in}} = data, - mentions, - tags - ) + def make_poll_data(%{"poll" => %{"options" => options, "expires_in" => expires_in}} = data) when is_list(options) and is_integer(expires_in) do - content_type = get_content_type(data["content_type"]) - # XXX: There is probably a more performant/cleaner way to do this - {poll, {mentions, tags}} = - Enum.map_reduce(options, {mentions, tags}, fn option, {mentions, tags} -> - # TODO: Custom emoji - {option, mentions_merge, tags_merge} = format_input(option, content_type) - mentions = mentions ++ mentions_merge - tags = tags ++ tags_merge - + {poll, emoji} = + Enum.map_reduce(options, %{}, fn option, emoji -> {%{ "name" => option, "type" => "Note", "replies" => %{"type" => "Collection", "totalItems" => 0} - }, {mentions, tags}} + }, Map.merge(emoji, Formatter.get_emoji_map(option))} end) end_time = @@ -136,11 +125,11 @@ defmodule Pleroma.Web.CommonAPI.Utils do %{"type" => "Question", "oneOf" => poll, "closed" => end_time} end - {poll, mentions, tags} + {poll, emoji} end - def make_poll_data(_data, mentions, tags) do - {%{}, mentions, tags} + def make_poll_data(_data) do + {%{}, %{}} end def make_content_html( diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index 2a5691e1f..0df8bb5c2 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -325,7 +325,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do end # TODO: Add tests for this view - def render("poll.json", %{object: object} = opts) do + def render("poll.json", %{object: object} = _opts) do {multiple, options} = case object.data do %{"anyOf" => options} when is_list(options) -> {true, options} @@ -350,13 +350,10 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do options = Enum.map(options, fn %{"name" => name} = option -> - name = - HTML.filter_tags( - name, - User.html_filter_policy(opts[:for]) - ) - - %{title: name, votes_count: option["replies"]["votes_count"] || 0} + %{ + title: HTML.strip_tags(name), + votes_count: option["replies"]["votes_count"] || 0 + } end) %{ @@ -367,6 +364,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do multiple: multiple, votes_count: votes_count, options: options, + # TODO: Actually check for a vote voted: false, emojis: build_emojis(object.data["emoji"]) } From 6430cb1bf78e7949cc023a30df7a8d1547c36524 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Sun, 19 May 2019 17:44:18 +0300 Subject: [PATCH 009/193] Restrict poll replies from fetch queries by default --- lib/pleroma/web/activity_pub/activity_pub.ex | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 5c3156978..035fb75d5 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -820,6 +820,16 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do defp restrict_muted_reblogs(query, _), do: query + defp restrict_poll_replies(query, %{"include_poll_replies" => "true"}), do: query + + defp restrict_poll_replies(query, _) do + if has_named_binding?(query, :object) do + from([activity, object: o] in query, where: fragment("?->'name' is null", o.data)) + else + query + end + end + defp maybe_preload_objects(query, %{"skip_preload" => true}), do: query defp maybe_preload_objects(query, _) do @@ -873,6 +883,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do |> restrict_pinned(opts) |> restrict_muted_reblogs(opts) |> Activity.restrict_deactivated_users() + |> restrict_poll_replies(opts) end def fetch_activities(recipients, opts \\ %{}) do From 75c7bb9289066bfaebe7d96aaa474d34ec4d5a5f Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 20 May 2019 17:18:59 -0500 Subject: [PATCH 010/193] Additional reserved usernames --- config/config.exs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/config/config.exs b/config/config.exs index 61e2648a9..387c1a5a7 100644 --- a/config/config.exs +++ b/config/config.exs @@ -369,6 +369,7 @@ config :pleroma, Pleroma.User, "activities", "api", "auth", + "check_password", "dev", "friend-requests", "inbox", @@ -389,6 +390,7 @@ config :pleroma, Pleroma.User, "status", "tag", "user-search", + "user_exists", "users", "web" ] From 76a7429befb2e9a819b653ff8328cc42a565c29d Mon Sep 17 00:00:00 2001 From: rinpatch Date: Tue, 21 May 2019 09:13:10 +0300 Subject: [PATCH 011/193] Add poll limits to /api/v1/instance and initial state --- config/config.exs | 6 ++++++ docs/config.md | 5 +++++ lib/pleroma/web/mastodon_api/mastodon_api_controller.ex | 4 +++- test/web/mastodon_api/mastodon_api_controller_test.exs | 3 ++- 4 files changed, 16 insertions(+), 2 deletions(-) diff --git a/config/config.exs b/config/config.exs index 9a10b0ff7..47d8dfb42 100644 --- a/config/config.exs +++ b/config/config.exs @@ -211,6 +211,12 @@ config :pleroma, :instance, avatar_upload_limit: 2_000_000, background_upload_limit: 4_000_000, banner_upload_limit: 4_000_000, + poll_limits: %{ + max_options: 20, + max_option_chars: 200, + min_expiration: 0, + max_expiration: 365 * 24 * 60 * 60 + }, registrations_open: true, federating: true, federation_reachability_timeout_days: 7, diff --git a/docs/config.md b/docs/config.md index 450d73fda..f9903332c 100644 --- a/docs/config.md +++ b/docs/config.md @@ -71,6 +71,11 @@ config :pleroma, Pleroma.Emails.Mailer, * `avatar_upload_limit`: File size limit of user’s profile avatars * `background_upload_limit`: File size limit of user’s profile backgrounds * `banner_upload_limit`: File size limit of user’s profile banners +* `poll_limits`: A map with poll limits for **local** polls + * `max_options`: Maximum number of options + * `max_option_chars`: Maximum number of characters per option + * `min_expiration`: Minimum expiration time (in seconds) + * `max_expiration`: Maximum expiration time (in seconds) * `registrations_open`: Enable registrations for anyone, invitations can be enabled when false. * `invites_enabled`: Enable user invitations for admins (depends on `registrations_open: false`). * `account_activation_required`: Require users to confirm their emails before signing in. diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index 1051861ff..81cc5a972 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -197,7 +197,8 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do languages: ["en"], registrations: Pleroma.Config.get([:instance, :registrations_open]), # Extra (not present in Mastodon): - max_toot_chars: Keyword.get(instance, :limit) + max_toot_chars: Keyword.get(instance, :limit), + poll_limits: Keyword.get(instance, :poll_limits) } json(conn, response) @@ -1331,6 +1332,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do max_toot_chars: limit, mascot: "/images/pleroma-fox-tan-smol.png" }, + poll_limits: Config.get([:instance, :poll_limits]), rights: %{ delete_others_notice: present?(user.info.is_moderator), admin: present?(user.info.is_admin) diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index 68fe9c1b4..48268d4f7 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -2494,7 +2494,8 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do "stats" => _, "thumbnail" => _, "languages" => _, - "registrations" => _ + "registrations" => _, + "poll_limits" => _ } = result assert email == from_config_email From 3f96b3e4b8114ec1cf924d452907b17c2aea2003 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Tue, 21 May 2019 10:54:20 +0300 Subject: [PATCH 012/193] Enforce poll limits and add error handling for MastodonAPI's post endpoint --- lib/pleroma/web/common_api/utils.ex | 71 ++++++++---- .../mastodon_api/mastodon_api_controller.ex | 43 ++++--- .../mastodon_api_controller_test.exs | 107 +++++++++++++++--- 3 files changed, 173 insertions(+), 48 deletions(-) diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex index cd8483c11..97172fd94 100644 --- a/lib/pleroma/web/common_api/utils.ex +++ b/lib/pleroma/web/common_api/utils.ex @@ -104,28 +104,61 @@ defmodule Pleroma.Web.CommonAPI.Utils do def make_poll_data(%{"poll" => %{"options" => options, "expires_in" => expires_in}} = data) when is_list(options) and is_integer(expires_in) do - {poll, emoji} = - Enum.map_reduce(options, %{}, fn option, emoji -> - {%{ - "name" => option, - "type" => "Note", - "replies" => %{"type" => "Collection", "totalItems" => 0} - }, Map.merge(emoji, Formatter.get_emoji_map(option))} - end) + %{max_expiration: max_expiration, min_expiration: min_expiration} = + limits = Pleroma.Config.get([:instance, :poll_limits]) - end_time = - NaiveDateTime.utc_now() - |> NaiveDateTime.add(expires_in) - |> NaiveDateTime.to_iso8601() - - poll = - if Pleroma.Web.ControllerHelper.truthy_param?(data["poll"]["multiple"]) do - %{"type" => "Question", "anyOf" => poll, "closed" => end_time} - else - %{"type" => "Question", "oneOf" => poll, "closed" => end_time} + # XXX: There is probably a cleaner way of doing this + try do + if Enum.count(options) > limits.max_options do + raise ArgumentError, message: "Poll can't contain more than #{limits.max_options} options" end - {poll, emoji} + {poll, emoji} = + Enum.map_reduce(options, %{}, fn option, emoji -> + if String.length(option) > limits.max_option_chars do + raise ArgumentError, + message: + "Poll options cannot be longer than #{limits.max_option_chars} characters each" + end + + {%{ + "name" => option, + "type" => "Note", + "replies" => %{"type" => "Collection", "totalItems" => 0} + }, Map.merge(emoji, Formatter.get_emoji_map(option))} + end) + + case expires_in do + expires_in when expires_in > max_expiration -> + raise ArgumentError, message: "Expiration date is too far in the future" + + expires_in when expires_in < min_expiration -> + raise ArgumentError, message: "Expiration date is too soon" + + _ -> + :noop + end + + end_time = + NaiveDateTime.utc_now() + |> NaiveDateTime.add(expires_in) + |> NaiveDateTime.to_iso8601() + + poll = + if Pleroma.Web.ControllerHelper.truthy_param?(data["poll"]["multiple"]) do + %{"type" => "Question", "anyOf" => poll, "closed" => end_time} + else + %{"type" => "Question", "oneOf" => poll, "closed" => end_time} + end + + {poll, emoji} + rescue + e in ArgumentError -> e.message + end + end + + def make_poll_data(%{"poll" => _}) do + "Invalid poll" end def make_poll_data(_data) do diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index 81cc5a972..e2cab86f1 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -473,12 +473,6 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do params |> Map.put("in_reply_to_status_id", params["in_reply_to_id"]) - idempotency_key = - case get_req_header(conn, "idempotency-key") do - [key] -> key - _ -> Ecto.UUID.generate() - end - scheduled_at = params["scheduled_at"] if scheduled_at && ScheduledActivity.far_enough?(scheduled_at) do @@ -491,17 +485,40 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do else params = Map.drop(params, ["scheduled_at"]) - {:ok, activity} = - Cachex.fetch!(:idempotency_cache, idempotency_key, fn _ -> - CommonAPI.post(user, params) - end) + case get_cached_status_or_post(conn, params) do + {:ignore, message} -> + conn + |> put_status(401) + |> json(%{error: message}) - conn - |> put_view(StatusView) - |> try_render("status.json", %{activity: activity, for: user, as: :activity}) + {:error, message} -> + conn + |> put_status(401) + |> json(%{error: message}) + + {_, activity} -> + conn + |> put_view(StatusView) + |> try_render("status.json", %{activity: activity, for: user, as: :activity}) + end end end + defp get_cached_status_or_post(%{assigns: %{user: user}} = conn, params) do + idempotency_key = + case get_req_header(conn, "idempotency-key") do + [key] -> key + _ -> Ecto.UUID.generate() + end + + Cachex.fetch(:idempotency_cache, idempotency_key, fn _ -> + case CommonAPI.post(user, params) do + {:ok, activity} -> activity + {:error, message} -> {:ignore, message} + end + end) + end + def delete_status(%{assigns: %{user: user}} = conn, %{"id" => id}) do with {:ok, %Activity{}} <- CommonAPI.delete(id, user) do json(conn, %{}) diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index 48268d4f7..e1df79ffb 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -146,26 +146,101 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do refute id == third_id end - test "posting a poll", %{conn: conn} do - user = insert(:user) - time = NaiveDateTime.utc_now() + describe "posting polls" do + test "posting a poll", %{conn: conn} do + user = insert(:user) + time = NaiveDateTime.utc_now() - conn = - conn - |> assign(:user, user) - |> post("/api/v1/statuses", %{ - "status" => "Who is the best girl?", - "poll" => %{"options" => ["Rei", "Asuka", "Misato"], "expires_in" => 420} - }) + conn = + conn + |> assign(:user, user) + |> post("/api/v1/statuses", %{ + "status" => "Who is the #bestgrill?", + "poll" => %{"options" => ["Rei", "Asuka", "Misato"], "expires_in" => 420} + }) - response = json_response(conn, 200) + response = json_response(conn, 200) - assert Enum.all?(response["poll"]["options"], fn %{"title" => title} -> - title in ["Rei", "Asuka", "Misato"] - end) + assert Enum.all?(response["poll"]["options"], fn %{"title" => title} -> + title in ["Rei", "Asuka", "Misato"] + end) - assert NaiveDateTime.diff(NaiveDateTime.from_iso8601!(response["poll"]["expires_at"]), time) in 420..430 - refute response["poll"]["expred"] + assert NaiveDateTime.diff(NaiveDateTime.from_iso8601!(response["poll"]["expires_at"]), time) in 420..430 + refute response["poll"]["expred"] + end + + test "option limit is enforced", %{conn: conn} do + user = insert(:user) + limit = Pleroma.Config.get([:instance, :poll_limits, :max_options]) + + conn = + conn + |> assign(:user, user) + |> post("/api/v1/statuses", %{ + "status" => "desu~", + "poll" => %{"options" => Enum.map(0..limit, fn _ -> "desu" end), "expires_in" => 1} + }) + + %{"error" => error} = json_response(conn, 401) + assert error == "Poll can't contain more than #{limit} options" + end + + test "option character limit is enforced", %{conn: conn} do + user = insert(:user) + limit = Pleroma.Config.get([:instance, :poll_limits, :max_option_chars]) + + conn = + conn + |> assign(:user, user) + |> post("/api/v1/statuses", %{ + "status" => "...", + "poll" => %{ + "options" => [Enum.reduce(0..limit, "", fn _, acc -> acc <> "." end)], + "expires_in" => 1 + } + }) + + %{"error" => error} = json_response(conn, 401) + assert error == "Poll options cannot be longer than #{limit} characters each" + end + + test "minimal date limit is enforced", %{conn: conn} do + user = insert(:user) + limit = Pleroma.Config.get([:instance, :poll_limits, :min_expiration]) + + conn = + conn + |> assign(:user, user) + |> post("/api/v1/statuses", %{ + "status" => "imagine arbitrary limits", + "poll" => %{ + "options" => ["this post was made by pleroma gang"], + "expires_in" => limit - 1 + } + }) + + %{"error" => error} = json_response(conn, 401) + assert error == "Expiration date is too soon" + end + + test "maximum date limit is enforced", %{conn: conn} do + user = insert(:user) + limit = Pleroma.Config.get([:instance, :poll_limits, :max_expiration]) + + conn = + conn + |> assign(:user, user) + |> post("/api/v1/statuses", %{ + "status" => "imagine arbitrary limits", + "poll" => %{ + "options" => ["this post was made by pleroma gang"], + "expires_in" => limit + 1 + } + }) + + %{"error" => error} = json_response(conn, 401) + assert error == "Expiration date is too far in the future" + end end test "posting a sensitive status", %{conn: conn} do From aafe30d94e68ccf251c56139d07bda154dde3af9 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Tue, 21 May 2019 14:12:10 +0300 Subject: [PATCH 013/193] Handle poll votes --- lib/pleroma/object.ex | 30 ++++++++++++++++++ lib/pleroma/web/activity_pub/activity_pub.ex | 10 ++++++ test/fixtures/mastodon-vote.json | 16 ++++++++++ test/web/activity_pub/transmogrifier_test.exs | 31 +++++++++++++++++++ 4 files changed, 87 insertions(+) create mode 100644 test/fixtures/mastodon-vote.json diff --git a/lib/pleroma/object.ex b/lib/pleroma/object.ex index 740d687a3..a0f7659eb 100644 --- a/lib/pleroma/object.ex +++ b/lib/pleroma/object.ex @@ -188,4 +188,34 @@ defmodule Pleroma.Object do _ -> {:error, "Not found"} end end + + def increase_vote_count(ap_id, name) do + with %Object{} = object <- Object.normalize(ap_id), + "Question" <- object.data["type"] do + multiple = Map.has_key?(object.data, "anyOf") + + options = + (object.data["anyOf"] || object.data["oneOf"] || []) + |> Enum.map(fn + %{"name" => ^name} = option -> + Kernel.update_in(option["replies"]["totalItems"], &(&1 + 1)) + + option -> + option + end) + + data = + if multiple do + Map.put(object.data, "anyOf", options) + else + Map.put(object.data, "oneOf", options) + end + + object + |> Object.change(%{data: data}) + |> update_and_set_cache() + else + _ -> :noop + end + end end diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 035fb75d5..ce78d895b 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -108,6 +108,15 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do def decrease_replies_count_if_reply(_object), do: :noop + def increase_poll_votes_if_vote(%{ + "object" => %{"inReplyTo" => reply_ap_id, "name" => name}, + "type" => "Create" + }) do + Object.increase_vote_count(reply_ap_id, name) + end + + def increase_poll_votes_if_vote(_create_data), do: :noop + def insert(map, local \\ true, fake \\ false) when is_map(map) do with nil <- Activity.normalize(map), map <- lazy_put_activity_defaults(map, fake), @@ -235,6 +244,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do {:ok, activity} <- insert(create_data, local, fake), {:fake, false, activity} <- {:fake, fake, activity}, _ <- increase_replies_count_if_reply(create_data), + _ <- increase_poll_votes_if_vote(create_data), # Changing note count prior to enqueuing federation task in order to avoid # race conditions on updating user.info {:ok, _actor} <- increase_note_count_if_public(actor, activity), diff --git a/test/fixtures/mastodon-vote.json b/test/fixtures/mastodon-vote.json new file mode 100644 index 000000000..c2c5f40c0 --- /dev/null +++ b/test/fixtures/mastodon-vote.json @@ -0,0 +1,16 @@ +{ + "@context": "https://www.w3.org/ns/activitystreams", + "actor": "https://mastodon.sdf.org/users/rinpatch", + "id": "https://mastodon.sdf.org/users/rinpatch#votes/387/activity", + "nickname": "rin", + "object": { + "attributedTo": "https://mastodon.sdf.org/users/rinpatch", + "id": "https://mastodon.sdf.org/users/rinpatch#votes/387", + "inReplyTo": "https://testing.uguu.ltd/objects/9d300947-2dcb-445d-8978-9a3b4b84fa14", + "name": "suya..", + "to": "https://testing.uguu.ltd/users/rin", + "type": "Note" + }, + "to": "https://testing.uguu.ltd/users/rin", + "type": "Create" +} diff --git a/test/web/activity_pub/transmogrifier_test.exs b/test/web/activity_pub/transmogrifier_test.exs index 727abbd17..32d94e3e9 100644 --- a/test/web/activity_pub/transmogrifier_test.exs +++ b/test/web/activity_pub/transmogrifier_test.exs @@ -130,6 +130,37 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do end) end + test "in increments vote counters on question activities" do + user = insert(:user) + + {:ok, activity} = + CommonAPI.post(user, %{ + "status" => "suya...", + "poll" => %{"options" => ["suya", "suya.", "suya.."], "expires_in" => 10} + }) + + object = Object.normalize(activity) + + data = + File.read!("test/fixtures/mastodon-vote.json") + |> Poison.decode!() + |> Kernel.put_in(["to"], user.ap_id) + |> Kernel.put_in(["object", "inReplyTo"], object.data["id"]) + |> Kernel.put_in(["object", "to"], user.ap_id) + + {:ok, %Activity{local: false}} = Transmogrifier.handle_incoming(data) + + object = Object.get_by_ap_id(object.data["id"]) + + assert Enum.any?( + object.data["oneOf"], + fn + %{"name" => "suya..", "replies" => %{"totalItems" => 1}} -> true + _ -> false + end + ) + end + test "it works for incoming notices with contentMap" do data = File.read!("test/fixtures/mastodon-post-activity-contentmap.json") |> Poison.decode!() From a53d06273080525fdda332291838b0c95ed69690 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Tue, 21 May 2019 14:19:03 +0300 Subject: [PATCH 014/193] Fix posting non-polls from mastofe --- lib/pleroma/web/common_api/utils.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex index 97172fd94..66153a105 100644 --- a/lib/pleroma/web/common_api/utils.ex +++ b/lib/pleroma/web/common_api/utils.ex @@ -157,7 +157,7 @@ defmodule Pleroma.Web.CommonAPI.Utils do end end - def make_poll_data(%{"poll" => _}) do + def make_poll_data(%{"poll" => poll}) when is_map(poll) do "Invalid poll" end From f28747858bf3265e8b82eb587919f5a89386bed7 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Tue, 21 May 2019 14:27:09 +0300 Subject: [PATCH 015/193] Actual vote count in poll view --- .../web/mastodon_api/views/status_view.ex | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index 0df8bb5c2..c501c213c 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -338,8 +338,6 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do (object.data["closed"] || object.data["endTime"]) |> NaiveDateTime.from_iso8601!() - votes_count = object.data["votes_count"] || 0 - expired = end_time |> NaiveDateTime.compare(NaiveDateTime.utc_now()) @@ -348,12 +346,14 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do _ -> false end - options = - Enum.map(options, fn %{"name" => name} = option -> - %{ - title: HTML.strip_tags(name), - votes_count: option["replies"]["votes_count"] || 0 - } + {options, votes_count} = + Enum.map_reduce(options, 0, fn %{"name" => name} = option, count -> + current_count = option["replies"]["totalItems"] || 0 + + {%{ + title: HTML.strip_tags(name), + votes_count: current_count + }, current_count + count} end) %{ From d7c4d029c82c833278b3640d585235dc704f30d1 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Tue, 21 May 2019 14:35:20 +0300 Subject: [PATCH 016/193] Restrict poll replies when fetching activiites for context --- lib/pleroma/web/activity_pub/activity_pub.ex | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index ce78d895b..c04e6c2b8 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -491,6 +491,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do from(activity in Activity) |> restrict_blocked(opts) + |> restrict_poll_replies(opts) |> restrict_recipients(recipients, opts["user"]) |> where( [activity], From ee682441415d7abe4acb2643e1d76fe8d78e80c1 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Tue, 21 May 2019 16:58:15 +0300 Subject: [PATCH 017/193] Do not stream out poll replies --- lib/pleroma/web/activity_pub/activity_pub.ex | 54 ++++++++++---------- 1 file changed, 28 insertions(+), 26 deletions(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index c04e6c2b8..fdebf1f6b 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -192,40 +192,42 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do public = "https://www.w3.org/ns/activitystreams#Public" if activity.data["type"] in ["Create", "Announce", "Delete"] do - Pleroma.Web.Streamer.stream("user", activity) - Pleroma.Web.Streamer.stream("list", activity) + object = Object.normalize(activity) + # Do not stream out poll replies + unless object.data["name"] do + Pleroma.Web.Streamer.stream("user", activity) + Pleroma.Web.Streamer.stream("list", activity) - if Enum.member?(activity.data["to"], public) do - Pleroma.Web.Streamer.stream("public", activity) + if Enum.member?(activity.data["to"], public) do + Pleroma.Web.Streamer.stream("public", activity) - if activity.local do - Pleroma.Web.Streamer.stream("public:local", activity) - end + if activity.local do + Pleroma.Web.Streamer.stream("public:local", activity) + end - if activity.data["type"] in ["Create"] do - object = Object.normalize(activity) + if activity.data["type"] in ["Create"] do + object.data + |> Map.get("tag", []) + |> Enum.filter(fn tag -> is_bitstring(tag) end) + |> Enum.each(fn tag -> Pleroma.Web.Streamer.stream("hashtag:" <> tag, activity) end) - object.data - |> Map.get("tag", []) - |> Enum.filter(fn tag -> is_bitstring(tag) end) - |> Enum.each(fn tag -> Pleroma.Web.Streamer.stream("hashtag:" <> tag, activity) end) + if object.data["attachment"] != [] do + Pleroma.Web.Streamer.stream("public:media", activity) - if object.data["attachment"] != [] do - Pleroma.Web.Streamer.stream("public:media", activity) - - if activity.local do - Pleroma.Web.Streamer.stream("public:local:media", activity) + if activity.local do + Pleroma.Web.Streamer.stream("public:local:media", activity) + end end end + else + # TODO: Write test, replace with visibility test + if !Enum.member?(activity.data["cc"] || [], public) && + !Enum.member?( + activity.data["to"], + User.get_cached_by_ap_id(activity.data["actor"]).follower_address + ), + do: Pleroma.Web.Streamer.stream("direct", activity) end - else - # TODO: Write test, replace with visibility test - if !Enum.member?(activity.data["cc"] || [], public) && - !Enum.member?( - activity.data["to"], - User.get_cached_by_ap_id(activity.data["actor"]).follower_address - ), - do: Pleroma.Web.Streamer.stream("direct", activity) end end end From 0407ffe75f7e91db240d491492eadf1385b1726b Mon Sep 17 00:00:00 2001 From: rinpatch Date: Tue, 21 May 2019 17:12:38 +0300 Subject: [PATCH 018/193] Change validation error status codes to be more appropriate --- lib/pleroma/web/mastodon_api/mastodon_api_controller.ex | 4 ++-- test/web/mastodon_api/mastodon_api_controller_test.exs | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index e2cab86f1..aef2abf0b 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -488,12 +488,12 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do case get_cached_status_or_post(conn, params) do {:ignore, message} -> conn - |> put_status(401) + |> put_status(422) |> json(%{error: message}) {:error, message} -> conn - |> put_status(401) + |> put_status(422) |> json(%{error: message}) {_, activity} -> diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index e1df79ffb..4f332f83c 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -181,7 +181,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do "poll" => %{"options" => Enum.map(0..limit, fn _ -> "desu" end), "expires_in" => 1} }) - %{"error" => error} = json_response(conn, 401) + %{"error" => error} = json_response(conn, 422) assert error == "Poll can't contain more than #{limit} options" end @@ -200,7 +200,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do } }) - %{"error" => error} = json_response(conn, 401) + %{"error" => error} = json_response(conn, 422) assert error == "Poll options cannot be longer than #{limit} characters each" end @@ -219,7 +219,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do } }) - %{"error" => error} = json_response(conn, 401) + %{"error" => error} = json_response(conn, 422) assert error == "Expiration date is too soon" end @@ -238,7 +238,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do } }) - %{"error" => error} = json_response(conn, 401) + %{"error" => error} = json_response(conn, 422) assert error == "Expiration date is too far in the future" end end From 5f67c26baf2bdc98480fa4cda5895f33351b13ab Mon Sep 17 00:00:00 2001 From: rinpatch Date: Tue, 21 May 2019 17:30:51 +0300 Subject: [PATCH 019/193] Accept strings in expires_in because sasuga javascript --- lib/pleroma/web/common_api/utils.ex | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex index 66153a105..1a239de97 100644 --- a/lib/pleroma/web/common_api/utils.ex +++ b/lib/pleroma/web/common_api/utils.ex @@ -103,12 +103,15 @@ defmodule Pleroma.Web.CommonAPI.Utils do end def make_poll_data(%{"poll" => %{"options" => options, "expires_in" => expires_in}} = data) - when is_list(options) and is_integer(expires_in) do + when is_list(options) do %{max_expiration: max_expiration, min_expiration: min_expiration} = limits = Pleroma.Config.get([:instance, :poll_limits]) # XXX: There is probably a cleaner way of doing this try do + # In some cases mastofe sends out strings instead of integers + expires_in = if is_binary(expires_in), do: String.to_integer(expires_in), else: expires_in + if Enum.count(options) > limits.max_options do raise ArgumentError, message: "Poll can't contain more than #{limits.max_options} options" end From ff61d345020b9ee79b243fb8d23a37a06cc41b8e Mon Sep 17 00:00:00 2001 From: rinpatch Date: Tue, 21 May 2019 17:33:54 +0300 Subject: [PATCH 020/193] Accept question objects for conversations --- lib/pleroma/conversation.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/conversation.ex b/lib/pleroma/conversation.ex index 238c1acf2..bc97b39ca 100644 --- a/lib/pleroma/conversation.ex +++ b/lib/pleroma/conversation.ex @@ -49,7 +49,7 @@ defmodule Pleroma.Conversation do with true <- Pleroma.Web.ActivityPub.Visibility.is_direct?(activity), "Create" <- activity.data["type"], object <- Pleroma.Object.normalize(activity), - "Note" <- object.data["type"], + true <- object.data["type"] in ["Note", "Question"], ap_id when is_binary(ap_id) and byte_size(ap_id) > 0 <- object.data["context"] do {:ok, conversation} = create_for_ap_id(ap_id) From d378b342ba0d7f54be3b47f031c602a578191dfd Mon Sep 17 00:00:00 2001 From: lain Date: Tue, 21 May 2019 18:57:36 +0200 Subject: [PATCH 021/193] MongooseIM: Add documentation. --- CHANGELOG.md | 1 + docs/config/howto_mongooseim.md | 10 ++++++++++ 2 files changed, 11 insertions(+) create mode 100644 docs/config/howto_mongooseim.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 12c439135..b50ca895c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## [unreleased] ### Added +- [MongooseIM](https://github.com/esl/MongooseIM) http authentication support. - LDAP authentication - External OAuth provider authentication - A [job queue](https://git.pleroma.social/pleroma/pleroma_job_queue) for federation, emails, web push, etc. diff --git a/docs/config/howto_mongooseim.md b/docs/config/howto_mongooseim.md new file mode 100644 index 000000000..a33e590a1 --- /dev/null +++ b/docs/config/howto_mongooseim.md @@ -0,0 +1,10 @@ +# Configuring MongooseIM (XMPP Server) to use Pleroma for authentication + +If you want to give your Pleroma users an XMPP (chat) account, you can configure [MongooseIM](https://github.com/esl/MongooseIM) to use your Pleroma server for user authentication, automatically giving every local user an XMPP account. + +In general, you just have to follow the configuration described at [https://mongooseim.readthedocs.io/en/latest/authentication-backends/HTTP-authentication-module/](https://mongooseim.readthedocs.io/en/latest/authentication-backends/HTTP-authentication-module/) and do these changes to your mongooseim.cfg. + +1. Set the auth_method to `{auth_method, http}`. +2. Add the http auth pool like this: `{http, global, auth, [{workers, 50}], [{server, "https://yourpleromainstance.com"}]}` + +Restart your MongooseIM server, your users should now be able to connect with their Pleroma credentials. From 63b0b7190cb652cd27d236e3f9daaaf5e50701a6 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Tue, 21 May 2019 20:40:35 +0300 Subject: [PATCH 022/193] MastoAPI: Add GET /api/v1/polls/:id --- lib/pleroma/object.ex | 3 ++ .../mastodon_api/mastodon_api_controller.ex | 20 +++++++++ lib/pleroma/web/router.ex | 2 + .../mastodon_api_controller_test.exs | 44 +++++++++++++++++++ 4 files changed, 69 insertions(+) diff --git a/lib/pleroma/object.ex b/lib/pleroma/object.ex index a0f7659eb..3ffa290eb 100644 --- a/lib/pleroma/object.ex +++ b/lib/pleroma/object.ex @@ -35,6 +35,9 @@ defmodule Pleroma.Object do |> unique_constraint(:ap_id, name: :objects_unique_apid_index) end + def get_by_id(nil), do: nil + def get_by_id(id), do: Repo.get(Object, id) + def get_by_ap_id(nil), do: nil def get_by_ap_id(ap_id) do diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index aef2abf0b..ecb7df459 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -410,6 +410,26 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do end end + def get_poll(%{assigns: %{user: user}} = conn, %{"id" => id}) do + with %Object{} = object <- Object.get_by_id(id), + %Activity{} = activity <- Activity.get_create_by_object_ap_id(object.data["id"]), + true <- Visibility.visible_for_user?(activity, user) do + conn + |> put_view(StatusView) + |> try_render("poll.json", %{object: object, for: user}) + else + nil -> + conn + |> put_status(404) + |> json(%{error: "Record not found"}) + + false -> + conn + |> put_status(404) + |> json(%{error: "Record not found"}) + end + end + def scheduled_statuses(%{assigns: %{user: user}} = conn, params) do with scheduled_activities <- MastodonAPI.get_scheduled_activities(user, params) do conn diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 6a4e4a1d4..e0611e3fc 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -423,6 +423,8 @@ defmodule Pleroma.Web.Router do get("/statuses/:id", MastodonAPIController, :get_status) get("/statuses/:id/context", MastodonAPIController, :get_context) + get("/polls/:id", MastodonAPIController, :get_poll) + get("/accounts/:id/statuses", MastodonAPIController, :user_statuses) get("/accounts/:id/followers", MastodonAPIController, :followers) get("/accounts/:id/following", MastodonAPIController, :following) diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index 4f332f83c..0d56b6ff2 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -3453,4 +3453,48 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do assert json_response(conn, 403) == %{"error" => "Rate limit exceeded."} end end + + describe "GET /api/v1/polls/:id" do + test "returns poll entity for object id", %{conn: conn} do + user = insert(:user) + + {:ok, activity} = + CommonAPI.post(user, %{ + "status" => "Pleroma does", + "poll" => %{"options" => ["what Mastodon't", "n't what Mastodoes"], "expires_in" => 20} + }) + + object = Object.normalize(activity) + + conn = + conn + |> assign(:user, user) + |> get("/api/v1/polls/#{object.id}") + + response = json_response(conn, 200) + id = object.id + assert %{"id" => ^id, "expired" => false, "multiple" => false} = response + end + + test "does not expose polls for private statuses", %{conn: conn} do + user = insert(:user) + other_user = insert(:user) + + {:ok, activity} = + CommonAPI.post(user, %{ + "status" => "Pleroma does", + "poll" => %{"options" => ["what Mastodon't", "n't what Mastodoes"], "expires_in" => 20}, + "visibility" => "private" + }) + + object = Object.normalize(activity) + + conn = + conn + |> assign(:user, other_user) + |> get("/api/v1/polls/#{object.id}") + + assert json_response(conn, 404) + end + end end From c2b0b82e6a6d40f945feacc001ad984a17e23336 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Tue, 21 May 2019 00:41:40 +0000 Subject: [PATCH 023/193] object: add Object.prune() --- lib/pleroma/object.ex | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/lib/pleroma/object.ex b/lib/pleroma/object.ex index 740d687a3..cc6fc9c5d 100644 --- a/lib/pleroma/object.ex +++ b/lib/pleroma/object.ex @@ -130,6 +130,13 @@ defmodule Pleroma.Object do end end + def prune(%Object{data: %{"id" => id}} = object) do + with {:ok, object} <- Repo.delete(object), + {:ok, true} <- Cachex.del(:object_cache, "object:#{id}") do + {:ok, object} + end + end + def set_cache(%Object{data: %{"id" => ap_id}} = object) do Cachex.put(:object_cache, "object:#{ap_id}", object) {:ok, object} From 73df9d690d5c1a9c11f0f04b8d877c0677022591 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Tue, 21 May 2019 00:41:58 +0000 Subject: [PATCH 024/193] object: fetcher: add support for reinjecting pruned objects --- lib/pleroma/object/fetcher.ex | 22 ++++++++++++++++++++-- test/object/fetcher_test.exs | 19 +++++++++++++++++++ 2 files changed, 39 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex index 8d4bcc95e..bb9388d4f 100644 --- a/lib/pleroma/object/fetcher.ex +++ b/lib/pleroma/object/fetcher.ex @@ -8,6 +8,19 @@ defmodule Pleroma.Object.Fetcher do @httpoison Application.get_env(:pleroma, :httpoison) + defp reinject_object(data) do + Logger.debug("Reinjecting object #{data["id"]}") + + with data <- Transmogrifier.fix_object(data), + {:ok, object} <- Object.create(data) do + {:ok, object} + else + e -> + Logger.error("Error while processing object: #{inspect(e)}") + {:error, e} + end + end + # TODO: # This will create a Create activity, which we need internally at the moment. def fetch_object_from_id(id) do @@ -26,12 +39,17 @@ defmodule Pleroma.Object.Fetcher do "object" => data }, :ok <- Containment.contain_origin(id, params), - {:ok, activity} <- Transmogrifier.handle_incoming(params) do - {:ok, Object.normalize(activity, false)} + {:ok, activity} <- Transmogrifier.handle_incoming(params), + {:object, _data, %Object{} = object} <- + {:object, data, Object.normalize(activity, false)} do + {:ok, object} else {:error, {:reject, nil}} -> {:reject, nil} + {:object, data, nil} -> + reinject_object(data) + object = %Object{} -> {:ok, object} diff --git a/test/object/fetcher_test.exs b/test/object/fetcher_test.exs index 72f616782..d604fd5f5 100644 --- a/test/object/fetcher_test.exs +++ b/test/object/fetcher_test.exs @@ -87,4 +87,23 @@ defmodule Pleroma.Object.FetcherTest do ) end end + + describe "pruning" do + test "it can refetch pruned objects" do + object_id = "http://mastodon.example.org/@admin/99541947525187367" + + {:ok, object} = Fetcher.fetch_object_from_id(object_id) + + assert object + + {:ok, _object} = Object.prune(object) + + refute Object.get_by_ap_id(object_id) + + {:ok, %Object{} = object_two} = Fetcher.fetch_object_from_id(object_id) + + assert object.data["id"] == object_two.data["id"] + assert object.id != object_two.id + end + end end From 16b260fb19cca02463766c2e36a41bfcc823af9b Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Tue, 21 May 2019 01:21:28 +0000 Subject: [PATCH 025/193] add mix task to prune the object database using a configured retention period --- config/config.exs | 3 ++- docs/config.md | 1 + lib/mix/tasks/pleroma/database.ex | 40 +++++++++++++++++++++++++++++++ 3 files changed, 43 insertions(+), 1 deletion(-) diff --git a/config/config.exs b/config/config.exs index 72908266d..466a6e9b7 100644 --- a/config/config.exs +++ b/config/config.exs @@ -239,7 +239,8 @@ config :pleroma, :instance, welcome_message: nil, max_report_comment_size: 1000, safe_dm_mentions: false, - healthcheck: false + healthcheck: false, + remote_post_retention_days: 90 config :pleroma, :app_account_creation, enabled: true, max_requests: 25, interval: 1800 diff --git a/docs/config.md b/docs/config.md index 197326bbd..a050068f4 100644 --- a/docs/config.md +++ b/docs/config.md @@ -104,6 +104,7 @@ config :pleroma, Pleroma.Emails.Mailer, * `max_report_comment_size`: The maximum size of the report comment (Default: `1000`) * `safe_dm_mentions`: If set to true, only mentions at the beginning of a post will be used to address people in direct messages. This is to prevent accidental mentioning of people when talking about them (e.g. "@friend hey i really don't like @enemy"). (Default: `false`) * `healthcheck`: if set to true, system data will be shown on ``/api/pleroma/healthcheck``. +* `remote_post_retention_days`: the default amount of days to retain remote posts when pruning the database ## :app_account_creation REST API for creating an account settings diff --git a/lib/mix/tasks/pleroma/database.ex b/lib/mix/tasks/pleroma/database.ex index f650b447d..fdb216037 100644 --- a/lib/mix/tasks/pleroma/database.ex +++ b/lib/mix/tasks/pleroma/database.ex @@ -23,6 +23,10 @@ defmodule Mix.Tasks.Pleroma.Database do Options: - `--vacuum` - run `VACUUM FULL` after the embedded objects are replaced with their references + ## Prune old objects from the database + + mix pleroma.database prune_objects + ## Create a conversation for all existing DMs. Can be safely re-run. mix pleroma.database bump_all_conversations @@ -72,4 +76,40 @@ defmodule Mix.Tasks.Pleroma.Database do Enum.each(users, &User.remove_duplicated_following/1) Enum.each(users, &User.update_follower_count/1) end + + def run(["prune_objects" | args]) do + {options, [], []} = + OptionParser.parse( + args, + strict: [ + vacuum: :boolean + ] + ) + + Common.start_pleroma() + + deadline = Pleroma.Config.get([:instance, :remote_post_retention_days]) + + Logger.info("Pruning objects older than #{deadline} days") + + time_deadline = + NaiveDateTime.utc_now() + |> NaiveDateTime.add(-(deadline * 86_400)) + + Repo.query!( + "DELETE FROM objects WHERE inserted_at < $1 AND split_part(data->>'actor', '/', 3) != $2", + [time_deadline, Pleroma.Web.Endpoint.host()], + timeout: :infinity + ) + + if Keyword.get(options, :vacuum) do + Logger.info("Runnning VACUUM FULL") + + Repo.query!( + "vacuum full;", + [], + timeout: :infinity + ) + end + end end From f446f94290a6cdae531859c2efa55f55dbaeb7e8 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Tue, 21 May 2019 01:22:27 +0000 Subject: [PATCH 026/193] add changelog entry for object pruning --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 256df91b7..a21c4bff2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,6 +39,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Metadata: RelMe provider - OAuth: added support for refresh tokens - Emoji packs and emoji pack manager +- Object pruning (`mix pleroma.database prune_objects`) ### Changed - **Breaking:** Configuration: move from Pleroma.Mailer to Pleroma.Emails.Mailer From a13d449b241b6e5fa14fb741694e63cd21569b2c Mon Sep 17 00:00:00 2001 From: Aaron Tinio Date: Tue, 21 May 2019 09:39:32 +0800 Subject: [PATCH 027/193] Add tests for fallback routes --- test/web/fallback_test.exs | 46 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 test/web/fallback_test.exs diff --git a/test/web/fallback_test.exs b/test/web/fallback_test.exs new file mode 100644 index 000000000..514923a20 --- /dev/null +++ b/test/web/fallback_test.exs @@ -0,0 +1,46 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.FallbackTest do + use Pleroma.Web.ConnCase + import Pleroma.Factory + + test "GET /registration/:token", %{conn: conn} do + assert conn + |> get("/registration/foo") + |> html_response(200) =~ "" + end + + test "GET /:maybe_nickname_or_id", %{conn: conn} do + user = insert(:user) + + assert conn + |> get("/foo") + |> html_response(200) =~ "" + + refute conn + |> get("/" <> user.nickname) + |> html_response(200) =~ "" + end + + test "GET /*path", %{conn: conn} do + assert conn + |> get("/foo") + |> html_response(200) =~ "" + + assert conn + |> get("/foo/bar") + |> html_response(200) =~ "" + end + + test "OPTIONS /*path", %{conn: conn} do + assert conn + |> options("/foo") + |> response(204) == "" + + assert conn + |> options("/foo/bar") + |> response(204) == "" + end +end From 3ab9255eda21a5f8a25047375af9608e0c0c7592 Mon Sep 17 00:00:00 2001 From: Aaron Tinio Date: Tue, 21 May 2019 09:40:29 +0800 Subject: [PATCH 028/193] Respond with a 404 Not implemented JSON error message when requested API is not implemented --- CHANGELOG.md | 1 + lib/pleroma/web/router.ex | 7 +++++++ test/web/fallback_test.exs | 6 ++++++ 3 files changed, 14 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 256df91b7..2ed380102 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -74,6 +74,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Hide deactivated users and their statuses - Posts which are marked sensitive or tagged nsfw no longer have link previews. - HTTP connection timeout is now set to 10 seconds. +- Respond with a 404 Not implemented JSON error message when requested API is not implemented ### Fixed - Added an FTS index on objects. Running `vacuum analyze` and setting a larger `work_mem` is recommended. diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 4c29b24eb..49e28cc2d 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -710,6 +710,7 @@ defmodule Pleroma.Web.Router do scope "/", Fallback do get("/registration/:token", RedirectController, :registration_page) get("/:maybe_nickname_or_id", RedirectController, :redirector_with_meta) + get("/api*path", RedirectController, :api_not_implemented) get("/*path", RedirectController, :redirector) options("/*path", RedirectController, :empty) @@ -721,6 +722,12 @@ defmodule Fallback.RedirectController do alias Pleroma.User alias Pleroma.Web.Metadata + def api_not_implemented(conn, _params) do + conn + |> put_status(404) + |> json(%{error: "Not implemented"}) + end + def redirector(conn, _params, code \\ 200) do conn |> put_resp_content_type("text/html") diff --git a/test/web/fallback_test.exs b/test/web/fallback_test.exs index 514923a20..cc78b3ae1 100644 --- a/test/web/fallback_test.exs +++ b/test/web/fallback_test.exs @@ -24,6 +24,12 @@ defmodule Pleroma.Web.FallbackTest do |> html_response(200) =~ "" end + test "GET /api*path", %{conn: conn} do + assert conn + |> get("/api/foo") + |> json_response(404) == %{"error" => "Not implemented"} + end + test "GET /*path", %{conn: conn} do assert conn |> get("/foo") From f76268135c014c20a482d30a7c9596ec2e7d6a69 Mon Sep 17 00:00:00 2001 From: Aaron Tinio Date: Wed, 22 May 2019 07:11:09 +0800 Subject: [PATCH 029/193] Fix failing test --- test/web/admin_api/admin_api_controller_test.exs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/web/admin_api/admin_api_controller_test.exs b/test/web/admin_api/admin_api_controller_test.exs index ca12c7215..c15c67e31 100644 --- a/test/web/admin_api/admin_api_controller_test.exs +++ b/test/web/admin_api/admin_api_controller_test.exs @@ -397,14 +397,14 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do end end - test "/api/pleroma/admin/invite_token" do + test "/api/pleroma/admin/users/invite_token" do admin = insert(:user, info: %{is_admin: true}) conn = build_conn() |> assign(:user, admin) |> put_req_header("accept", "application/json") - |> get("/api/pleroma/admin/invite_token") + |> get("/api/pleroma/admin/users/invite_token") assert conn.status == 200 end From a023ca004cbd90e330cab35e4dfda16346d08668 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Wed, 22 May 2019 03:12:48 +0000 Subject: [PATCH 030/193] prune objects task: use Repo.delete_all() --- lib/mix/tasks/pleroma/database.ex | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/lib/mix/tasks/pleroma/database.ex b/lib/mix/tasks/pleroma/database.ex index fdb216037..f9bafb277 100644 --- a/lib/mix/tasks/pleroma/database.ex +++ b/lib/mix/tasks/pleroma/database.ex @@ -5,6 +5,7 @@ defmodule Mix.Tasks.Pleroma.Database do alias Mix.Tasks.Pleroma.Common alias Pleroma.Conversation + alias Pleroma.Object alias Pleroma.Repo alias Pleroma.User require Logger @@ -78,6 +79,8 @@ defmodule Mix.Tasks.Pleroma.Database do end def run(["prune_objects" | args]) do + import Ecto.Query + {options, [], []} = OptionParser.parse( args, @@ -96,11 +99,15 @@ defmodule Mix.Tasks.Pleroma.Database do NaiveDateTime.utc_now() |> NaiveDateTime.add(-(deadline * 86_400)) - Repo.query!( - "DELETE FROM objects WHERE inserted_at < $1 AND split_part(data->>'actor', '/', 3) != $2", - [time_deadline, Pleroma.Web.Endpoint.host()], - timeout: :infinity + public = "https://www.w3.org/ns/activitystreams#Public" + + from(o in Object, + where: fragment("?->'to' \\? ? OR ?->'cc' \\? ?", o.data, ^public, o.data, ^public), + where: o.inserted_at < ^time_deadline, + where: + fragment("split_part(?->>'actor', '/', 3) != ?", o.data, ^Pleroma.Web.Endpoint.host()) ) + |> Repo.delete_all() if Keyword.get(options, :vacuum) do Logger.info("Runnning VACUUM FULL") From 045803346d70c1f9c6ea770485904fd7cc52969a Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Wed, 22 May 2019 03:58:15 +0000 Subject: [PATCH 031/193] move key generation functions into Pleroma.Keys module --- lib/pleroma/keys.ex | 44 +++++++++++++++++++ lib/pleroma/signature.ex | 7 ++- lib/pleroma/user.ex | 21 +++++++++ .../activity_pub/activity_pub_controller.ex | 14 +++--- .../web/activity_pub/views/user_view.ex | 11 +++-- lib/pleroma/web/federator/federator.ex | 6 +-- lib/pleroma/web/salmon/salmon.ex | 44 ++----------------- lib/pleroma/web/web_finger/web_finger.ex | 26 +---------- test/keys_test.exs | 20 +++++++++ test/user_test.exs | 15 +++++++ test/web/activity_pub/activity_pub_test.exs | 2 +- .../web/activity_pub/views/user_view_test.exs | 13 +++--- test/web/salmon/salmon_test.exs | 19 ++------ test/web/web_finger/web_finger_test.exs | 15 ------- 14 files changed, 133 insertions(+), 124 deletions(-) create mode 100644 lib/pleroma/keys.ex create mode 100644 test/keys_test.exs diff --git a/lib/pleroma/keys.ex b/lib/pleroma/keys.ex new file mode 100644 index 000000000..b7bc7a4da --- /dev/null +++ b/lib/pleroma/keys.ex @@ -0,0 +1,44 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Keys do + # Native generation of RSA keys is only available since OTP 20+ and in default build conditions + # We try at compile time to generate natively an RSA key otherwise we fallback on the old way. + try do + _ = :public_key.generate_key({:rsa, 2048, 65_537}) + + def generate_rsa_pem do + key = :public_key.generate_key({:rsa, 2048, 65_537}) + entry = :public_key.pem_entry_encode(:RSAPrivateKey, key) + pem = :public_key.pem_encode([entry]) |> String.trim_trailing() + {:ok, pem} + end + rescue + _ -> + def generate_rsa_pem do + port = Port.open({:spawn, "openssl genrsa"}, [:binary]) + + {:ok, pem} = + receive do + {^port, {:data, pem}} -> {:ok, pem} + end + + Port.close(port) + + if Regex.match?(~r/RSA PRIVATE KEY/, pem) do + {:ok, pem} + else + :error + end + end + end + + def keys_from_pem(pem) do + [private_key_code] = :public_key.pem_decode(pem) + private_key = :public_key.pem_entry_decode(private_key_code) + {:RSAPrivateKey, _, modulus, exponent, _, _, _, _, _, _, _} = private_key + public_key = {:RSAPublicKey, modulus, exponent} + {:ok, private_key, public_key} + end +end diff --git a/lib/pleroma/signature.ex b/lib/pleroma/signature.ex index b7ecf00a0..1a4d54c62 100644 --- a/lib/pleroma/signature.ex +++ b/lib/pleroma/signature.ex @@ -5,11 +5,10 @@ defmodule Pleroma.Signature do @behaviour HTTPSignatures.Adapter + alias Pleroma.Keys alias Pleroma.User alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.ActivityPub.Utils - alias Pleroma.Web.Salmon - alias Pleroma.Web.WebFinger def fetch_public_key(conn) do with actor_id <- Utils.get_ap_id(conn.params["actor"]), @@ -33,8 +32,8 @@ defmodule Pleroma.Signature do end def sign(%User{} = user, headers) do - with {:ok, %{info: %{keys: keys}}} <- WebFinger.ensure_keys_present(user), - {:ok, private_key, _} <- Salmon.keys_from_pem(keys) do + with {:ok, %{info: %{keys: keys}}} <- User.ensure_keys_present(user), + {:ok, private_key, _} <- Keys.keys_from_pem(keys) do HTTPSignatures.sign(private_key, user.ap_id <> "#main-key", headers) end end diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 05fe58f7c..653dec95f 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -10,6 +10,7 @@ defmodule Pleroma.User do alias Comeonin.Pbkdf2 alias Pleroma.Activity + alias Pleroma.Keys alias Pleroma.Notification alias Pleroma.Object alias Pleroma.Registration @@ -1422,4 +1423,24 @@ defmodule Pleroma.User do } } end + + def ensure_keys_present(user) do + info = user.info + + if info.keys do + {:ok, user} + else + {:ok, pem} = Keys.generate_rsa_pem() + + info_cng = + info + |> User.Info.set_keys(pem) + + cng = + Ecto.Changeset.change(user) + |> Ecto.Changeset.put_embed(:info, info_cng) + + update_and_set_cache(cng) + end + end end diff --git a/lib/pleroma/web/activity_pub/activity_pub_controller.ex b/lib/pleroma/web/activity_pub/activity_pub_controller.ex index c967ab7a9..ad2ca1e54 100644 --- a/lib/pleroma/web/activity_pub/activity_pub_controller.ex +++ b/lib/pleroma/web/activity_pub/activity_pub_controller.ex @@ -39,7 +39,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do def user(conn, %{"nickname" => nickname}) do with %User{} = user <- User.get_cached_by_nickname(nickname), - {:ok, user} <- Pleroma.Web.WebFinger.ensure_keys_present(user) do + {:ok, user} <- User.ensure_keys_present(user) do conn |> put_resp_header("content-type", "application/activity+json") |> json(UserView.render("user.json", %{user: user})) @@ -106,7 +106,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do def following(conn, %{"nickname" => nickname, "page" => page}) do with %User{} = user <- User.get_cached_by_nickname(nickname), - {:ok, user} <- Pleroma.Web.WebFinger.ensure_keys_present(user) do + {:ok, user} <- User.ensure_keys_present(user) do {page, _} = Integer.parse(page) conn @@ -117,7 +117,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do def following(conn, %{"nickname" => nickname}) do with %User{} = user <- User.get_cached_by_nickname(nickname), - {:ok, user} <- Pleroma.Web.WebFinger.ensure_keys_present(user) do + {:ok, user} <- User.ensure_keys_present(user) do conn |> put_resp_header("content-type", "application/activity+json") |> json(UserView.render("following.json", %{user: user})) @@ -126,7 +126,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do def followers(conn, %{"nickname" => nickname, "page" => page}) do with %User{} = user <- User.get_cached_by_nickname(nickname), - {:ok, user} <- Pleroma.Web.WebFinger.ensure_keys_present(user) do + {:ok, user} <- User.ensure_keys_present(user) do {page, _} = Integer.parse(page) conn @@ -137,7 +137,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do def followers(conn, %{"nickname" => nickname}) do with %User{} = user <- User.get_cached_by_nickname(nickname), - {:ok, user} <- Pleroma.Web.WebFinger.ensure_keys_present(user) do + {:ok, user} <- User.ensure_keys_present(user) do conn |> put_resp_header("content-type", "application/activity+json") |> json(UserView.render("followers.json", %{user: user})) @@ -146,7 +146,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do def outbox(conn, %{"nickname" => nickname} = params) do with %User{} = user <- User.get_cached_by_nickname(nickname), - {:ok, user} <- Pleroma.Web.WebFinger.ensure_keys_present(user) do + {:ok, user} <- User.ensure_keys_present(user) do conn |> put_resp_header("content-type", "application/activity+json") |> json(UserView.render("outbox.json", %{user: user, max_id: params["max_id"]})) @@ -195,7 +195,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do def relay(conn, _params) do with %User{} = user <- Relay.get_actor(), - {:ok, user} <- Pleroma.Web.WebFinger.ensure_keys_present(user) do + {:ok, user} <- User.ensure_keys_present(user) do conn |> put_resp_header("content-type", "application/activity+json") |> json(UserView.render("user.json", %{user: user})) diff --git a/lib/pleroma/web/activity_pub/views/user_view.ex b/lib/pleroma/web/activity_pub/views/user_view.ex index 1254fdf6c..327e0e05b 100644 --- a/lib/pleroma/web/activity_pub/views/user_view.ex +++ b/lib/pleroma/web/activity_pub/views/user_view.ex @@ -5,6 +5,7 @@ defmodule Pleroma.Web.ActivityPub.UserView do use Pleroma.Web, :view + alias Pleroma.Keys alias Pleroma.Repo alias Pleroma.User alias Pleroma.Web.ActivityPub.ActivityPub @@ -12,8 +13,6 @@ defmodule Pleroma.Web.ActivityPub.UserView do alias Pleroma.Web.ActivityPub.Utils alias Pleroma.Web.Endpoint alias Pleroma.Web.Router.Helpers - alias Pleroma.Web.Salmon - alias Pleroma.Web.WebFinger import Ecto.Query @@ -34,8 +33,8 @@ defmodule Pleroma.Web.ActivityPub.UserView do # the instance itself is not a Person, but instead an Application def render("user.json", %{user: %{nickname: nil} = user}) do - {:ok, user} = WebFinger.ensure_keys_present(user) - {:ok, _, public_key} = Salmon.keys_from_pem(user.info.keys) + {:ok, user} = User.ensure_keys_present(user) + {:ok, _, public_key} = Keys.keys_from_pem(user.info.keys) public_key = :public_key.pem_entry_encode(:SubjectPublicKeyInfo, public_key) public_key = :public_key.pem_encode([public_key]) @@ -62,8 +61,8 @@ defmodule Pleroma.Web.ActivityPub.UserView do end def render("user.json", %{user: user}) do - {:ok, user} = WebFinger.ensure_keys_present(user) - {:ok, _, public_key} = Salmon.keys_from_pem(user.info.keys) + {:ok, user} = User.ensure_keys_present(user) + {:ok, _, public_key} = Keys.keys_from_pem(user.info.keys) public_key = :public_key.pem_entry_encode(:SubjectPublicKeyInfo, public_key) public_key = :public_key.pem_encode([public_key]) diff --git a/lib/pleroma/web/federator/federator.ex b/lib/pleroma/web/federator/federator.ex index 169fdf4dc..6b0b75284 100644 --- a/lib/pleroma/web/federator/federator.ex +++ b/lib/pleroma/web/federator/federator.ex @@ -11,7 +11,6 @@ defmodule Pleroma.Web.Federator do alias Pleroma.Web.ActivityPub.Utils alias Pleroma.Web.Federator.Publisher alias Pleroma.Web.Federator.RetryQueue - alias Pleroma.Web.WebFinger alias Pleroma.Web.Websub require Logger @@ -77,9 +76,8 @@ defmodule Pleroma.Web.Federator do def perform(:publish, activity) do Logger.debug(fn -> "Running publish for #{activity.data["id"]}" end) - with actor when not is_nil(actor) <- User.get_cached_by_ap_id(activity.data["actor"]) do - {:ok, actor} = WebFinger.ensure_keys_present(actor) - + with %User{} = actor <- User.get_cached_by_ap_id(activity.data["actor"]), + {:ok, actor} <- User.ensure_keys_present(actor) do Publisher.publish(actor, activity) end end diff --git a/lib/pleroma/web/salmon/salmon.ex b/lib/pleroma/web/salmon/salmon.ex index 42709ab47..fa30f73cd 100644 --- a/lib/pleroma/web/salmon/salmon.ex +++ b/lib/pleroma/web/salmon/salmon.ex @@ -10,6 +10,7 @@ defmodule Pleroma.Web.Salmon do use Bitwise alias Pleroma.Activity + alias Pleroma.Keys alias Pleroma.Instances alias Pleroma.User alias Pleroma.Web.ActivityPub.Visibility @@ -89,45 +90,6 @@ defmodule Pleroma.Web.Salmon do "RSA.#{modulus_enc}.#{exponent_enc}" end - # Native generation of RSA keys is only available since OTP 20+ and in default build conditions - # We try at compile time to generate natively an RSA key otherwise we fallback on the old way. - try do - _ = :public_key.generate_key({:rsa, 2048, 65_537}) - - def generate_rsa_pem do - key = :public_key.generate_key({:rsa, 2048, 65_537}) - entry = :public_key.pem_entry_encode(:RSAPrivateKey, key) - pem = :public_key.pem_encode([entry]) |> String.trim_trailing() - {:ok, pem} - end - rescue - _ -> - def generate_rsa_pem do - port = Port.open({:spawn, "openssl genrsa"}, [:binary]) - - {:ok, pem} = - receive do - {^port, {:data, pem}} -> {:ok, pem} - end - - Port.close(port) - - if Regex.match?(~r/RSA PRIVATE KEY/, pem) do - {:ok, pem} - else - :error - end - end - end - - def keys_from_pem(pem) do - [private_key_code] = :public_key.pem_decode(pem) - private_key = :public_key.pem_entry_decode(private_key_code) - {:RSAPrivateKey, _, modulus, exponent, _, _, _, _, _, _, _} = private_key - public_key = {:RSAPublicKey, modulus, exponent} - {:ok, private_key, public_key} - end - def encode(private_key, doc) do type = "application/atom+xml" encoding = "base64url" @@ -227,7 +189,7 @@ defmodule Pleroma.Web.Salmon do |> :xmerl.export_simple(:xmerl_xml) |> to_string - {:ok, private, _} = keys_from_pem(keys) + {:ok, private, _} = Keys.keys_from_pem(keys) {:ok, feed} = encode(private, feed) remote_users = remote_users(activity) @@ -253,7 +215,7 @@ defmodule Pleroma.Web.Salmon do def publish(%{id: id}, _), do: Logger.debug(fn -> "Keys missing for user #{id}" end) def gather_webfinger_links(%User{} = user) do - {:ok, _private, public} = keys_from_pem(user.info.keys) + {:ok, _private, public} = Keys.keys_from_pem(user.info.keys) magic_key = encode_key(public) [ diff --git a/lib/pleroma/web/web_finger/web_finger.ex b/lib/pleroma/web/web_finger/web_finger.ex index 1239b962a..c5b7d4acb 100644 --- a/lib/pleroma/web/web_finger/web_finger.ex +++ b/lib/pleroma/web/web_finger/web_finger.ex @@ -8,7 +8,6 @@ defmodule Pleroma.Web.WebFinger do alias Pleroma.User alias Pleroma.Web alias Pleroma.Web.Federator.Publisher - alias Pleroma.Web.Salmon alias Pleroma.Web.XML alias Pleroma.XmlBuilder require Jason @@ -61,7 +60,7 @@ defmodule Pleroma.Web.WebFinger do end def represent_user(user, "JSON") do - {:ok, user} = ensure_keys_present(user) + {:ok, user} = User.ensure_keys_present(user) %{ "subject" => "acct:#{user.nickname}@#{Pleroma.Web.Endpoint.host()}", @@ -71,7 +70,7 @@ defmodule Pleroma.Web.WebFinger do end def represent_user(user, "XML") do - {:ok, user} = ensure_keys_present(user) + {:ok, user} = User.ensure_keys_present(user) links = gather_links(user) @@ -88,27 +87,6 @@ defmodule Pleroma.Web.WebFinger do |> XmlBuilder.to_doc() end - # This seems a better fit in Salmon - def ensure_keys_present(user) do - info = user.info - - if info.keys do - {:ok, user} - else - {:ok, pem} = Salmon.generate_rsa_pem() - - info_cng = - info - |> User.Info.set_keys(pem) - - cng = - Ecto.Changeset.change(user) - |> Ecto.Changeset.put_embed(:info, info_cng) - - User.update_and_set_cache(cng) - end - end - defp get_magic_key(magic_key) do "data:application/magic-public-key," <> magic_key = magic_key {:ok, magic_key} diff --git a/test/keys_test.exs b/test/keys_test.exs new file mode 100644 index 000000000..776fdea6f --- /dev/null +++ b/test/keys_test.exs @@ -0,0 +1,20 @@ +defmodule Pleroma.KeysTest do + use Pleroma.DataCase + + alias Pleroma.Keys + + test "generates an RSA private key pem" do + {:ok, key} = Keys.generate_rsa_pem() + + assert is_binary(key) + assert Regex.match?(~r/RSA/, key) + end + + test "returns a public and private key from a pem" do + pem = File.read!("test/fixtures/private_key.pem") + {:ok, private, public} = Keys.keys_from_pem(pem) + + assert elem(private, 0) == :RSAPrivateKey + assert elem(public, 0) == :RSAPublicKey + end +end diff --git a/test/user_test.exs b/test/user_test.exs index cb6afbe07..019f2b56d 100644 --- a/test/user_test.exs +++ b/test/user_test.exs @@ -1251,4 +1251,19 @@ defmodule Pleroma.UserTest do refute user.info.confirmation_token end end + + describe "ensure_keys_present" do + test "it creates keys for a user and stores them in info" do + user = insert(:user) + refute is_binary(user.info.keys) + {:ok, user} = User.ensure_keys_present(user) + assert is_binary(user.info.keys) + end + + test "it doesn't create keys if there already are some" do + user = insert(:user, %{info: %{keys: "xxx"}}) + {:ok, user} = User.ensure_keys_present(user) + assert user.info.keys == "xxx" + end + end end diff --git a/test/web/activity_pub/activity_pub_test.exs b/test/web/activity_pub/activity_pub_test.exs index c18e0ab5f..f743f380b 100644 --- a/test/web/activity_pub/activity_pub_test.exs +++ b/test/web/activity_pub/activity_pub_test.exs @@ -1005,7 +1005,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do describe "update" do test "it creates an update activity with the new user data" do user = insert(:user) - {:ok, user} = Pleroma.Web.WebFinger.ensure_keys_present(user) + {:ok, user} = User.ensure_keys_present(user) user_data = Pleroma.Web.ActivityPub.UserView.render("user.json", %{user: user}) {:ok, update} = diff --git a/test/web/activity_pub/views/user_view_test.exs b/test/web/activity_pub/views/user_view_test.exs index 9fb9455d2..e6483db8b 100644 --- a/test/web/activity_pub/views/user_view_test.exs +++ b/test/web/activity_pub/views/user_view_test.exs @@ -2,11 +2,12 @@ defmodule Pleroma.Web.ActivityPub.UserViewTest do use Pleroma.DataCase import Pleroma.Factory + alias Pleroma.User alias Pleroma.Web.ActivityPub.UserView test "Renders a user, including the public key" do user = insert(:user) - {:ok, user} = Pleroma.Web.WebFinger.ensure_keys_present(user) + {:ok, user} = User.ensure_keys_present(user) result = UserView.render("user.json", %{user: user}) @@ -18,7 +19,7 @@ defmodule Pleroma.Web.ActivityPub.UserViewTest do test "Does not add an avatar image if the user hasn't set one" do user = insert(:user) - {:ok, user} = Pleroma.Web.WebFinger.ensure_keys_present(user) + {:ok, user} = User.ensure_keys_present(user) result = UserView.render("user.json", %{user: user}) refute result["icon"] @@ -32,7 +33,7 @@ defmodule Pleroma.Web.ActivityPub.UserViewTest do } ) - {:ok, user} = Pleroma.Web.WebFinger.ensure_keys_present(user) + {:ok, user} = User.ensure_keys_present(user) result = UserView.render("user.json", %{user: user}) assert result["icon"]["url"] == "https://someurl" @@ -42,7 +43,7 @@ defmodule Pleroma.Web.ActivityPub.UserViewTest do describe "endpoints" do test "local users have a usable endpoints structure" do user = insert(:user) - {:ok, user} = Pleroma.Web.WebFinger.ensure_keys_present(user) + {:ok, user} = User.ensure_keys_present(user) result = UserView.render("user.json", %{user: user}) @@ -58,7 +59,7 @@ defmodule Pleroma.Web.ActivityPub.UserViewTest do test "remote users have an empty endpoints structure" do user = insert(:user, local: false) - {:ok, user} = Pleroma.Web.WebFinger.ensure_keys_present(user) + {:ok, user} = User.ensure_keys_present(user) result = UserView.render("user.json", %{user: user}) @@ -68,7 +69,7 @@ defmodule Pleroma.Web.ActivityPub.UserViewTest do test "instance users do not expose oAuth endpoints" do user = insert(:user, nickname: nil, local: true) - {:ok, user} = Pleroma.Web.WebFinger.ensure_keys_present(user) + {:ok, user} = User.ensure_keys_present(user) result = UserView.render("user.json", %{user: user}) diff --git a/test/web/salmon/salmon_test.exs b/test/web/salmon/salmon_test.exs index 232082779..e86e76fe9 100644 --- a/test/web/salmon/salmon_test.exs +++ b/test/web/salmon/salmon_test.exs @@ -5,6 +5,7 @@ defmodule Pleroma.Web.Salmon.SalmonTest do use Pleroma.DataCase alias Pleroma.Activity + alias Pleroma.Keys alias Pleroma.Repo alias Pleroma.User alias Pleroma.Web.Federator.Publisher @@ -34,12 +35,6 @@ defmodule Pleroma.Web.Salmon.SalmonTest do assert Salmon.decode_and_validate(@wrong_magickey, salmon) == :error end - test "generates an RSA private key pem" do - {:ok, key} = Salmon.generate_rsa_pem() - assert is_binary(key) - assert Regex.match?(~r/RSA/, key) - end - test "it encodes a magic key from a public key" do key = Salmon.decode_key(@magickey) magic_key = Salmon.encode_key(key) @@ -51,18 +46,10 @@ defmodule Pleroma.Web.Salmon.SalmonTest do _key = Salmon.decode_key(@magickey_friendica) end - test "returns a public and private key from a pem" do - pem = File.read!("test/fixtures/private_key.pem") - {:ok, private, public} = Salmon.keys_from_pem(pem) - - assert elem(private, 0) == :RSAPrivateKey - assert elem(public, 0) == :RSAPublicKey - end - test "encodes an xml payload with a private key" do doc = File.read!("test/fixtures/incoming_note_activity.xml") pem = File.read!("test/fixtures/private_key.pem") - {:ok, private, public} = Salmon.keys_from_pem(pem) + {:ok, private, public} = Keys.keys_from_pem(pem) # Let's try a roundtrip. {:ok, salmon} = Salmon.encode(private, doc) @@ -105,7 +92,7 @@ defmodule Pleroma.Web.Salmon.SalmonTest do {:ok, activity} = Repo.insert(%Activity{data: activity_data, recipients: activity_data["to"]}) user = User.get_cached_by_ap_id(activity.data["actor"]) - {:ok, user} = Pleroma.Web.WebFinger.ensure_keys_present(user) + {:ok, user} = User.ensure_keys_present(user) Salmon.publish(user, activity) diff --git a/test/web/web_finger/web_finger_test.exs b/test/web/web_finger/web_finger_test.exs index 6b20d8d56..335c95b18 100644 --- a/test/web/web_finger/web_finger_test.exs +++ b/test/web/web_finger/web_finger_test.exs @@ -105,19 +105,4 @@ defmodule Pleroma.Web.WebFingerTest do assert template == "http://status.alpicola.com/main/xrd?uri={uri}" end end - - describe "ensure_keys_present" do - test "it creates keys for a user and stores them in info" do - user = insert(:user) - refute is_binary(user.info.keys) - {:ok, user} = WebFinger.ensure_keys_present(user) - assert is_binary(user.info.keys) - end - - test "it doesn't create keys if there already are some" do - user = insert(:user, %{info: %{keys: "xxx"}}) - {:ok, user} = WebFinger.ensure_keys_present(user) - assert user.info.keys == "xxx" - end - end end From 913484817076bf5ca4bdbe7c3c1ff34f7debd3e5 Mon Sep 17 00:00:00 2001 From: Sergey Suprunenko Date: Wed, 22 May 2019 04:04:20 +0000 Subject: [PATCH 032/193] Do not truncate DM when it contains newlines and safe_dm_mentions is set to true --- lib/pleroma/formatter.ex | 2 +- test/formatter_test.exs | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/formatter.ex b/lib/pleroma/formatter.ex index 3d7c36d21..3e3b9fe97 100644 --- a/lib/pleroma/formatter.ex +++ b/lib/pleroma/formatter.ex @@ -8,7 +8,7 @@ defmodule Pleroma.Formatter do alias Pleroma.User alias Pleroma.Web.MediaProxy - @safe_mention_regex ~r/^(\s*(?@.+?\s+)+)(?.*)/ + @safe_mention_regex ~r/^(\s*(?@.+?\s+)+)(?.*)/s @link_regex ~r"((?:http(s)?:\/\/)?[\w.-]+(?:\.[\w\.-]+)+[\w\-\._~%:/?#[\]@!\$&'\(\)\*\+,;=.]+)|[0-9a-z+\-\.]+:[0-9a-z$-_.+!*'(),]+"ui @markdown_characters_regex ~r/(`|\*|_|{|}|[|]|\(|\)|#|\+|-|\.|!)/ diff --git a/test/formatter_test.exs b/test/formatter_test.exs index 5e7011160..47b91b121 100644 --- a/test/formatter_test.exs +++ b/test/formatter_test.exs @@ -206,6 +206,15 @@ defmodule Pleroma.FormatterTest do assert mentions == [] assert expected_text == text end + + test "given the 'safe_mention' option, it will keep text after newlines" do + user = insert(:user) + text = " @#{user.nickname}\n hey dude\n\nhow are you doing?" + + {expected_text, _, _} = Formatter.linkify(text, safe_mention: true) + + assert expected_text =~ "how are you doing?" + end end describe ".parse_tags" do From 0e2c215a006c7ca5756e80a357ff6395a4325946 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Wed, 22 May 2019 07:22:19 +0200 Subject: [PATCH 033/193] MastoAPI AccountView: fill source.note with plaintext version of note Closes: https://git.pleroma.social/pleroma/pleroma/issues/926 --- lib/pleroma/web/mastodon_api/views/account_view.ex | 2 +- test/web/mastodon_api/account_view_test.exs | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex index 134c07b7e..b82d3319b 100644 --- a/lib/pleroma/web/mastodon_api/views/account_view.ex +++ b/lib/pleroma/web/mastodon_api/views/account_view.ex @@ -112,7 +112,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do fields: fields, bot: bot, source: %{ - note: "", + note: HTML.strip_tags((user.bio || "") |> String.replace("
", "\n")), sensitive: false, pleroma: %{} }, diff --git a/test/web/mastodon_api/account_view_test.exs b/test/web/mastodon_api/account_view_test.exs index a24f2a050..aaf2261bb 100644 --- a/test/web/mastodon_api/account_view_test.exs +++ b/test/web/mastodon_api/account_view_test.exs @@ -55,7 +55,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountViewTest do fields: [], bot: false, source: %{ - note: "", + note: "valid html", sensitive: false, pleroma: %{} }, @@ -120,7 +120,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountViewTest do fields: [], bot: true, source: %{ - note: "", + note: user.bio, sensitive: false, pleroma: %{} }, @@ -209,7 +209,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountViewTest do fields: [], bot: true, source: %{ - note: "", + note: user.bio, sensitive: false, pleroma: %{} }, From 1344c85e2fe636fd6b9d033eb7add1c3a9701c7f Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Wed, 22 May 2019 05:58:51 +0000 Subject: [PATCH 034/193] salmon: fix credo --- lib/pleroma/web/salmon/salmon.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/salmon/salmon.ex b/lib/pleroma/web/salmon/salmon.ex index fa30f73cd..f25d92fad 100644 --- a/lib/pleroma/web/salmon/salmon.ex +++ b/lib/pleroma/web/salmon/salmon.ex @@ -10,8 +10,8 @@ defmodule Pleroma.Web.Salmon do use Bitwise alias Pleroma.Activity - alias Pleroma.Keys alias Pleroma.Instances + alias Pleroma.Keys alias Pleroma.User alias Pleroma.Web.ActivityPub.Visibility alias Pleroma.Web.Federator.Publisher From f9e0d09ec0082a096dcd4980bc5ffebe8e3139ae Mon Sep 17 00:00:00 2001 From: lain Date: Wed, 22 May 2019 10:17:32 +0200 Subject: [PATCH 035/193] Changelog: Add SSH mode. --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3ff70e6e5..3d1e7640d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## [unreleased] ### Added +- Optional SSH access mode. - [MongooseIM](https://github.com/esl/MongooseIM) http authentication support. - LDAP authentication - External OAuth provider authentication From b6cf62ddeab04db6bd2695c5537c81e0fb1aecaf Mon Sep 17 00:00:00 2001 From: lain Date: Wed, 22 May 2019 10:28:50 +0200 Subject: [PATCH 036/193] Mix: Don't start esshd application if we don't need it. --- mix.exs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/mix.exs b/mix.exs index 95c052c34..bc5b6204f 100644 --- a/mix.exs +++ b/mix.exs @@ -40,9 +40,16 @@ defmodule Pleroma.Mixfile do # # Type `mix help compile.app` for more information. def application do + extra_applications = [:logger, :runtime_tools, :comeonin, :quack] + extra_applications = if Application.get_env(:esshd, :enabled, false) do + [:esshd | extra_applications] + else + extra_applications + end + [ mod: {Pleroma.Application, []}, - extra_applications: [:logger, :runtime_tools, :comeonin, :esshd, :quack], + extra_applications: extra_applications, included_applications: [:ex_syslogger] ] end From db9a82d168cfc452611a44d92df2b81a5e6d1e69 Mon Sep 17 00:00:00 2001 From: lain Date: Wed, 22 May 2019 10:40:15 +0200 Subject: [PATCH 037/193] Linting. --- mix.exs | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/mix.exs b/mix.exs index bc5b6204f..25ec46a46 100644 --- a/mix.exs +++ b/mix.exs @@ -41,11 +41,13 @@ defmodule Pleroma.Mixfile do # Type `mix help compile.app` for more information. def application do extra_applications = [:logger, :runtime_tools, :comeonin, :quack] - extra_applications = if Application.get_env(:esshd, :enabled, false) do - [:esshd | extra_applications] - else - extra_applications - end + + extra_applications = + if Application.get_env(:esshd, :enabled, false) do + [:esshd | extra_applications] + else + extra_applications + end [ mod: {Pleroma.Application, []}, From b22145cbc40b57cf83f6389f063a76a03625ff16 Mon Sep 17 00:00:00 2001 From: lain Date: Wed, 22 May 2019 10:44:26 +0200 Subject: [PATCH 038/193] Documentation: Specify PEM format for SSH keys. Otherwise openssh-client 7.9 will generate a different format that can't be used by esshd. --- docs/config.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/config.md b/docs/config.md index 197326bbd..63ca61d1e 100644 --- a/docs/config.md +++ b/docs/config.md @@ -477,7 +477,7 @@ config :esshd, password_authenticator: "Pleroma.BBS.Authenticator" ``` -Feel free to adjust the priv_dir and port number. Then you will have to create the key for the keys (in the example `priv/ssh_keys`) and create the host keys with `ssh-keygen -N "" -b 2048 -t rsa -f ssh_host_rsa_key`. After restarting, you should be able to connect to your Pleroma instance with `ssh username@server -p $PORT` +Feel free to adjust the priv_dir and port number. Then you will have to create the key for the keys (in the example `priv/ssh_keys`) and create the host keys with `ssh-keygen -m PEM -N "" -b 2048 -t rsa -f ssh_host_rsa_key`. After restarting, you should be able to connect to your Pleroma instance with `ssh username@server -p $PORT` ## :auth From 10ca1f91de12c3ff3a1adff2cf36e9ec47c659dd Mon Sep 17 00:00:00 2001 From: rinpatch Date: Wed, 22 May 2019 11:56:53 +0300 Subject: [PATCH 039/193] Add GIN index on object data->'name' --- .../migrations/20190522084924_add_object_index_on_name.exs | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 priv/repo/migrations/20190522084924_add_object_index_on_name.exs diff --git a/priv/repo/migrations/20190522084924_add_object_index_on_name.exs b/priv/repo/migrations/20190522084924_add_object_index_on_name.exs new file mode 100644 index 000000000..886e8499e --- /dev/null +++ b/priv/repo/migrations/20190522084924_add_object_index_on_name.exs @@ -0,0 +1,7 @@ +defmodule Pleroma.Repo.Migrations.AddObjectIndexOnName do + use Ecto.Migration + + def change do + create(index(:objects, ["(data->'name')"], name: :objects_name_index, using: :gin)) + end +end From 3b12e1ba7c99382c678ce17629352135f44dcb9f Mon Sep 17 00:00:00 2001 From: lain Date: Wed, 22 May 2019 11:01:10 +0200 Subject: [PATCH 040/193] Changelog: Add tip for debian users. --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3d1e7640d..b88edd072 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## [unreleased] ### Added -- Optional SSH access mode. +- Optional SSH access mode. (Needs `erlang-ssh` package on Debian). - [MongooseIM](https://github.com/esl/MongooseIM) http authentication support. - LDAP authentication - External OAuth provider authentication From f4cfcead8868481c19ebd93b0fcd2b942dc0e477 Mon Sep 17 00:00:00 2001 From: lain Date: Wed, 22 May 2019 11:44:17 +0200 Subject: [PATCH 041/193] Mix: Bring ecto-sql back to mainline. --- mix.exs | 5 +---- mix.lock | 2 +- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/mix.exs b/mix.exs index 95c052c34..9149b241f 100644 --- a/mix.exs +++ b/mix.exs @@ -66,10 +66,7 @@ defmodule Pleroma.Mixfile do {:plug_cowboy, "~> 2.0"}, {:phoenix_pubsub, "~> 1.1"}, {:phoenix_ecto, "~> 4.0"}, - {:ecto_sql, - git: "https://github.com/elixir-ecto/ecto_sql", - ref: "14cb065a74c488d737d973f7a91bc036c6245f78", - override: true}, + {:ecto_sql, "~> 3.1"}, {:postgrex, ">= 0.13.5"}, {:gettext, "~> 0.15"}, {:comeonin, "~> 4.1.1"}, diff --git a/mix.lock b/mix.lock index bacc09787..857bfca79 100644 --- a/mix.lock +++ b/mix.lock @@ -21,7 +21,7 @@ "deep_merge": {:hex, :deep_merge, "1.0.0", "b4aa1a0d1acac393bdf38b2291af38cb1d4a52806cf7a4906f718e1feb5ee961", [:mix], [], "hexpm"}, "earmark": {:hex, :earmark, "1.3.2", "b840562ea3d67795ffbb5bd88940b1bed0ed9fa32834915125ea7d02e35888a5", [:mix], [], "hexpm"}, "ecto": {:hex, :ecto, "3.1.4", "69d852da7a9f04ede725855a35ede48d158ca11a404fe94f8b2fb3b2162cd3c9", [:mix], [{:decimal, "~> 1.6", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm"}, - "ecto_sql": {:git, "https://github.com/elixir-ecto/ecto_sql", "14cb065a74c488d737d973f7a91bc036c6245f78", [ref: "14cb065a74c488d737d973f7a91bc036c6245f78"]}, + "ecto_sql": {:hex, :ecto_sql, "3.1.3", "2c536139190492d9de33c5fefac7323c5eaaa82e1b9bf93482a14649042f7cd9", [:mix], [{:db_connection, "~> 2.0", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.1.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:mariaex, "~> 0.9.1", [hex: :mariaex, repo: "hexpm", optional: true]}, {:myxql, "~> 0.2.0", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.14.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm"}, "esshd": {:hex, :esshd, "0.1.0", "6f93a2062adb43637edad0ea7357db2702a4b80dd9683482fe00f5134e97f4c1", [:mix], [], "hexpm"}, "eternal": {:hex, :eternal, "1.2.0", "e2a6b6ce3b8c248f7dc31451aefca57e3bdf0e48d73ae5043229380a67614c41", [:mix], [], "hexpm"}, "ex2ms": {:hex, :ex2ms, "1.5.0", "19e27f9212be9a96093fed8cdfbef0a2b56c21237196d26760f11dfcfae58e97", [:mix], [], "hexpm"}, From f323031927ecaf155e661b17cc9b96333fb9e4ad Mon Sep 17 00:00:00 2001 From: lain Date: Wed, 22 May 2019 12:57:20 +0200 Subject: [PATCH 042/193] Mix: Only start sshd when needed, second try. --- mix.exs | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/mix.exs b/mix.exs index 25ec46a46..b2c075c85 100644 --- a/mix.exs +++ b/mix.exs @@ -40,18 +40,9 @@ defmodule Pleroma.Mixfile do # # Type `mix help compile.app` for more information. def application do - extra_applications = [:logger, :runtime_tools, :comeonin, :quack] - - extra_applications = - if Application.get_env(:esshd, :enabled, false) do - [:esshd | extra_applications] - else - extra_applications - end - [ mod: {Pleroma.Application, []}, - extra_applications: extra_applications, + extra_applications: [:logger, :runtime_tools, :comeonin, :quack], included_applications: [:ex_syslogger] ] end @@ -129,7 +120,7 @@ defmodule Pleroma.Mixfile do {:recon, github: "ferd/recon", tag: "2.4.0"}, {:quack, "~> 0.1.1"}, {:benchee, "~> 1.0"}, - {:esshd, "~> 0.1.0"}, + {:esshd, "~> 0.1.0", runtime: Application.get_env(:esshd, :enabled, false)}, {:ex_rated, "~> 1.2"}, {:plug_static_index_html, "~> 1.0.0"}, {:excoveralls, "~> 0.11.1", only: :test} From 78ac8ee56139ed98625c54ce627eb37047a361f0 Mon Sep 17 00:00:00 2001 From: lambda Date: Wed, 22 May 2019 11:07:51 +0000 Subject: [PATCH 043/193] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b88edd072..bb2306fc4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## [unreleased] ### Added -- Optional SSH access mode. (Needs `erlang-ssh` package on Debian). +- Optional SSH access mode. (Needs `erlang-ssh` package on some distributions). - [MongooseIM](https://github.com/esl/MongooseIM) http authentication support. - LDAP authentication - External OAuth provider authentication From 620908a2db86942a00bc0ba9c71c037061e26967 Mon Sep 17 00:00:00 2001 From: Maksim Date: Wed, 22 May 2019 15:44:50 +0000 Subject: [PATCH 044/193] [#699] add worker to clean expired oauth tokens --- CHANGELOG.md | 1 + config/config.exs | 4 +- docs/config.md | 2 + lib/pleroma/application.ex | 1 + lib/pleroma/web/oauth/token.ex | 36 ++++++-------- lib/pleroma/web/oauth/token/clean_worker.ex | 41 +++++++++++++++ lib/pleroma/web/oauth/token/query.ex | 55 +++++++++++++++++++++ test/web/oauth/token_test.exs | 13 +++++ 8 files changed, 132 insertions(+), 21 deletions(-) create mode 100644 lib/pleroma/web/oauth/token/clean_worker.ex create mode 100644 lib/pleroma/web/oauth/token/query.ex diff --git a/CHANGELOG.md b/CHANGELOG.md index b5c42d1fd..02d64a850 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -41,6 +41,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - OAuth: added support for refresh tokens - Emoji packs and emoji pack manager - Object pruning (`mix pleroma.database prune_objects`) +- OAuth: added job to clean expired access tokens ### Changed - **Breaking:** Configuration: move from Pleroma.Mailer to Pleroma.Emails.Mailer diff --git a/config/config.exs b/config/config.exs index a05f8b1d2..33b7e713d 100644 --- a/config/config.exs +++ b/config/config.exs @@ -481,7 +481,9 @@ config :pleroma, Pleroma.ScheduledActivity, config :pleroma, :oauth2, token_expires_in: 600, - issue_new_refresh_token: true + issue_new_refresh_token: true, + clean_expired_tokens: false, + clean_expired_tokens_interval: 86_400_000 config :pleroma, :database, rum_enabled: false diff --git a/docs/config.md b/docs/config.md index a050068f4..264b65499 100644 --- a/docs/config.md +++ b/docs/config.md @@ -550,6 +550,8 @@ Configure OAuth 2 provider capabilities: * `token_expires_in` - The lifetime in seconds of the access token. * `issue_new_refresh_token` - Keeps old refresh token or generate new refresh token when to obtain an access token. +* `clean_expired_tokens` - Enable a background job to clean expired oauth tokens. Defaults to `false`. +* `clean_expired_tokens_interval` - Interval to run the job to clean expired tokens. Defaults to `86_400_000` (24 hours). ## :emoji * `shortcode_globs`: Location of custom emoji files. `*` can be used as a wildcard. Example `["/emoji/custom/**/*.png"]` diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index dab45a0b2..76df3945e 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -110,6 +110,7 @@ defmodule Pleroma.Application do hackney_pool_children() ++ [ worker(Pleroma.Web.Federator.RetryQueue, []), + worker(Pleroma.Web.OAuth.Token.CleanWorker, []), worker(Pleroma.Stats, []), worker(Task, [&Pleroma.Web.Push.init/0], restart: :temporary, id: :web_push_init), worker(Task, [&Pleroma.Web.Federator.init/0], restart: :temporary, id: :federator_init) diff --git a/lib/pleroma/web/oauth/token.ex b/lib/pleroma/web/oauth/token.ex index 66c95c2e9..f412f7eb2 100644 --- a/lib/pleroma/web/oauth/token.ex +++ b/lib/pleroma/web/oauth/token.ex @@ -5,7 +5,6 @@ defmodule Pleroma.Web.OAuth.Token do use Ecto.Schema - import Ecto.Query import Ecto.Changeset alias Pleroma.Repo @@ -13,6 +12,7 @@ defmodule Pleroma.Web.OAuth.Token do alias Pleroma.Web.OAuth.App alias Pleroma.Web.OAuth.Authorization alias Pleroma.Web.OAuth.Token + alias Pleroma.Web.OAuth.Token.Query @expires_in Pleroma.Config.get([:oauth2, :token_expires_in], 600) @type t :: %__MODULE__{} @@ -31,17 +31,17 @@ defmodule Pleroma.Web.OAuth.Token do @doc "Gets token for app by access token" @spec get_by_token(App.t(), String.t()) :: {:ok, t()} | {:error, :not_found} def get_by_token(%App{id: app_id} = _app, token) do - from(t in __MODULE__, where: t.app_id == ^app_id and t.token == ^token) + Query.get_by_app(app_id) + |> Query.get_by_token(token) |> Repo.find_resource() end @doc "Gets token for app by refresh token" @spec get_by_refresh_token(App.t(), String.t()) :: {:ok, t()} | {:error, :not_found} def get_by_refresh_token(%App{id: app_id} = _app, token) do - from(t in __MODULE__, - where: t.app_id == ^app_id and t.refresh_token == ^token, - preload: [:user] - ) + Query.get_by_app(app_id) + |> Query.get_by_refresh_token(token) + |> Query.preload([:user]) |> Repo.find_resource() end @@ -97,29 +97,25 @@ defmodule Pleroma.Web.OAuth.Token do end def delete_user_tokens(%User{id: user_id}) do - from( - t in Token, - where: t.user_id == ^user_id - ) + Query.get_by_user(user_id) |> Repo.delete_all() end def delete_user_token(%User{id: user_id}, token_id) do - from( - t in Token, - where: t.user_id == ^user_id, - where: t.id == ^token_id - ) + Query.get_by_user(user_id) + |> Query.get_by_id(token_id) + |> Repo.delete_all() + end + + def delete_expired_tokens do + Query.get_expired_tokens() |> Repo.delete_all() end def get_user_tokens(%User{id: user_id}) do - from( - t in Token, - where: t.user_id == ^user_id - ) + Query.get_by_user(user_id) + |> Query.preload([:app]) |> Repo.all() - |> Repo.preload(:app) end def is_expired?(%__MODULE__{valid_until: valid_until}) do diff --git a/lib/pleroma/web/oauth/token/clean_worker.ex b/lib/pleroma/web/oauth/token/clean_worker.ex new file mode 100644 index 000000000..dca852449 --- /dev/null +++ b/lib/pleroma/web/oauth/token/clean_worker.ex @@ -0,0 +1,41 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2018 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.OAuth.Token.CleanWorker do + @moduledoc """ + The module represents functions to clean an expired oauth tokens. + """ + + # 10 seconds + @start_interval 10_000 + @interval Pleroma.Config.get( + # 24 hours + [:oauth2, :clean_expired_tokens_interval], + 86_400_000 + ) + @queue :background + + alias Pleroma.Web.OAuth.Token + + def start_link, do: GenServer.start_link(__MODULE__, nil) + + def init(_) do + if Pleroma.Config.get([:oauth2, :clean_expired_tokens], false) do + Process.send_after(self(), :perform, @start_interval) + {:ok, nil} + else + :ignore + end + end + + @doc false + def handle_info(:perform, state) do + Process.send_after(self(), :perform, @interval) + PleromaJobQueue.enqueue(@queue, __MODULE__, [:clean]) + {:noreply, state} + end + + # Job Worker Callbacks + def perform(:clean), do: Token.delete_expired_tokens() +end diff --git a/lib/pleroma/web/oauth/token/query.ex b/lib/pleroma/web/oauth/token/query.ex new file mode 100644 index 000000000..d92e1f071 --- /dev/null +++ b/lib/pleroma/web/oauth/token/query.ex @@ -0,0 +1,55 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2018 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.OAuth.Token.Query do + @moduledoc """ + Contains queries for OAuth Token. + """ + + import Ecto.Query, only: [from: 2] + + @type query :: Ecto.Queryable.t() | Token.t() + + alias Pleroma.Web.OAuth.Token + + @spec get_by_refresh_token(query, String.t()) :: query + def get_by_refresh_token(query \\ Token, refresh_token) do + from(q in query, where: q.refresh_token == ^refresh_token) + end + + @spec get_by_token(query, String.t()) :: query + def get_by_token(query \\ Token, token) do + from(q in query, where: q.token == ^token) + end + + @spec get_by_app(query, String.t()) :: query + def get_by_app(query \\ Token, app_id) do + from(q in query, where: q.app_id == ^app_id) + end + + @spec get_by_id(query, String.t()) :: query + def get_by_id(query \\ Token, id) do + from(q in query, where: q.id == ^id) + end + + @spec get_expired_tokens(query, DateTime.t() | nil) :: query + def get_expired_tokens(query \\ Token, date \\ nil) do + expired_date = date || Timex.now() + from(q in query, where: fragment("?", q.valid_until) < ^expired_date) + end + + @spec get_by_user(query, String.t()) :: query + def get_by_user(query \\ Token, user_id) do + from(q in query, where: q.user_id == ^user_id) + end + + @spec preload(query, any) :: query + def preload(query \\ Token, assoc_preload \\ []) + + def preload(query, assoc_preload) when is_list(assoc_preload) do + from(q in query, preload: ^assoc_preload) + end + + def preload(query, _assoc_preload), do: query +end diff --git a/test/web/oauth/token_test.exs b/test/web/oauth/token_test.exs index ad2a49f09..3c07309b7 100644 --- a/test/web/oauth/token_test.exs +++ b/test/web/oauth/token_test.exs @@ -69,4 +69,17 @@ defmodule Pleroma.Web.OAuth.TokenTest do assert tokens == 2 end + + test "deletes expired tokens" do + insert(:oauth_token, valid_until: Timex.shift(Timex.now(), days: -3)) + insert(:oauth_token, valid_until: Timex.shift(Timex.now(), days: -3)) + t3 = insert(:oauth_token) + t4 = insert(:oauth_token, valid_until: Timex.shift(Timex.now(), minutes: 10)) + {tokens, _} = Token.delete_expired_tokens() + assert tokens == 2 + available_tokens = Pleroma.Repo.all(Token) + + token_ids = available_tokens |> Enum.map(& &1.id) + assert token_ids == [t3.id, t4.id] + end end From 54e10a3e55fe46b71ef7f330605baf8bcccd5a44 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Wed, 22 May 2019 20:10:52 +0300 Subject: [PATCH 045/193] Disable timeouts for object pruning query --- lib/mix/tasks/pleroma/database.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/mix/tasks/pleroma/database.ex b/lib/mix/tasks/pleroma/database.ex index f9bafb277..4d480ac3f 100644 --- a/lib/mix/tasks/pleroma/database.ex +++ b/lib/mix/tasks/pleroma/database.ex @@ -107,7 +107,7 @@ defmodule Mix.Tasks.Pleroma.Database do where: fragment("split_part(?->>'actor', '/', 3) != ?", o.data, ^Pleroma.Web.Endpoint.host()) ) - |> Repo.delete_all() + |> Repo.delete_all(timeout: :infinity) if Keyword.get(options, :vacuum) do Logger.info("Runnning VACUUM FULL") From 19c90d47c4f649d0962098050f6cbc65eeb889d0 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Wed, 22 May 2019 21:17:57 +0300 Subject: [PATCH 046/193] Normalize poll votes to Answer objects --- lib/pleroma/web/activity_pub/activity_pub.ex | 14 +-------- .../web/activity_pub/transmogrifier.ex | 22 ++++++++++++- lib/pleroma/web/activity_pub/utils.ex | 2 +- test/web/activity_pub/transmogrifier_test.exs | 31 +++++++++++++++++-- 4 files changed, 51 insertions(+), 18 deletions(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index fdebf1f6b..5b3fb7e84 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -194,7 +194,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do if activity.data["type"] in ["Create", "Announce", "Delete"] do object = Object.normalize(activity) # Do not stream out poll replies - unless object.data["name"] do + unless object.data["type"] == "Answer" do Pleroma.Web.Streamer.stream("user", activity) Pleroma.Web.Streamer.stream("list", activity) @@ -493,7 +493,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do from(activity in Activity) |> restrict_blocked(opts) - |> restrict_poll_replies(opts) |> restrict_recipients(recipients, opts["user"]) |> where( [activity], @@ -833,16 +832,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do defp restrict_muted_reblogs(query, _), do: query - defp restrict_poll_replies(query, %{"include_poll_replies" => "true"}), do: query - - defp restrict_poll_replies(query, _) do - if has_named_binding?(query, :object) do - from([activity, object: o] in query, where: fragment("?->'name' is null", o.data)) - else - query - end - end - defp maybe_preload_objects(query, %{"skip_preload" => true}), do: query defp maybe_preload_objects(query, _) do @@ -896,7 +885,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do |> restrict_pinned(opts) |> restrict_muted_reblogs(opts) |> Activity.restrict_deactivated_users() - |> restrict_poll_replies(opts) end def fetch_activities(recipients, opts \\ %{}) do diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 8b2427258..70b467b3f 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -35,6 +35,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do |> fix_likes |> fix_addressing |> fix_summary + |> fix_type end def fix_summary(%{"summary" => nil} = object) do @@ -328,6 +329,18 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do def fix_content_map(object), do: object + def fix_type(%{"inReplyTo" => reply_id} = object) when is_binary(reply_id) do + reply = Object.normalize(reply_id) + + if reply.data["type"] == "Question" and object["name"] do + Map.put(object, "type", "Answer") + else + object + end + end + + def fix_type(object), do: object + defp mastodon_follow_hack(%{"id" => id, "actor" => follower_id}, followed) do with true <- id =~ "follows", %User{local: true} = follower <- User.get_cached_by_ap_id(follower_id), @@ -398,7 +411,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do # - tags # - emoji def handle_incoming(%{"type" => "Create", "object" => %{"type" => objtype} = object} = data) - when objtype in ["Article", "Note", "Video", "Page", "Question"] do + when objtype in ["Article", "Note", "Video", "Page", "Question", "Answer"] do actor = Containment.get_actor(data) data = @@ -731,6 +744,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do |> set_reply_to_uri |> strip_internal_fields |> strip_internal_tags + |> set_type end # @doc @@ -895,6 +909,12 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do Map.put(object, "sensitive", "nsfw" in tags) end + def set_type(%{"type" => "Answer"} = object) do + Map.put(object, "type", "Note") + end + + def set_type(object), do: object + def add_attributed_to(object) do attributed_to = object["attributedTo"] || object["actor"] diff --git a/lib/pleroma/web/activity_pub/utils.ex b/lib/pleroma/web/activity_pub/utils.ex index 63454e3f7..9646bbee9 100644 --- a/lib/pleroma/web/activity_pub/utils.ex +++ b/lib/pleroma/web/activity_pub/utils.ex @@ -19,7 +19,7 @@ defmodule Pleroma.Web.ActivityPub.Utils do require Logger - @supported_object_types ["Article", "Note", "Video", "Page", "Question"] + @supported_object_types ["Article", "Note", "Video", "Page", "Question", "Answer"] @supported_report_states ~w(open closed resolved) @valid_visibilities ~w(public unlisted private direct) diff --git a/test/web/activity_pub/transmogrifier_test.exs b/test/web/activity_pub/transmogrifier_test.exs index 32d94e3e9..8422fc3d5 100644 --- a/test/web/activity_pub/transmogrifier_test.exs +++ b/test/web/activity_pub/transmogrifier_test.exs @@ -130,7 +130,7 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do end) end - test "in increments vote counters on question activities" do + test "it rewrites Note votes to Answers and increments vote counters on question activities" do user = insert(:user) {:ok, activity} = @@ -148,8 +148,9 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do |> Kernel.put_in(["object", "inReplyTo"], object.data["id"]) |> Kernel.put_in(["object", "to"], user.ap_id) - {:ok, %Activity{local: false}} = Transmogrifier.handle_incoming(data) - + {:ok, %Activity{local: false} = activity} = Transmogrifier.handle_incoming(data) + answer_object = Object.normalize(activity) + assert answer_object.data["type"] == "Answer" object = Object.get_by_ap_id(object.data["id"]) assert Enum.any?( @@ -1257,4 +1258,28 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do {:ok, _} = Transmogrifier.prepare_outgoing(activity.data) end end + + test "Rewrites Answers to Notes" do + user = insert(:user) + + {:ok, poll_activity} = + CommonAPI.post(user, %{ + "status" => "suya...", + "poll" => %{"options" => ["suya", "suya.", "suya.."], "expires_in" => 10} + }) + + poll_object = Object.normalize(poll_activity) + # TODO: Replace with CommonAPI vote creation when implemented + data = + File.read!("test/fixtures/mastodon-vote.json") + |> Poison.decode!() + |> Kernel.put_in(["to"], user.ap_id) + |> Kernel.put_in(["object", "inReplyTo"], poll_object.data["id"]) + |> Kernel.put_in(["object", "to"], user.ap_id) + + {:ok, %Activity{local: false} = activity} = Transmogrifier.handle_incoming(data) + {:ok, data} = Transmogrifier.prepare_outgoing(activity.data) + + assert data["object"]["type"] == "Note" + end end From ac7702f8009bf244a9e77cba39114dbb6584739f Mon Sep 17 00:00:00 2001 From: rinpatch Date: Wed, 22 May 2019 21:49:19 +0300 Subject: [PATCH 047/193] Exclude Answers from fetching by default --- lib/pleroma/web/activity_pub/activity_pub.ex | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 5b3fb7e84..db5a4a7ee 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -492,6 +492,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do if opts["user"], do: [opts["user"].ap_id | opts["user"].following] ++ public, else: public from(activity in Activity) + |> Activity.with_preloaded_object() + |> exclude_poll_votes(opts) |> restrict_blocked(opts) |> restrict_recipients(recipients, opts["user"]) |> where( @@ -832,6 +834,18 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do defp restrict_muted_reblogs(query, _), do: query + defp exclude_poll_votes(query, %{"include_poll_votes" => "true"}), do: query + + defp exclude_poll_votes(query, _) do + if has_named_binding?(query, :object) do + from([activity, object: o] in query, + where: fragment("not(?->>'type' = ?)", o.data, "Answer") + ) + else + query + end + end + defp maybe_preload_objects(query, %{"skip_preload" => true}), do: query defp maybe_preload_objects(query, _) do @@ -865,6 +879,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do |> maybe_preload_objects(opts) |> maybe_preload_bookmarks(opts) |> maybe_order(opts) + |> exclude_poll_votes(opts) |> restrict_recipients(recipients, opts["user"]) |> restrict_tag(opts) |> restrict_tag_reject(opts) From 75b6c4b00433560fb5ee502f13e8261b4b8a246a Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Wed, 22 May 2019 04:15:59 +0000 Subject: [PATCH 048/193] mrf: defang policy modules for filtering user profile objects --- lib/pleroma/web/activity_pub/mrf/simple_policy.ex | 6 ++++-- lib/pleroma/web/activity_pub/mrf/user_allowlist.ex | 6 ++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex index 7190652d2..ffaa4b7db 100644 --- a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex @@ -105,8 +105,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do defp check_report_removal(_actor_info, object), do: {:ok, object} @impl true - def filter(object) do - actor_info = URI.parse(object["actor"]) + def filter(%{"actor" => actor} = object) do + actor_info = URI.parse(actor) with {:ok, object} <- check_accept(actor_info, object), {:ok, object} <- check_reject(actor_info, object), @@ -119,4 +119,6 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do _e -> {:reject, nil} end end + + def filter(object), do: {:ok, object} end diff --git a/lib/pleroma/web/activity_pub/mrf/user_allowlist.ex b/lib/pleroma/web/activity_pub/mrf/user_allowlist.ex index f5078d818..47663414a 100644 --- a/lib/pleroma/web/activity_pub/mrf/user_allowlist.ex +++ b/lib/pleroma/web/activity_pub/mrf/user_allowlist.ex @@ -19,10 +19,12 @@ defmodule Pleroma.Web.ActivityPub.MRF.UserAllowListPolicy do end @impl true - def filter(object) do - actor_info = URI.parse(object["actor"]) + def filter(%{"actor" => actor} = object) do + actor_info = URI.parse(actor) allow_list = Config.get([:mrf_user_allowlist, String.to_atom(actor_info.host)], []) filter_by_list(object, allow_list) end + + def filter(object), do: {:ok, object} end From 60f882b09f5f837546d59f8eef56b905e065ec60 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Wed, 22 May 2019 04:33:10 +0000 Subject: [PATCH 049/193] activitypub: run user objects through MRF filters --- lib/pleroma/web/activity_pub/activity_pub.ex | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 3d9679ec0..aa0229db7 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -909,7 +909,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do end end - def user_data_from_user_object(data) do + defp object_to_user_data(data) do avatar = data["icon"]["url"] && %{ @@ -956,9 +956,19 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do {:ok, user_data} end + def user_data_from_user_object(data) do + with {:ok, data} <- MRF.filter(data), + {:ok, data} <- object_to_user_data(data) do + {:ok, data} + else + e -> {:error, e} + end + end + def fetch_and_prepare_user_from_ap_id(ap_id) do - with {:ok, data} <- Fetcher.fetch_and_contain_remote_object_from_id(ap_id) do - user_data_from_user_object(data) + with {:ok, data} <- Fetcher.fetch_and_contain_remote_object_from_id(ap_id), + {:ok, data} <- user_data_from_user_object(data) do + {:ok, data} else e -> Logger.error("Could not decode user at fetch #{ap_id}, #{inspect(e)}") end From baf72d6c580e5c05ef5fea8a57c57150a5d38589 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Wed, 22 May 2019 04:55:16 +0000 Subject: [PATCH 050/193] mrf: simple policy: add the ability to strip avatars and banners from user profiles --- config/config.exs | 4 ++- .../web/activity_pub/mrf/simple_policy.ex | 32 +++++++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/config/config.exs b/config/config.exs index 33b7e713d..e90821d66 100644 --- a/config/config.exs +++ b/config/config.exs @@ -314,7 +314,9 @@ config :pleroma, :mrf_simple, federated_timeline_removal: [], report_removal: [], reject: [], - accept: [] + accept: [], + avatar_removal: [], + banner_removal: [] config :pleroma, :mrf_keyword, reject: [], diff --git a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex index ffaa4b7db..890d70a7a 100644 --- a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex @@ -104,6 +104,26 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do defp check_report_removal(_actor_info, object), do: {:ok, object} + defp check_avatar_removal(%{host: actor_host} = _actor_info, %{"icon" => _icon} = object) do + if actor_host in Pleroma.Config.get([:mrf_simple, :avatar_removal]) do + {:ok, Map.delete(object, "icon")} + else + {:ok, object} + end + end + + defp check_avatar_removal(_actor_info, object), do: {:ok, object} + + defp check_banner_removal(%{host: actor_host} = _actor_info, %{"image" => _image} = object) do + if actor_host in Pleroma.Config.get([:mrf_simple, :banner_removal]) do + {:ok, Map.delete(object, "image")} + else + {:ok, object} + end + end + + defp check_banner_removal(_actor_info, object), do: {:ok, object} + @impl true def filter(%{"actor" => actor} = object) do actor_info = URI.parse(actor) @@ -120,5 +140,17 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do end end + def filter(%{"id" => actor, "type" => obj_type} = object) + when obj_type in ["Application", "Group", "Organization", "Person", "Service"] do + actor_info = URI.parse(actor) + + with {:ok, object} <- check_avatar_removal(actor_info, object), + {:ok, object} <- check_banner_removal(actor_info, object) do + {:ok, object} + else + _e -> {:reject, nil} + end + end + def filter(object), do: {:ok, object} end From 8086c7aed6cdc3b2ac1c09c6c40344e47be08ed9 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Wed, 22 May 2019 05:08:37 +0000 Subject: [PATCH 051/193] tests: add tests for banner and avatar removal --- .../activity_pub/mrf/simple_policy_test.exs | 73 ++++++++++++++++++- 1 file changed, 72 insertions(+), 1 deletion(-) diff --git a/test/web/activity_pub/mrf/simple_policy_test.exs b/test/web/activity_pub/mrf/simple_policy_test.exs index 74af7dcde..3d1f26e60 100644 --- a/test/web/activity_pub/mrf/simple_policy_test.exs +++ b/test/web/activity_pub/mrf/simple_policy_test.exs @@ -17,7 +17,9 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicyTest do federated_timeline_removal: [], report_removal: [], reject: [], - accept: [] + accept: [], + avatar_removal: [], + banner_removal: [] ) on_exit(fn -> @@ -206,6 +208,60 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicyTest do end end + describe "when :avatar_removal" do + test "is empty" do + Config.put([:mrf_simple, :avatar_removal], []) + + remote_user = build_remote_user() + + assert SimplePolicy.filter(remote_user) == {:ok, remote_user} + end + + test "is not empty but it doesn't have a matching host" do + Config.put([:mrf_simple, :avatar_removal], ["non.matching.remote"]) + + remote_user = build_remote_user() + + assert SimplePolicy.filter(remote_user) == {:ok, remote_user} + end + + test "has a matching host" do + Config.put([:mrf_simple, :avatar_removal], ["remote.instance"]) + + remote_user = build_remote_user() + {:ok, filtered} = SimplePolicy.filter(remote_user) + + refute filtered["icon"] + end + end + + describe "when :banner_removal" do + test "is empty" do + Config.put([:mrf_simple, :banner_removal], []) + + remote_user = build_remote_user() + + assert SimplePolicy.filter(remote_user) == {:ok, remote_user} + end + + test "is not empty but it doesn't have a matching host" do + Config.put([:mrf_simple, :banner_removal], ["non.matching.remote"]) + + remote_user = build_remote_user() + + assert SimplePolicy.filter(remote_user) == {:ok, remote_user} + end + + test "has a matching host" do + Config.put([:mrf_simple, :banner_removal], ["remote.instance"]) + + remote_user = build_remote_user() + {:ok, filtered} = SimplePolicy.filter(remote_user) + + refute filtered["image"] + end + end + defp build_local_message do %{ "actor" => "#{Pleroma.Web.base_url()}/users/alice", @@ -217,4 +273,19 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicyTest do defp build_remote_message do %{"actor" => "https://remote.instance/users/bob"} end + + defp build_remote_user do + %{ + "id" => "https://remote.instance/users/bob", + "icon" => %{ + "url" => "http://example.com/image.jpg", + "type" => "Image" + }, + "image" => %{ + "url" => "http://example.com/image.jpg", + "type" => "Image" + }, + "type" => "Person" + } + end end From 7d9b33b3cebeed451210f754a8c34cc14a9e969b Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Wed, 22 May 2019 05:55:09 +0000 Subject: [PATCH 052/193] update documentation for the new MRF features [no-ci] --- CHANGELOG.md | 2 ++ docs/config.md | 3 +++ 2 files changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 02d64a850..2a2b11ddf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -42,6 +42,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Emoji packs and emoji pack manager - Object pruning (`mix pleroma.database prune_objects`) - OAuth: added job to clean expired access tokens +- MRF: Support for rejecting reports from specific instances (`mrf_simple`) +- MRF: Support for stripping avatars and banner images from specific instances (`mrf_simple`) ### Changed - **Breaking:** Configuration: move from Pleroma.Mailer to Pleroma.Emails.Mailer diff --git a/docs/config.md b/docs/config.md index 264b65499..1d1d24c32 100644 --- a/docs/config.md +++ b/docs/config.md @@ -220,6 +220,9 @@ relates to mascots on the mastodon frontend * `federated_timeline_removal`: List of instances to remove from Federated (aka The Whole Known Network) Timeline * `reject`: List of instances to reject any activities from * `accept`: List of instances to accept any activities from +* `report_removal`: List of instances to reject reports from +* `avatar_removal`: List of instances to strip avatars from +* `banner_removal`: List of instances to strip banners from ## :mrf_rejectnonpublic * `allow_followersonly`: whether to allow followers-only posts From e6b175ed6cafcd6a05120e003cfe40a04b38849f Mon Sep 17 00:00:00 2001 From: rinpatch Date: Wed, 22 May 2019 21:57:46 +0300 Subject: [PATCH 053/193] Fix credo issues --- lib/pleroma/web/mastodon_api/views/status_view.ex | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index c501c213c..7a393a817 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -357,7 +357,8 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do end) %{ - # Mastodon uses separate ids for polls, but an object can't have more than one poll embedded so object id is fine + # Mastodon uses separate ids for polls, but an object can't have + # more than one poll embedded so object id is fine id: object.id, expires_at: Utils.to_masto_date(end_time), expired: expired, From 8b2d39c1ec8f47df8a2159c23eabdc61b983ddf0 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Thu, 23 May 2019 14:03:16 +0300 Subject: [PATCH 054/193] Change the order of preloading when fetching activities for context --- lib/pleroma/web/activity_pub/activity_pub.ex | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index db5a4a7ee..b06200cb5 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -492,7 +492,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do if opts["user"], do: [opts["user"].ap_id | opts["user"].following] ++ public, else: public from(activity in Activity) - |> Activity.with_preloaded_object() + |> maybe_preload_objects(opts) |> exclude_poll_votes(opts) |> restrict_blocked(opts) |> restrict_recipients(recipients, opts["user"]) @@ -513,7 +513,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do def fetch_activities_for_context(context, opts \\ %{}) do context |> fetch_activities_for_context_query(opts) - |> Activity.with_preloaded_object() |> Repo.all() end @@ -521,7 +520,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do Pleroma.FlakeId.t() | nil def fetch_latest_activity_id_for_context(context, opts \\ %{}) do context - |> fetch_activities_for_context_query(opts) + |> fetch_activities_for_context_query(Map.merge(%{"skip_preload" => true}, opts)) |> limit(1) |> select([a], a.id) |> Repo.one() From 356c047759735bcd984ebd44059a21a3cf22af0e Mon Sep 17 00:00:00 2001 From: Alfie Pates Date: Thu, 23 May 2019 22:33:27 +0100 Subject: [PATCH 055/193] explicitly set reverse proxy upstream to IPv4 since Pleroma.Web.Endpoint binds on IPv4 only and `localhost.` resolves to [::0] on some systems fixes #930. --- installation/caddyfile-pleroma.example | 4 +++- installation/pleroma-apache.conf | 6 ++++-- installation/pleroma.nginx | 4 +++- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/installation/caddyfile-pleroma.example b/installation/caddyfile-pleroma.example index fcf76718e..7985d9c67 100644 --- a/installation/caddyfile-pleroma.example +++ b/installation/caddyfile-pleroma.example @@ -10,7 +10,9 @@ example.tld { gzip - proxy / localhost:4000 { + # this is explicitly IPv4 since Pleroma.Web.Endpoint binds on IPv4 only + # and `localhost.` resolves to [::0] on some systems: see issue #930 + proxy / 127.0.0.1:4000 { websocket transparent } diff --git a/installation/pleroma-apache.conf b/installation/pleroma-apache.conf index 2beb7c4cc..b5640ac3d 100644 --- a/installation/pleroma-apache.conf +++ b/installation/pleroma-apache.conf @@ -58,8 +58,10 @@ CustomLog ${APACHE_LOG_DIR}/access.log combined RewriteRule /(.*) ws://localhost:4000/$1 [P,L] ProxyRequests off - ProxyPass / http://localhost:4000/ - ProxyPassReverse / http://localhost:4000/ + # this is explicitly IPv4 since Pleroma.Web.Endpoint binds on IPv4 only + # and `localhost.` resolves to [::0] on some systems: see issue #930 + ProxyPass / http://127.0.0.1:4000/ + ProxyPassReverse / http://127.0.0.1:4000/ RequestHeader set Host ${servername} ProxyPreserveHost On diff --git a/installation/pleroma.nginx b/installation/pleroma.nginx index cc75d78b2..7425da33f 100644 --- a/installation/pleroma.nginx +++ b/installation/pleroma.nginx @@ -69,7 +69,9 @@ server { proxy_set_header Connection "upgrade"; proxy_set_header Host $http_host; - proxy_pass http://localhost:4000; + # this is explicitly IPv4 since Pleroma.Web.Endpoint binds on IPv4 only + # and `localhost.` resolves to [::0] on some systems: see issue #930 + proxy_pass http://127.0.0.1:4000; client_max_body_size 16m; } From f916e4cdd9a502b83c615146c598be135f47e57a Mon Sep 17 00:00:00 2001 From: feld Date: Fri, 24 May 2019 20:33:55 +0000 Subject: [PATCH 056/193] Move the Cache Control header test to its own file We can consolidate our cache control header tests here --- lib/pleroma/web/endpoint.ex | 19 +++++++++++++++++-- test/plugs/cache_control_test.exs | 20 ++++++++++++++++++++ 2 files changed, 37 insertions(+), 2 deletions(-) create mode 100644 test/plugs/cache_control_test.exs diff --git a/lib/pleroma/web/endpoint.ex b/lib/pleroma/web/endpoint.ex index 9ef30e885..8cd7a2270 100644 --- a/lib/pleroma/web/endpoint.ex +++ b/lib/pleroma/web/endpoint.ex @@ -16,17 +16,32 @@ defmodule Pleroma.Web.Endpoint do plug(Pleroma.Plugs.UploadedMedia) + @static_cache_control "public, no-cache" + # InstanceStatic needs to be before Plug.Static to be able to override shipped-static files # If you're adding new paths to `only:` you'll need to configure them in InstanceStatic as well - plug(Pleroma.Plugs.InstanceStatic, at: "/") + # Cache-control headers are duplicated in case we turn off etags in the future + plug(Pleroma.Plugs.InstanceStatic, + at: "/", + gzip: true, + cache_control_for_etags: @static_cache_control, + headers: %{ + "cache-control" => @static_cache_control + } + ) plug( Plug.Static, at: "/", from: :pleroma, only: - ~w(index.html robots.txt static finmoji emoji packs sounds images instance sw.js sw-pleroma.js favicon.png schemas doc) + ~w(index.html robots.txt static finmoji emoji packs sounds images instance sw.js sw-pleroma.js favicon.png schemas doc), # credo:disable-for-previous-line Credo.Check.Readability.MaxLineLength + gzip: true, + cache_control_for_etags: @static_cache_control, + headers: %{ + "cache-control" => @static_cache_control + } ) plug(Plug.Static.IndexHtml, at: "/pleroma/admin/") diff --git a/test/plugs/cache_control_test.exs b/test/plugs/cache_control_test.exs new file mode 100644 index 000000000..45151b289 --- /dev/null +++ b/test/plugs/cache_control_test.exs @@ -0,0 +1,20 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2018 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.CacheControlTest do + use Pleroma.Web.ConnCase + alias Plug.Conn + + test "Verify Cache-Control header on static assets", %{conn: conn} do + conn = get(conn, "/index.html") + + assert Conn.get_resp_header(conn, "cache-control") == ["public, no-cache"] + end + + test "Verify Cache-Control header on the API", %{conn: conn} do + conn = get(conn, "/api/v1/instance") + + assert Conn.get_resp_header(conn, "cache-control") == ["max-age=0, private, must-revalidate"] + end +end From bbea5691da67916151a883f09e24da7c2e27d9ba Mon Sep 17 00:00:00 2001 From: Sergey Suprunenko Date: Fri, 24 May 2019 20:34:23 +0000 Subject: [PATCH 057/193] Mention all people in the beginning of DM --- lib/pleroma/formatter.ex | 2 +- test/formatter_test.exs | 10 ++++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/lib/pleroma/formatter.ex b/lib/pleroma/formatter.ex index 3e3b9fe97..607843a5b 100644 --- a/lib/pleroma/formatter.ex +++ b/lib/pleroma/formatter.ex @@ -8,7 +8,7 @@ defmodule Pleroma.Formatter do alias Pleroma.User alias Pleroma.Web.MediaProxy - @safe_mention_regex ~r/^(\s*(?@.+?\s+)+)(?.*)/s + @safe_mention_regex ~r/^(\s*(?(@.+?\s+){1,})+)(?.*)/s @link_regex ~r"((?:http(s)?:\/\/)?[\w.-]+(?:\.[\w\.-]+)+[\w\-\._~%:/?#[\]@!\$&'\(\)\*\+,;=.]+)|[0-9a-z+\-\.]+:[0-9a-z$-_.+!*'(),]+"ui @markdown_characters_regex ~r/(`|\*|_|{|}|[|]|\(|\)|#|\+|-|\.|!)/ diff --git a/test/formatter_test.exs b/test/formatter_test.exs index 47b91b121..bfa673049 100644 --- a/test/formatter_test.exs +++ b/test/formatter_test.exs @@ -184,17 +184,19 @@ defmodule Pleroma.FormatterTest do test "given the 'safe_mention' option, it will only mention people in the beginning" do user = insert(:user) - _other_user = insert(:user) + other_user = insert(:user) third_user = insert(:user) - text = " @#{user.nickname} hey dude i hate @#{third_user.nickname}" + text = " @#{user.nickname} @#{other_user.nickname} hey dudes i hate @#{third_user.nickname}" {expected_text, mentions, [] = _tags} = Formatter.linkify(text, safe_mention: true) - assert mentions == [{"@#{user.nickname}", user}] + assert mentions == [{"@#{user.nickname}", user}, {"@#{other_user.nickname}", other_user}] assert expected_text == "@#{user.nickname} hey dude i hate @#{user.nickname} @#{other_user.nickname} hey dudes i hate @#{third_user.nickname}" end From 9415932af5829f2aa19e362076e0653dd1ce9c5a Mon Sep 17 00:00:00 2001 From: Aaron Tinio Date: Sat, 25 May 2019 08:15:12 +0800 Subject: [PATCH 058/193] Keep nodeinfo available when not federating --- .../web/nodeinfo/nodeinfo_controller.ex | 2 - test/web/node_info_test.exs | 80 ++++--------------- 2 files changed, 16 insertions(+), 66 deletions(-) diff --git a/lib/pleroma/web/nodeinfo/nodeinfo_controller.ex b/lib/pleroma/web/nodeinfo/nodeinfo_controller.ex index 3bf2a0fbc..45f90c579 100644 --- a/lib/pleroma/web/nodeinfo/nodeinfo_controller.ex +++ b/lib/pleroma/web/nodeinfo/nodeinfo_controller.ex @@ -12,8 +12,6 @@ defmodule Pleroma.Web.Nodeinfo.NodeinfoController do alias Pleroma.Web.ActivityPub.MRF alias Pleroma.Web.Federator.Publisher - plug(Pleroma.Web.FederatingPlug) - def schemas(conn, _params) do response = %{ links: [ diff --git a/test/web/node_info_test.exs b/test/web/node_info_test.exs index 2fc42b7cc..be1173513 100644 --- a/test/web/node_info_test.exs +++ b/test/web/node_info_test.exs @@ -7,6 +7,22 @@ defmodule Pleroma.Web.NodeInfoTest do import Pleroma.Factory + test "GET /.well-known/nodeinfo", %{conn: conn} do + links = + conn + |> get("/.well-known/nodeinfo") + |> json_response(200) + |> Map.fetch!("links") + + Enum.each(links, fn link -> + href = Map.fetch!(link, "href") + + conn + |> get(href) + |> json_response(200) + end) + end + test "nodeinfo shows staff accounts", %{conn: conn} do moderator = insert(:user, %{local: true, info: %{is_moderator: true}}) admin = insert(:user, %{local: true, info: %{is_admin: true}}) @@ -32,70 +48,6 @@ defmodule Pleroma.Web.NodeInfoTest do result["metadata"]["restrictedNicknames"] end - test "returns 404 when federation is disabled", %{conn: conn} do - instance = - Application.get_env(:pleroma, :instance) - |> Keyword.put(:federating, false) - - Application.put_env(:pleroma, :instance, instance) - - conn - |> get("/.well-known/nodeinfo") - |> json_response(404) - - conn - |> get("/nodeinfo/2.1.json") - |> json_response(404) - - instance = - Application.get_env(:pleroma, :instance) - |> Keyword.put(:federating, true) - - Application.put_env(:pleroma, :instance, instance) - end - - test "returns 200 when federation is enabled", %{conn: conn} do - conn - |> get("/.well-known/nodeinfo") - |> json_response(200) - - conn - |> get("/nodeinfo/2.1.json") - |> json_response(200) - end - - test "returns 404 when federation is disabled (nodeinfo 2.0)", %{conn: conn} do - instance = - Application.get_env(:pleroma, :instance) - |> Keyword.put(:federating, false) - - Application.put_env(:pleroma, :instance, instance) - - conn - |> get("/.well-known/nodeinfo") - |> json_response(404) - - conn - |> get("/nodeinfo/2.0.json") - |> json_response(404) - - instance = - Application.get_env(:pleroma, :instance) - |> Keyword.put(:federating, true) - - Application.put_env(:pleroma, :instance, instance) - end - - test "returns 200 when federation is enabled (nodeinfo 2.0)", %{conn: conn} do - conn - |> get("/.well-known/nodeinfo") - |> json_response(200) - - conn - |> get("/nodeinfo/2.0.json") - |> json_response(200) - end - test "returns software.repository field in nodeinfo 2.1", %{conn: conn} do conn |> get("/.well-known/nodeinfo") From 9bec891eb4d5d06e6bd84dd2c95259d2c1a4f563 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sat, 25 May 2019 04:24:21 +0000 Subject: [PATCH 059/193] kill @httpoison --- config/config.exs | 1 - lib/pleroma/object/fetcher.ex | 5 ++--- lib/pleroma/reverse_proxy.ex | 5 +++-- lib/pleroma/uploaders/mdii.ex | 5 ++--- lib/pleroma/web/activity_pub/publisher.ex | 5 ++--- .../web/mastodon_api/mastodon_api_controller.ex | 4 ++-- lib/pleroma/web/ostatus/ostatus.ex | 7 +++---- lib/pleroma/web/salmon/salmon.ex | 5 ++--- lib/pleroma/web/web_finger/web_finger.ex | 9 ++++----- lib/pleroma/web/websub/websub.ex | 11 +++++------ 10 files changed, 25 insertions(+), 32 deletions(-) diff --git a/config/config.exs b/config/config.exs index e90821d66..01827fb5f 100644 --- a/config/config.exs +++ b/config/config.exs @@ -186,7 +186,6 @@ config :mime, :types, %{ config :pleroma, :websub, Pleroma.Web.Websub config :pleroma, :ostatus, Pleroma.Web.OStatus -config :pleroma, :httpoison, Pleroma.HTTP config :tesla, adapter: Tesla.Adapter.Hackney # Configures http settings, upstream proxy etc. diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex index bb9388d4f..ca980c629 100644 --- a/lib/pleroma/object/fetcher.ex +++ b/lib/pleroma/object/fetcher.ex @@ -1,4 +1,5 @@ defmodule Pleroma.Object.Fetcher do + alias Pleroma.HTTP alias Pleroma.Object alias Pleroma.Object.Containment alias Pleroma.Web.ActivityPub.Transmogrifier @@ -6,8 +7,6 @@ defmodule Pleroma.Object.Fetcher do require Logger - @httpoison Application.get_env(:pleroma, :httpoison) - defp reinject_object(data) do Logger.debug("Reinjecting object #{data["id"]}") @@ -78,7 +77,7 @@ defmodule Pleroma.Object.Fetcher do with true <- String.starts_with?(id, "http"), {:ok, %{body: body, status: code}} when code in 200..299 <- - @httpoison.get( + HTTP.get( id, [{:Accept, "application/activity+json"}] ), diff --git a/lib/pleroma/reverse_proxy.ex b/lib/pleroma/reverse_proxy.ex index a3f177fec..6e5feb4c3 100644 --- a/lib/pleroma/reverse_proxy.ex +++ b/lib/pleroma/reverse_proxy.ex @@ -3,6 +3,8 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.ReverseProxy do + alias Pleroma.HTTP + @keep_req_headers ~w(accept user-agent accept-encoding cache-control if-modified-since) ++ ~w(if-unmodified-since if-none-match if-range range) @resp_cache_headers ~w(etag date last-modified cache-control) @@ -60,7 +62,6 @@ defmodule Pleroma.ReverseProxy do """ @hackney Application.get_env(:pleroma, :hackney, :hackney) - @httpoison Application.get_env(:pleroma, :httpoison, HTTPoison) @default_hackney_options [] @@ -97,7 +98,7 @@ defmodule Pleroma.ReverseProxy do hackney_opts = @default_hackney_options |> Keyword.merge(Keyword.get(opts, :http, [])) - |> @httpoison.process_request_options() + |> HTTP.process_request_options() req_headers = build_req_headers(conn.req_headers, opts) diff --git a/lib/pleroma/uploaders/mdii.ex b/lib/pleroma/uploaders/mdii.ex index 190ed9f3a..237544337 100644 --- a/lib/pleroma/uploaders/mdii.ex +++ b/lib/pleroma/uploaders/mdii.ex @@ -4,11 +4,10 @@ defmodule Pleroma.Uploaders.MDII do alias Pleroma.Config + alias Pleroma.HTTP @behaviour Pleroma.Uploaders.Uploader - @httpoison Application.get_env(:pleroma, :httpoison) - # MDII-hosted images are never passed through the MediaPlug; only local media. # Delegate to Pleroma.Uploaders.Local def get_file(file) do @@ -25,7 +24,7 @@ defmodule Pleroma.Uploaders.MDII do query = "#{cgi}?#{extension}" with {:ok, %{status: 200, body: body}} <- - @httpoison.post(query, file_data, [], adapter: [pool: :default]) do + HTTP.post(query, file_data, [], adapter: [pool: :default]) do remote_file_name = String.split(body) |> List.first() public_url = "#{files}/#{remote_file_name}.#{extension}" {:ok, {:url, public_url}} diff --git a/lib/pleroma/web/activity_pub/publisher.ex b/lib/pleroma/web/activity_pub/publisher.ex index 11dba87de..8f1399ce6 100644 --- a/lib/pleroma/web/activity_pub/publisher.ex +++ b/lib/pleroma/web/activity_pub/publisher.ex @@ -5,6 +5,7 @@ defmodule Pleroma.Web.ActivityPub.Publisher do alias Pleroma.Activity alias Pleroma.Config + alias Pleroma.HTTP alias Pleroma.Instances alias Pleroma.User alias Pleroma.Web.ActivityPub.Relay @@ -16,8 +17,6 @@ defmodule Pleroma.Web.ActivityPub.Publisher do require Logger - @httpoison Application.get_env(:pleroma, :httpoison) - @moduledoc """ ActivityPub outgoing federation module. """ @@ -63,7 +62,7 @@ defmodule Pleroma.Web.ActivityPub.Publisher do with {:ok, %{status: code}} when code in 200..299 <- result = - @httpoison.post( + HTTP.post( inbox, json, [ diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index 1ec0f30a1..0fe09c285 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -11,6 +11,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do alias Pleroma.Conversation.Participation alias Pleroma.Filter alias Pleroma.Formatter + alias Pleroma.HTTP alias Pleroma.Notification alias Pleroma.Object alias Pleroma.Object.Fetcher @@ -55,7 +56,6 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do when action in [:account_register] ) - @httpoison Application.get_env(:pleroma, :httpoison) @local_mastodon_name "Mastodon-Local" action_fallback(:errors) @@ -1691,7 +1691,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do |> String.replace("{{user}}", user) with {:ok, %{status: 200, body: body}} <- - @httpoison.get( + HTTP.get( url, [], adapter: [ diff --git a/lib/pleroma/web/ostatus/ostatus.ex b/lib/pleroma/web/ostatus/ostatus.ex index 61515b31e..6ed089d84 100644 --- a/lib/pleroma/web/ostatus/ostatus.ex +++ b/lib/pleroma/web/ostatus/ostatus.ex @@ -3,13 +3,12 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.OStatus do - @httpoison Application.get_env(:pleroma, :httpoison) - import Ecto.Query import Pleroma.Web.XML require Logger alias Pleroma.Activity + alias Pleroma.HTTP alias Pleroma.Object alias Pleroma.Repo alias Pleroma.User @@ -363,7 +362,7 @@ defmodule Pleroma.Web.OStatus do def fetch_activity_from_atom_url(url) do with true <- String.starts_with?(url, "http"), {:ok, %{body: body, status: code}} when code in 200..299 <- - @httpoison.get( + HTTP.get( url, [{:Accept, "application/atom+xml"}] ) do @@ -380,7 +379,7 @@ defmodule Pleroma.Web.OStatus do Logger.debug("Trying to fetch #{url}") with true <- String.starts_with?(url, "http"), - {:ok, %{body: body}} <- @httpoison.get(url, []), + {:ok, %{body: body}} <- HTTP.get(url, []), {:ok, atom_url} <- get_atom_url(body) do fetch_activity_from_atom_url(atom_url) else diff --git a/lib/pleroma/web/salmon/salmon.ex b/lib/pleroma/web/salmon/salmon.ex index f25d92fad..9e91a5a40 100644 --- a/lib/pleroma/web/salmon/salmon.ex +++ b/lib/pleroma/web/salmon/salmon.ex @@ -5,11 +5,10 @@ defmodule Pleroma.Web.Salmon do @behaviour Pleroma.Web.Federator.Publisher - @httpoison Application.get_env(:pleroma, :httpoison) - use Bitwise alias Pleroma.Activity + alias Pleroma.HTTP alias Pleroma.Instances alias Pleroma.Keys alias Pleroma.User @@ -138,7 +137,7 @@ defmodule Pleroma.Web.Salmon do def publish_one(%{recipient: url, feed: feed} = params) when is_binary(url) do with {:ok, %{status: code}} when code in 200..299 <- - @httpoison.post( + HTTP.post( url, feed, [{"Content-Type", "application/magic-envelope+xml"}] diff --git a/lib/pleroma/web/web_finger/web_finger.ex b/lib/pleroma/web/web_finger/web_finger.ex index c5b7d4acb..3fca72de8 100644 --- a/lib/pleroma/web/web_finger/web_finger.ex +++ b/lib/pleroma/web/web_finger/web_finger.ex @@ -3,8 +3,7 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.WebFinger do - @httpoison Application.get_env(:pleroma, :httpoison) - + alias Pleroma.HTTP alias Pleroma.User alias Pleroma.Web alias Pleroma.Web.Federator.Publisher @@ -176,11 +175,11 @@ defmodule Pleroma.Web.WebFinger do def find_lrdd_template(domain) do with {:ok, %{status: status, body: body}} when status in 200..299 <- - @httpoison.get("http://#{domain}/.well-known/host-meta", []) do + HTTP.get("http://#{domain}/.well-known/host-meta", []) do get_template_from_xml(body) else _ -> - with {:ok, %{body: body}} <- @httpoison.get("https://#{domain}/.well-known/host-meta", []) do + with {:ok, %{body: body}} <- HTTP.get("https://#{domain}/.well-known/host-meta", []) do get_template_from_xml(body) else e -> {:error, "Can't find LRDD template: #{inspect(e)}"} @@ -209,7 +208,7 @@ defmodule Pleroma.Web.WebFinger do end with response <- - @httpoison.get( + HTTP.get( address, Accept: "application/xrd+xml,application/jrd+json" ), diff --git a/lib/pleroma/web/websub/websub.ex b/lib/pleroma/web/websub/websub.ex index 7ad0414ab..b61f388b8 100644 --- a/lib/pleroma/web/websub/websub.ex +++ b/lib/pleroma/web/websub/websub.ex @@ -5,6 +5,7 @@ defmodule Pleroma.Web.Websub do alias Ecto.Changeset alias Pleroma.Activity + alias Pleroma.HTTP alias Pleroma.Instances alias Pleroma.Repo alias Pleroma.User @@ -24,9 +25,7 @@ defmodule Pleroma.Web.Websub do @behaviour Pleroma.Web.Federator.Publisher - @httpoison Application.get_env(:pleroma, :httpoison) - - def verify(subscription, getter \\ &@httpoison.get/3) do + def verify(subscription, getter \\ &HTTP.get/3) do challenge = Base.encode16(:crypto.strong_rand_bytes(8)) lease_seconds = NaiveDateTime.diff(subscription.valid_until, subscription.updated_at) lease_seconds = lease_seconds |> to_string @@ -207,7 +206,7 @@ defmodule Pleroma.Web.Websub do requester.(subscription) end - def gather_feed_data(topic, getter \\ &@httpoison.get/1) do + def gather_feed_data(topic, getter \\ &HTTP.get/1) do with {:ok, response} <- getter.(topic), status when status in 200..299 <- response.status, body <- response.body, @@ -236,7 +235,7 @@ defmodule Pleroma.Web.Websub do end end - def request_subscription(websub, poster \\ &@httpoison.post/3, timeout \\ 10_000) do + def request_subscription(websub, poster \\ &HTTP.post/3, timeout \\ 10_000) do data = [ "hub.mode": "subscribe", "hub.topic": websub.topic, @@ -294,7 +293,7 @@ defmodule Pleroma.Web.Websub do Logger.info(fn -> "Pushing #{topic} to #{callback}" end) with {:ok, %{status: code}} when code in 200..299 <- - @httpoison.post( + HTTP.post( callback, xml, [ From 80d55d428fedfe4ebc44569c6134908d269db698 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sat, 25 May 2019 04:34:16 +0000 Subject: [PATCH 060/193] tests: websub: check only that signature validation succeeds or fails --- config/test.exs | 4 ++-- test/web/websub/websub_controller_test.exs | 6 +----- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/config/test.exs b/config/test.exs index 6100989c4..3bc407840 100644 --- a/config/test.exs +++ b/config/test.exs @@ -39,8 +39,8 @@ config :pleroma, Pleroma.Repo, # Reduce hash rounds for testing config :pbkdf2_elixir, rounds: 1 -config :pleroma, :websub, Pleroma.Web.WebsubMock -config :pleroma, :ostatus, Pleroma.Web.OStatusMock +#config :pleroma, :websub, Pleroma.Web.WebsubMock +#config :pleroma, :ostatus, Pleroma.Web.OStatusMock config :tesla, adapter: Tesla.Mock config :pleroma, :rich_media, enabled: false diff --git a/test/web/websub/websub_controller_test.exs b/test/web/websub/websub_controller_test.exs index 1e69ed01a..bf2ee31ee 100644 --- a/test/web/websub/websub_controller_test.exs +++ b/test/web/websub/websub_controller_test.exs @@ -52,7 +52,7 @@ defmodule Pleroma.Web.Websub.WebsubControllerTest do end describe "websub_incoming" do - test "handles incoming feed updates", %{conn: conn} do + test "accepts incoming feed updates", %{conn: conn} do websub = insert(:websub_client_subscription) doc = "some stuff" signature = Websub.sign(websub.secret, doc) @@ -64,8 +64,6 @@ defmodule Pleroma.Web.Websub.WebsubControllerTest do |> post("/push/subscriptions/#{websub.id}", doc) assert response(conn, 200) == "OK" - - assert length(Repo.all(Activity)) == 1 end test "rejects incoming feed updates with the wrong signature", %{conn: conn} do @@ -80,8 +78,6 @@ defmodule Pleroma.Web.Websub.WebsubControllerTest do |> post("/push/subscriptions/#{websub.id}", doc) assert response(conn, 500) == "Error" - - assert Enum.empty?(Repo.all(Activity)) end end end From 56fd7dbdd7e3f1a0bb05430ed4805d8715e54935 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sat, 25 May 2019 04:39:32 +0000 Subject: [PATCH 061/193] remove @websub and @ostatus module-level constants --- config/config.exs | 2 -- config/test.exs | 2 -- lib/pleroma/web/federator/federator.ex | 8 +++----- test/support/ostatus_mock.ex | 11 ----------- test/support/websub_mock.ex | 9 --------- test/web/websub/websub_controller_test.exs | 1 - 6 files changed, 3 insertions(+), 30 deletions(-) delete mode 100644 test/support/ostatus_mock.ex delete mode 100644 test/support/websub_mock.ex diff --git a/config/config.exs b/config/config.exs index 01827fb5f..68168b279 100644 --- a/config/config.exs +++ b/config/config.exs @@ -184,8 +184,6 @@ config :mime, :types, %{ "application/ld+json" => ["activity+json"] } -config :pleroma, :websub, Pleroma.Web.Websub -config :pleroma, :ostatus, Pleroma.Web.OStatus config :tesla, adapter: Tesla.Adapter.Hackney # Configures http settings, upstream proxy etc. diff --git a/config/test.exs b/config/test.exs index 3bc407840..41cddb9bd 100644 --- a/config/test.exs +++ b/config/test.exs @@ -39,8 +39,6 @@ config :pleroma, Pleroma.Repo, # Reduce hash rounds for testing config :pbkdf2_elixir, rounds: 1 -#config :pleroma, :websub, Pleroma.Web.WebsubMock -#config :pleroma, :ostatus, Pleroma.Web.OStatusMock config :tesla, adapter: Tesla.Mock config :pleroma, :rich_media, enabled: false diff --git a/lib/pleroma/web/federator/federator.ex b/lib/pleroma/web/federator/federator.ex index 6b0b75284..f4c9fe284 100644 --- a/lib/pleroma/web/federator/federator.ex +++ b/lib/pleroma/web/federator/federator.ex @@ -11,13 +11,11 @@ defmodule Pleroma.Web.Federator do alias Pleroma.Web.ActivityPub.Utils alias Pleroma.Web.Federator.Publisher alias Pleroma.Web.Federator.RetryQueue + alias Pleroma.Web.OStatus alias Pleroma.Web.Websub require Logger - @websub Application.get_env(:pleroma, :websub) - @ostatus Application.get_env(:pleroma, :ostatus) - def init do # 1 minute Process.sleep(1000 * 60) @@ -87,12 +85,12 @@ defmodule Pleroma.Web.Federator do "Running WebSub verification for #{websub.id} (#{websub.topic}, #{websub.callback})" end) - @websub.verify(websub) + Websub.verify(websub) end def perform(:incoming_doc, doc) do Logger.info("Got document, trying to parse") - @ostatus.handle_incoming(doc) + OStatus.handle_incoming(doc) end def perform(:incoming_ap_doc, params) do diff --git a/test/support/ostatus_mock.ex b/test/support/ostatus_mock.ex deleted file mode 100644 index 9c0f2f323..000000000 --- a/test/support/ostatus_mock.ex +++ /dev/null @@ -1,11 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2018 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Web.OStatusMock do - import Pleroma.Factory - - def handle_incoming(_doc) do - insert(:note_activity) - end -end diff --git a/test/support/websub_mock.ex b/test/support/websub_mock.ex deleted file mode 100644 index e3d5a5b16..000000000 --- a/test/support/websub_mock.ex +++ /dev/null @@ -1,9 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2018 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Web.WebsubMock do - def verify(sub) do - {:ok, sub} - end -end diff --git a/test/web/websub/websub_controller_test.exs b/test/web/websub/websub_controller_test.exs index bf2ee31ee..f79745d58 100644 --- a/test/web/websub/websub_controller_test.exs +++ b/test/web/websub/websub_controller_test.exs @@ -5,7 +5,6 @@ defmodule Pleroma.Web.Websub.WebsubControllerTest do use Pleroma.Web.ConnCase import Pleroma.Factory - alias Pleroma.Activity alias Pleroma.Repo alias Pleroma.Web.Websub alias Pleroma.Web.Websub.WebsubClientSubscription From 4030837d91b7ff8525513589d5d810e9a1a6b959 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sat, 25 May 2019 05:19:47 +0000 Subject: [PATCH 062/193] notification: add non_follows/non_followers notification control settings --- lib/pleroma/notification.ex | 32 +++++++++++++++++++++++++++++++- lib/pleroma/user/info.ex | 11 +++++++++-- 2 files changed, 40 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex index 844264307..4095e0474 100644 --- a/lib/pleroma/notification.ex +++ b/lib/pleroma/notification.ex @@ -166,7 +166,17 @@ defmodule Pleroma.Notification do def get_notified_from_activity(_, _local_only), do: [] def skip?(activity, user) do - [:self, :blocked, :local, :muted, :followers, :follows, :recently_followed] + [ + :self, + :blocked, + :local, + :muted, + :followers, + :follows, + :non_followers, + :non_follows, + :recently_followed + ] |> Enum.any?(&skip?(&1, activity, user)) end @@ -201,12 +211,32 @@ defmodule Pleroma.Notification do User.following?(follower, user) end + def skip?( + :non_followers, + activity, + %{info: %{notification_settings: %{"non_followers" => false}}} = user + ) do + actor = activity.data["actor"] + follower = User.get_cached_by_ap_id(actor) + !User.following?(follower, user) + end + def skip?(:follows, activity, %{info: %{notification_settings: %{"follows" => false}}} = user) do actor = activity.data["actor"] followed = User.get_cached_by_ap_id(actor) User.following?(user, followed) end + def skip?( + :non_follows, + activity, + %{info: %{notification_settings: %{"non_follows" => false}}} = user + ) do + actor = activity.data["actor"] + followed = User.get_cached_by_ap_id(actor) + !User.following?(user, followed) + end + def skip?(:recently_followed, %{data: %{"type" => "Follow"}} = activity, user) do actor = activity.data["actor"] diff --git a/lib/pleroma/user/info.ex b/lib/pleroma/user/info.ex index 6397e2737..fb4cf3cc3 100644 --- a/lib/pleroma/user/info.ex +++ b/lib/pleroma/user/info.ex @@ -47,7 +47,14 @@ defmodule Pleroma.User.Info do field(:emoji, {:array, :map}, default: []) field(:notification_settings, :map, - default: %{"remote" => true, "local" => true, "followers" => true, "follows" => true} + default: %{ + "remote" => true, + "local" => true, + "followers" => true, + "follows" => true, + "non_follows" => true, + "non_followers" => true + } ) # Found in the wild @@ -71,7 +78,7 @@ defmodule Pleroma.User.Info do notification_settings = info.notification_settings |> Map.merge(settings) - |> Map.take(["remote", "local", "followers", "follows"]) + |> Map.take(["remote", "local", "followers", "follows", "non_follows", "non_followers"]) params = %{notification_settings: notification_settings} From 1542cccbbcc2935add83d1428b4cd9e4b146f1ec Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sat, 25 May 2019 05:22:13 +0000 Subject: [PATCH 063/193] tests: chase notification setting changes --- test/web/mastodon_api/account_view_test.exs | 4 +++- test/web/twitter_api/util_controller_test.exs | 10 ++++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/test/web/mastodon_api/account_view_test.exs b/test/web/mastodon_api/account_view_test.exs index aaf2261bb..6f8480ee2 100644 --- a/test/web/mastodon_api/account_view_test.exs +++ b/test/web/mastodon_api/account_view_test.exs @@ -81,7 +81,9 @@ defmodule Pleroma.Web.MastodonAPI.AccountViewTest do "remote" => true, "local" => true, "followers" => true, - "follows" => true + "follows" => true, + "non_follows" => true, + "non_followers" => true } privacy = user.info.default_scope diff --git a/test/web/twitter_api/util_controller_test.exs b/test/web/twitter_api/util_controller_test.exs index 2cd82b3e7..ca0b8cc26 100644 --- a/test/web/twitter_api/util_controller_test.exs +++ b/test/web/twitter_api/util_controller_test.exs @@ -110,8 +110,14 @@ defmodule Pleroma.Web.TwitterAPI.UtilControllerTest do user = Repo.get(User, user.id) - assert %{"remote" => false, "local" => true, "followers" => false, "follows" => true} == - user.info.notification_settings + assert %{ + "remote" => false, + "local" => true, + "followers" => false, + "follows" => true, + "non_follows" => true, + "non_followers" => true + } == user.info.notification_settings end end From 0f7eeb0943472da80405639f17b9bc99e4fcd417 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sat, 25 May 2019 05:25:40 +0000 Subject: [PATCH 064/193] tests: add tests for non-follows/non-followers settings --- test/notification_test.exs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/test/notification_test.exs b/test/notification_test.exs index 581db58a8..b54414dcd 100644 --- a/test/notification_test.exs +++ b/test/notification_test.exs @@ -113,6 +113,13 @@ defmodule Pleroma.NotificationTest do assert nil == Notification.create_notification(activity, followed) end + test "it disables notifications from non-followers" do + follower = insert(:user) + followed = insert(:user, info: %{notification_settings: %{"non_followers" => false}}) + {:ok, activity} = CommonAPI.post(follower, %{"status" => "hey @#{followed.nickname}"}) + assert nil == Notification.create_notification(activity, followed) + end + test "it disables notifications from people the user follows" do follower = insert(:user, info: %{notification_settings: %{"follows" => false}}) followed = insert(:user) @@ -122,6 +129,13 @@ defmodule Pleroma.NotificationTest do assert nil == Notification.create_notification(activity, follower) end + test "it disables notifications from people the user does not follow" do + follower = insert(:user, info: %{notification_settings: %{"non_follows" => false}}) + followed = insert(:user) + {:ok, activity} = CommonAPI.post(followed, %{"status" => "hey @#{follower.nickname}"}) + assert nil == Notification.create_notification(activity, follower) + end + test "it doesn't create a notification for user if he is the activity author" do activity = insert(:note_activity) author = User.get_cached_by_ap_id(activity.data["actor"]) From 59a703fcbe6436c92d0e276caaf55f599c3165f4 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sat, 25 May 2019 05:31:13 +0000 Subject: [PATCH 065/193] twitter api: user view: expose user notification settings under pleroma object --- lib/pleroma/web/twitter_api/views/user_view.ex | 7 +++++++ test/web/twitter_api/views/user_view_test.exs | 2 ++ 2 files changed, 9 insertions(+) diff --git a/lib/pleroma/web/twitter_api/views/user_view.ex b/lib/pleroma/web/twitter_api/views/user_view.ex index f0a4ddbd3..550f35f5f 100644 --- a/lib/pleroma/web/twitter_api/views/user_view.ex +++ b/lib/pleroma/web/twitter_api/views/user_view.ex @@ -121,6 +121,7 @@ defmodule Pleroma.Web.TwitterAPI.UserView do "tags" => user.tags } |> maybe_with_activation_status(user, for_user) + |> with_notification_settings(user, for_user) } |> maybe_with_user_settings(user, for_user) |> maybe_with_role(user, for_user) @@ -132,6 +133,12 @@ defmodule Pleroma.Web.TwitterAPI.UserView do end end + defp with_notification_settings(data, %User{id: user_id} = user, %User{id: user_id}) do + Map.put(data, "notification_settings", user.info.notification_settings) + end + + defp with_notification_settings(data, _, _), do: data + defp maybe_with_activation_status(data, user, %User{info: %{is_admin: true}}) do Map.put(data, "deactivated", user.info.deactivated) end diff --git a/test/web/twitter_api/views/user_view_test.exs b/test/web/twitter_api/views/user_view_test.exs index 74526673c..a48fc9b78 100644 --- a/test/web/twitter_api/views/user_view_test.exs +++ b/test/web/twitter_api/views/user_view_test.exs @@ -112,9 +112,11 @@ defmodule Pleroma.Web.TwitterAPI.UserViewTest do as_user = UserView.render("show.json", %{user: user, for: user}) assert as_user["default_scope"] == user.info.default_scope assert as_user["no_rich_text"] == user.info.no_rich_text + assert as_user["pleroma"]["notification_settings"] == user.info.notification_settings as_stranger = UserView.render("show.json", %{user: user}) refute as_stranger["default_scope"] refute as_stranger["no_rich_text"] + refute as_stranger["pleroma"]["notification_settings"] end test "A user for a given other follower", %{user: user} do From e7e2e7a1a633e2f7e493d040e290c931320d8cc8 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sat, 25 May 2019 05:54:02 +0000 Subject: [PATCH 066/193] user info: allow formdata for notification settings like every other API --- lib/pleroma/user/info.ex | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/pleroma/user/info.ex b/lib/pleroma/user/info.ex index fb4cf3cc3..b0bfdf4f4 100644 --- a/lib/pleroma/user/info.ex +++ b/lib/pleroma/user/info.ex @@ -75,6 +75,11 @@ defmodule Pleroma.User.Info do end def update_notification_settings(info, settings) do + settings = + settings + |> Enum.map(fn {k, v} -> {k, v in [true, "true", "True", "1"]} end) + |> Map.new() + notification_settings = info.notification_settings |> Map.merge(settings) From 5fbbc57c1b497f8d194f3bbdefd889f0a943525a Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sat, 25 May 2019 07:25:13 +0000 Subject: [PATCH 067/193] add migration to add notification settings to user accounts --- ...d_non_followers_fields_to_notification_settings.exs | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 priv/repo/migrations/20190525071417_add_non_follows_and_non_followers_fields_to_notification_settings.exs diff --git a/priv/repo/migrations/20190525071417_add_non_follows_and_non_followers_fields_to_notification_settings.exs b/priv/repo/migrations/20190525071417_add_non_follows_and_non_followers_fields_to_notification_settings.exs new file mode 100644 index 000000000..a88b0ea61 --- /dev/null +++ b/priv/repo/migrations/20190525071417_add_non_follows_and_non_followers_fields_to_notification_settings.exs @@ -0,0 +1,10 @@ +defmodule Pleroma.Repo.Migrations.AddNonFollowsAndNonFollowersFieldsToNotificationSettings do + use Ecto.Migration + + def change do + execute(""" + update users set info = jsonb_set(info, '{notification_settings}', '{"local": true, "remote": true, "follows": true, "followers": true, "non_follows": true, "non_followers": true}') + where local=true + """) + end +end From 750ede5764d30063587182696f7ebc50c05f8278 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sun, 26 May 2019 00:05:47 +0000 Subject: [PATCH 068/193] notification: remove local/remote match rules (too complicated) --- lib/pleroma/notification.ex | 7 ------- lib/pleroma/user/info.ex | 4 +--- 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex index 4095e0474..35beffb87 100644 --- a/lib/pleroma/notification.ex +++ b/lib/pleroma/notification.ex @@ -169,7 +169,6 @@ defmodule Pleroma.Notification do [ :self, :blocked, - :local, :muted, :followers, :follows, @@ -189,12 +188,6 @@ defmodule Pleroma.Notification do User.blocks?(user, %{ap_id: actor}) end - def skip?(:local, %{local: true}, %{info: %{notification_settings: %{"local" => false}}}), - do: true - - def skip?(:local, %{local: false}, %{info: %{notification_settings: %{"remote" => false}}}), - do: true - def skip?(:muted, activity, user) do actor = activity.data["actor"] diff --git a/lib/pleroma/user/info.ex b/lib/pleroma/user/info.ex index b0bfdf4f4..74573ba83 100644 --- a/lib/pleroma/user/info.ex +++ b/lib/pleroma/user/info.ex @@ -48,8 +48,6 @@ defmodule Pleroma.User.Info do field(:notification_settings, :map, default: %{ - "remote" => true, - "local" => true, "followers" => true, "follows" => true, "non_follows" => true, @@ -83,7 +81,7 @@ defmodule Pleroma.User.Info do notification_settings = info.notification_settings |> Map.merge(settings) - |> Map.take(["remote", "local", "followers", "follows", "non_follows", "non_followers"]) + |> Map.take(["followers", "follows", "non_follows", "non_followers"]) params = %{notification_settings: notification_settings} From 45e4642a58f5299d2cd3f142aea110a474eb477f Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sun, 26 May 2019 00:20:54 +0000 Subject: [PATCH 069/193] tests: chase remote/local removal --- test/notification_test.exs | 27 ------------------- test/web/mastodon_api/account_view_test.exs | 2 -- test/web/twitter_api/util_controller_test.exs | 3 --- 3 files changed, 32 deletions(-) diff --git a/test/notification_test.exs b/test/notification_test.exs index b54414dcd..be292abd9 100644 --- a/test/notification_test.exs +++ b/test/notification_test.exs @@ -78,33 +78,6 @@ defmodule Pleroma.NotificationTest do assert nil == Notification.create_notification(activity, muter) end - test "it disables notifications from people on remote instances" do - user = insert(:user, info: %{notification_settings: %{"remote" => false}}) - other_user = insert(:user) - - create_activity = %{ - "@context" => "https://www.w3.org/ns/activitystreams", - "type" => "Create", - "to" => ["https://www.w3.org/ns/activitystreams#Public"], - "actor" => other_user.ap_id, - "object" => %{ - "type" => "Note", - "content" => "Hi @#{user.nickname}", - "attributedTo" => other_user.ap_id - } - } - - {:ok, %{local: false} = activity} = Transmogrifier.handle_incoming(create_activity) - assert nil == Notification.create_notification(activity, user) - end - - test "it disables notifications from people on the local instance" do - user = insert(:user, info: %{notification_settings: %{"local" => false}}) - other_user = insert(:user) - {:ok, activity} = CommonAPI.post(other_user, %{"status" => "hey @#{user.nickname}"}) - assert nil == Notification.create_notification(activity, user) - end - test "it disables notifications from followers" do follower = insert(:user) followed = insert(:user, info: %{notification_settings: %{"followers" => false}}) diff --git a/test/web/mastodon_api/account_view_test.exs b/test/web/mastodon_api/account_view_test.exs index 6f8480ee2..23f250990 100644 --- a/test/web/mastodon_api/account_view_test.exs +++ b/test/web/mastodon_api/account_view_test.exs @@ -78,8 +78,6 @@ defmodule Pleroma.Web.MastodonAPI.AccountViewTest do user = insert(:user) notification_settings = %{ - "remote" => true, - "local" => true, "followers" => true, "follows" => true, "non_follows" => true, diff --git a/test/web/twitter_api/util_controller_test.exs b/test/web/twitter_api/util_controller_test.exs index ca0b8cc26..cab9e5d90 100644 --- a/test/web/twitter_api/util_controller_test.exs +++ b/test/web/twitter_api/util_controller_test.exs @@ -102,7 +102,6 @@ defmodule Pleroma.Web.TwitterAPI.UtilControllerTest do conn |> assign(:user, user) |> put("/api/pleroma/notification_settings", %{ - "remote" => false, "followers" => false, "bar" => 1 }) @@ -111,8 +110,6 @@ defmodule Pleroma.Web.TwitterAPI.UtilControllerTest do user = Repo.get(User, user.id) assert %{ - "remote" => false, - "local" => true, "followers" => false, "follows" => true, "non_follows" => true, From 79503ce90f6d85f00ee9e2dbc6358df2237d5036 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sun, 26 May 2019 01:57:22 +0000 Subject: [PATCH 070/193] mrf: simple policy: fix matching imported activitypub and ostatus statuses --- CHANGELOG.md | 1 + .../web/activity_pub/mrf/simple_policy.ex | 3 +-- .../activity_pub/mrf/simple_policy_test.exs | 18 ++++++++++++++++++ 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7869e299b..f689160e9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -113,6 +113,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Mastodon API: Exposing default scope of the user to anyone - Mastodon API: Make `irreversible` field default to `false` [`POST /api/v1/filters`] - User-Agent is now sent correctly for all HTTP requests. +- MRF: Simple policy now properly delists imported or relayed statuses ## Removed - Configuration: `config :pleroma, :fe` in favor of the more flexible `config :pleroma, :frontend_configurations` diff --git a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex index 890d70a7a..433d23c5f 100644 --- a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex @@ -74,8 +74,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do actor_host ), user <- User.get_cached_by_ap_id(object["actor"]), - true <- "https://www.w3.org/ns/activitystreams#Public" in object["to"], - true <- user.follower_address in object["cc"] do + true <- "https://www.w3.org/ns/activitystreams#Public" in object["to"] do to = List.delete(object["to"], "https://www.w3.org/ns/activitystreams#Public") ++ [user.follower_address] diff --git a/test/web/activity_pub/mrf/simple_policy_test.exs b/test/web/activity_pub/mrf/simple_policy_test.exs index 3d1f26e60..0fd68e103 100644 --- a/test/web/activity_pub/mrf/simple_policy_test.exs +++ b/test/web/activity_pub/mrf/simple_policy_test.exs @@ -145,6 +145,24 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicyTest do assert SimplePolicy.filter(local_message) == {:ok, local_message} end + + test "has a matching host but only as:Public in to" do + {_actor, ftl_message} = build_ftl_actor_and_message() + + ftl_message_actor_host = + ftl_message + |> Map.fetch!("actor") + |> URI.parse() + |> Map.fetch!(:host) + + ftl_message = Map.put(ftl_message, "cc", []) + + Config.put([:mrf_simple, :federated_timeline_removal], [ftl_message_actor_host]) + + assert {:ok, ftl_message} = SimplePolicy.filter(ftl_message) + refute "https://www.w3.org/ns/activitystreams#Public" in ftl_message["to"] + assert "https://www.w3.org/ns/activitystreams#Public" in ftl_message["cc"] + end end defp build_ftl_actor_and_message do From 9f3bcf0efe7fa06e2b8970386c099c1ea2974d0a Mon Sep 17 00:00:00 2001 From: jeff Date: Tue, 28 May 2019 06:49:53 +0000 Subject: [PATCH 071/193] Respect proxy settings federation --- lib/pleroma/http/connection.ex | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/pleroma/http/connection.ex b/lib/pleroma/http/connection.ex index 558005c19..c216cdcb1 100644 --- a/lib/pleroma/http/connection.ex +++ b/lib/pleroma/http/connection.ex @@ -32,9 +32,11 @@ defmodule Pleroma.HTTP.Connection do defp hackney_options(opts) do options = Keyword.get(opts, :adapter, []) adapter_options = Pleroma.Config.get([:http, :adapter], []) + proxy_url = Pleroma.Config.get([:http, :proxy_url], nil) @hackney_options |> Keyword.merge(adapter_options) |> Keyword.merge(options) + |> Keyword.merge(proxy: proxy_url) end end From abc15b6dcca39f62f043fcaf0292443a3dcb3d76 Mon Sep 17 00:00:00 2001 From: feld Date: Tue, 28 May 2019 21:20:24 +0000 Subject: [PATCH 072/193] Improve Varnish config. We set sane headers from the backend now. --- installation/pleroma.vcl | 70 ++++++++++++++++++++-------------------- 1 file changed, 35 insertions(+), 35 deletions(-) diff --git a/installation/pleroma.vcl b/installation/pleroma.vcl index 92153d8ef..154747aa6 100644 --- a/installation/pleroma.vcl +++ b/installation/pleroma.vcl @@ -1,4 +1,4 @@ -vcl 4.0; +vcl 4.1; import std; backend default { @@ -35,24 +35,6 @@ sub vcl_recv { } return(purge); } - - # Pleroma MediaProxy - strip headers that will affect caching - if (req.url ~ "^/proxy/") { - unset req.http.Cookie; - unset req.http.Authorization; - unset req.http.Accept; - return (hash); - } - - # Strip headers that will affect caching from all other static content - # This also permits caching of individual toots and AP Activities - if ((req.url ~ "^/(media|static)/") || - (req.url ~ "(?i)\.(html|js|css|jpg|jpeg|png|gif|gz|tgz|bz2|tbz|mp3|mp4|ogg|webm|svg|swf|ttf|pdf|woff|woff2)$")) - { - unset req.http.Cookie; - unset req.http.Authorization; - return (hash); - } } sub vcl_backend_response { @@ -61,6 +43,12 @@ sub vcl_backend_response { set beresp.do_gzip = true; } + # Retry broken backend responses. + if (beresp.status == 503) { + set bereq.http.X-Varnish-Backend-503 = "1"; + return (retry); + } + # CHUNKED SUPPORT if (bereq.http.x-range ~ "bytes=" && beresp.status == 206) { set beresp.ttl = 10m; @@ -73,8 +61,6 @@ sub vcl_backend_response { return (deliver); } - # Default object caching of 86400s; - set beresp.ttl = 86400s; # Allow serving cached content for 6h in case backend goes down set beresp.grace = 6h; @@ -90,20 +76,6 @@ sub vcl_backend_response { set beresp.ttl = 30s; return (deliver); } - - # Pleroma MediaProxy internally sets headers properly - if (bereq.url ~ "^/proxy/") { - return (deliver); - } - - # Strip cache-restricting headers from Pleroma on static content that we want to cache - if (bereq.url ~ "(?i)\.(js|css|jpg|jpeg|png|gif|gz|tgz|bz2|tbz|mp3|mp4|ogg|webm|svg|swf|ttf|pdf|woff|woff2)$") - { - unset beresp.http.set-cookie; - unset beresp.http.Cache-Control; - unset beresp.http.x-request-id; - set beresp.http.Cache-Control = "public, max-age=86400"; - } } # The synthetic response for 301 redirects @@ -132,10 +104,32 @@ sub vcl_hash { } sub vcl_backend_fetch { + # Be more lenient for slow servers on the fediverse + if bereq.url ~ "^/proxy/" { + set bereq.first_byte_timeout = 300s; + } + # CHUNKED SUPPORT if (bereq.http.x-range) { set bereq.http.Range = bereq.http.x-range; } + + if (bereq.retries == 0) { + # Clean up the X-Varnish-Backend-503 flag that is used internally + # to mark broken backend responses that should be retried. + unset bereq.http.X-Varnish-Backend-503; + } else { + if (bereq.http.X-Varnish-Backend-503) { + if (bereq.method != "POST" && + std.healthy(bereq.backend) && + bereq.retries <= 4) { + # Flush broken backend response flag & try again. + unset bereq.http.X-Varnish-Backend-503; + } else { + return (abandon); + } + } + } } sub vcl_deliver { @@ -145,3 +139,9 @@ sub vcl_deliver { unset resp.http.CR; } } + +sub vcl_backend_error { + # Retry broken backend responses. + set bereq.http.X-Varnish-Backend-503 = "1"; + return (retry); +} From 0159a6dbe97330150d2913c7d7a060151f83f7eb Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Wed, 29 May 2019 10:58:45 +0000 Subject: [PATCH 073/193] router: require oauth_read for searching Search calls are generally expensive and allow unauthenticated users to crawl the instance for user profiles or posts which contain specified keywords. An adversary can build a distributed search engine which not only will consume significant instance resources, but also can be used for undesirable purposes such as datamining. Accordingly, require authenticated access to use the search API endpoints. This acts as a nice balance as it allows guest users to make use of most functionality available in Pleroma FE while ensuring that Pleroma instances are reasonably protected from resource exhaustion. It also removes Pleroma as a potential vector in distributed search engines. --- lib/pleroma/web/router.ex | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 352268b96..08c74a742 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -414,7 +414,12 @@ defmodule Pleroma.Web.Router do get("/trends", MastodonAPIController, :empty_array) - get("/accounts/search", MastodonAPIController, :account_search) + scope [] do + pipe_through(:oauth_read) + + get("/search", MastodonAPIController, :search) + get("/accounts/search", MastodonAPIController, :account_search) + end scope [] do pipe_through(:oauth_read_or_public) @@ -431,14 +436,12 @@ defmodule Pleroma.Web.Router do get("/accounts/:id/following", MastodonAPIController, :following) get("/accounts/:id", MastodonAPIController, :user) - get("/search", MastodonAPIController, :search) - get("/pleroma/accounts/:id/favourites", MastodonAPIController, :user_favourites) end end scope "/api/v2", Pleroma.Web.MastodonAPI do - pipe_through([:api, :oauth_read_or_public]) + pipe_through([:api, :oauth_read]) get("/search", MastodonAPIController, :search2) end @@ -480,9 +483,14 @@ defmodule Pleroma.Web.Router do get("/statuses/show/:id", TwitterAPI.Controller, :fetch_status) get("/statusnet/conversation/:id", TwitterAPI.Controller, :fetch_conversation) - get("/search", TwitterAPI.Controller, :search) get("/statusnet/tags/timeline/:tag", TwitterAPI.Controller, :public_and_external_timeline) end + + scope [] do + pipe_through(:oauth_read) + + get("/search", TwitterAPI.Controller, :search) + end end scope "/api", Pleroma.Web do @@ -500,7 +508,7 @@ defmodule Pleroma.Web.Router do end scope "/api", Pleroma.Web, as: :twitter_api_search do - pipe_through([:api, :oauth_read_or_public]) + pipe_through([:api, :oauth_read]) get("/pleroma/search_user", TwitterAPI.Controller, :search_user) end From 672fddb7213b69f3ee8b9a7728426373090847ed Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 29 May 2019 08:06:26 -0500 Subject: [PATCH 074/193] Default search limit should be 40 https://docs.joinmastodon.org/api/rest/search/ --- lib/pleroma/web/mastodon_api/mastodon_api_controller.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index 0fe09c285..2110027c3 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -1084,7 +1084,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do from([a, o] in Activity.with_preloaded_object(Activity), where: fragment("?->>'type' = 'Create'", a.data), where: "https://www.w3.org/ns/activitystreams#Public" in a.recipients, - limit: 20 + limit: 40 ) q = From db94294dfb6f25ce0f08acfb78b3c4cc2241e9f5 Mon Sep 17 00:00:00 2001 From: Maksim Date: Wed, 29 May 2019 14:04:58 +0000 Subject: [PATCH 075/193] [#936] fix tests --- .../admin_api/admin_api_controller_test.exs | 176 ++++++++++-------- 1 file changed, 96 insertions(+), 80 deletions(-) diff --git a/test/web/admin_api/admin_api_controller_test.exs b/test/web/admin_api/admin_api_controller_test.exs index c15c67e31..43dcf945a 100644 --- a/test/web/admin_api/admin_api_controller_test.exs +++ b/test/web/admin_api/admin_api_controller_test.exs @@ -437,27 +437,31 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do user = insert(:user, local: false, tags: ["foo", "bar"]) conn = get(conn, "/api/pleroma/admin/users?page=1") + users = + [ + %{ + "deactivated" => admin.info.deactivated, + "id" => admin.id, + "nickname" => admin.nickname, + "roles" => %{"admin" => true, "moderator" => false}, + "local" => true, + "tags" => [] + }, + %{ + "deactivated" => user.info.deactivated, + "id" => user.id, + "nickname" => user.nickname, + "roles" => %{"admin" => false, "moderator" => false}, + "local" => false, + "tags" => ["foo", "bar"] + } + ] + |> Enum.sort_by(& &1["nickname"]) + assert json_response(conn, 200) == %{ "count" => 2, "page_size" => 50, - "users" => [ - %{ - "deactivated" => admin.info.deactivated, - "id" => admin.id, - "nickname" => admin.nickname, - "roles" => %{"admin" => true, "moderator" => false}, - "local" => true, - "tags" => [] - }, - %{ - "deactivated" => user.info.deactivated, - "id" => user.id, - "nickname" => user.nickname, - "roles" => %{"admin" => false, "moderator" => false}, - "local" => false, - "tags" => ["foo", "bar"] - } - ] + "users" => users } end @@ -659,35 +663,39 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do |> assign(:user, admin) |> get("/api/pleroma/admin/users?filters=local") + users = + [ + %{ + "deactivated" => user.info.deactivated, + "id" => user.id, + "nickname" => user.nickname, + "roles" => %{"admin" => false, "moderator" => false}, + "local" => true, + "tags" => [] + }, + %{ + "deactivated" => admin.info.deactivated, + "id" => admin.id, + "nickname" => admin.nickname, + "roles" => %{"admin" => true, "moderator" => false}, + "local" => true, + "tags" => [] + }, + %{ + "deactivated" => false, + "id" => old_admin.id, + "local" => true, + "nickname" => old_admin.nickname, + "roles" => %{"admin" => true, "moderator" => false}, + "tags" => [] + } + ] + |> Enum.sort_by(& &1["nickname"]) + assert json_response(conn, 200) == %{ "count" => 3, "page_size" => 50, - "users" => [ - %{ - "deactivated" => user.info.deactivated, - "id" => user.id, - "nickname" => user.nickname, - "roles" => %{"admin" => false, "moderator" => false}, - "local" => true, - "tags" => [] - }, - %{ - "deactivated" => admin.info.deactivated, - "id" => admin.id, - "nickname" => admin.nickname, - "roles" => %{"admin" => true, "moderator" => false}, - "local" => true, - "tags" => [] - }, - %{ - "deactivated" => false, - "id" => old_admin.id, - "local" => true, - "nickname" => old_admin.nickname, - "roles" => %{"admin" => true, "moderator" => false}, - "tags" => [] - } - ] + "users" => users } end @@ -698,27 +706,31 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do conn = get(conn, "/api/pleroma/admin/users?filters=is_admin") + users = + [ + %{ + "deactivated" => false, + "id" => admin.id, + "nickname" => admin.nickname, + "roles" => %{"admin" => true, "moderator" => false}, + "local" => admin.local, + "tags" => [] + }, + %{ + "deactivated" => false, + "id" => second_admin.id, + "nickname" => second_admin.nickname, + "roles" => %{"admin" => true, "moderator" => false}, + "local" => second_admin.local, + "tags" => [] + } + ] + |> Enum.sort_by(& &1["nickname"]) + assert json_response(conn, 200) == %{ "count" => 2, "page_size" => 50, - "users" => [ - %{ - "deactivated" => false, - "id" => admin.id, - "nickname" => admin.nickname, - "roles" => %{"admin" => true, "moderator" => false}, - "local" => admin.local, - "tags" => [] - }, - %{ - "deactivated" => false, - "id" => second_admin.id, - "nickname" => second_admin.nickname, - "roles" => %{"admin" => true, "moderator" => false}, - "local" => second_admin.local, - "tags" => [] - } - ] + "users" => users } end @@ -753,27 +765,31 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do conn = get(conn, "/api/pleroma/admin/users?tags[]=first&tags[]=second") + users = + [ + %{ + "deactivated" => false, + "id" => user1.id, + "nickname" => user1.nickname, + "roles" => %{"admin" => false, "moderator" => false}, + "local" => user1.local, + "tags" => ["first"] + }, + %{ + "deactivated" => false, + "id" => user2.id, + "nickname" => user2.nickname, + "roles" => %{"admin" => false, "moderator" => false}, + "local" => user2.local, + "tags" => ["second"] + } + ] + |> Enum.sort_by(& &1["nickname"]) + assert json_response(conn, 200) == %{ "count" => 2, "page_size" => 50, - "users" => [ - %{ - "deactivated" => false, - "id" => user1.id, - "nickname" => user1.nickname, - "roles" => %{"admin" => false, "moderator" => false}, - "local" => user1.local, - "tags" => ["first"] - }, - %{ - "deactivated" => false, - "id" => user2.id, - "nickname" => user2.nickname, - "roles" => %{"admin" => false, "moderator" => false}, - "local" => user2.local, - "tags" => ["second"] - } - ] + "users" => users } end From 6aec0d1b58c12db2fa669fcff043338174290a35 Mon Sep 17 00:00:00 2001 From: kaniini Date: Wed, 29 May 2019 22:10:16 +0000 Subject: [PATCH 076/193] Revert "Merge branch 'feature/search-authenticated-only' into 'develop'" This reverts merge request !1209 --- lib/pleroma/web/router.ex | 20 ++++++-------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 08c74a742..352268b96 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -414,12 +414,7 @@ defmodule Pleroma.Web.Router do get("/trends", MastodonAPIController, :empty_array) - scope [] do - pipe_through(:oauth_read) - - get("/search", MastodonAPIController, :search) - get("/accounts/search", MastodonAPIController, :account_search) - end + get("/accounts/search", MastodonAPIController, :account_search) scope [] do pipe_through(:oauth_read_or_public) @@ -436,12 +431,14 @@ defmodule Pleroma.Web.Router do get("/accounts/:id/following", MastodonAPIController, :following) get("/accounts/:id", MastodonAPIController, :user) + get("/search", MastodonAPIController, :search) + get("/pleroma/accounts/:id/favourites", MastodonAPIController, :user_favourites) end end scope "/api/v2", Pleroma.Web.MastodonAPI do - pipe_through([:api, :oauth_read]) + pipe_through([:api, :oauth_read_or_public]) get("/search", MastodonAPIController, :search2) end @@ -483,13 +480,8 @@ defmodule Pleroma.Web.Router do get("/statuses/show/:id", TwitterAPI.Controller, :fetch_status) get("/statusnet/conversation/:id", TwitterAPI.Controller, :fetch_conversation) - get("/statusnet/tags/timeline/:tag", TwitterAPI.Controller, :public_and_external_timeline) - end - - scope [] do - pipe_through(:oauth_read) - get("/search", TwitterAPI.Controller, :search) + get("/statusnet/tags/timeline/:tag", TwitterAPI.Controller, :public_and_external_timeline) end end @@ -508,7 +500,7 @@ defmodule Pleroma.Web.Router do end scope "/api", Pleroma.Web, as: :twitter_api_search do - pipe_through([:api, :oauth_read]) + pipe_through([:api, :oauth_read_or_public]) get("/pleroma/search_user", TwitterAPI.Controller, :search_user) end From 99f70c7e2011ab29746a8a61d3a4b354f4513045 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Thu, 30 May 2019 15:33:58 +0700 Subject: [PATCH 077/193] Use Pleroma.Config everywhere --- lib/healthcheck.ex | 4 +- lib/pleroma/emoji.ex | 4 +- lib/pleroma/html.ex | 20 ++---- lib/pleroma/http/http.ex | 7 +- lib/pleroma/plugs/federating_plug.ex | 2 +- lib/pleroma/reverse_proxy.ex | 2 +- lib/pleroma/user.ex | 4 +- lib/pleroma/web/activity_pub/activity_pub.ex | 12 ++-- .../activity_pub/activity_pub_controller.ex | 2 +- lib/pleroma/web/activity_pub/mrf.ex | 4 +- lib/pleroma/web/endpoint.ex | 2 +- lib/pleroma/web/media_proxy/media_proxy.ex | 34 +++++----- .../web/nodeinfo/nodeinfo_controller.ex | 66 ++++++++----------- lib/pleroma/web/templates/layout/app.html.eex | 4 +- .../mastodon_api/mastodon/index.html.eex | 2 +- .../web/twitter_api/twitter_api_controller.ex | 2 +- test/web/plugs/federating_plug_test.exs | 12 +--- .../twitter_api_controller_test.exs | 48 +++----------- 18 files changed, 80 insertions(+), 151 deletions(-) diff --git a/lib/healthcheck.ex b/lib/healthcheck.ex index 646fb3b9d..32aafc210 100644 --- a/lib/healthcheck.ex +++ b/lib/healthcheck.ex @@ -29,13 +29,13 @@ defmodule Pleroma.Healthcheck do end defp assign_db_info(healthcheck) do - database = Application.get_env(:pleroma, Repo)[:database] + database = Pleroma.Config.get([Repo, :database]) query = "select state, count(pid) from pg_stat_activity where datname = '#{database}' group by state;" result = Repo.query!(query) - pool_size = Application.get_env(:pleroma, Repo)[:pool_size] + pool_size = Pleroma.Config.get([Repo, :pool_size]) db_info = Enum.reduce(result.rows, %{active: 0, idle: 0}, fn [state, cnt], states -> diff --git a/lib/pleroma/emoji.ex b/lib/pleroma/emoji.ex index 6390cce4c..7d12eff7f 100644 --- a/lib/pleroma/emoji.ex +++ b/lib/pleroma/emoji.ex @@ -22,7 +22,7 @@ defmodule Pleroma.Emoji do @ets __MODULE__.Ets @ets_options [:ordered_set, :protected, :named_table, {:read_concurrency, true}] - @groups Application.get_env(:pleroma, :emoji)[:groups] + @groups Pleroma.Config.get([:emoji, :groups]) @doc false def start_link do @@ -112,7 +112,7 @@ defmodule Pleroma.Emoji do # Compat thing for old custom emoji handling & default emoji, # it should run even if there are no emoji packs - shortcode_globs = Application.get_env(:pleroma, :emoji)[:shortcode_globs] || [] + shortcode_globs = Pleroma.Config.get([:emoji, :shortcode_globs], []) emojis = (load_from_file("config/emoji.txt") ++ diff --git a/lib/pleroma/html.ex b/lib/pleroma/html.ex index d1da746de..e5e78ee4f 100644 --- a/lib/pleroma/html.ex +++ b/lib/pleroma/html.ex @@ -104,7 +104,6 @@ defmodule Pleroma.HTML.Scrubber.TwitterText do paragraphs, breaks and links are allowed through the filter. """ - @markup Application.get_env(:pleroma, :markup) @valid_schemes Pleroma.Config.get([:uri_schemes, :valid_schemes], []) require HtmlSanitizeEx.Scrubber.Meta @@ -142,9 +141,7 @@ defmodule Pleroma.HTML.Scrubber.TwitterText do Meta.allow_tag_with_these_attributes("span", []) # allow inline images for custom emoji - @allow_inline_images Keyword.get(@markup, :allow_inline_images) - - if @allow_inline_images do + if Pleroma.Config.get([:markup, :allow_inline_images]) do # restrict img tags to http/https only, because of MediaProxy. Meta.allow_tag_with_uri_attributes("img", ["src"], ["http", "https"]) @@ -168,7 +165,6 @@ defmodule Pleroma.HTML.Scrubber.Default do # credo:disable-for-previous-line # No idea how to fix this one… - @markup Application.get_env(:pleroma, :markup) @valid_schemes Pleroma.Config.get([:uri_schemes, :valid_schemes], []) Meta.remove_cdata_sections_before_scrub() @@ -213,7 +209,7 @@ defmodule Pleroma.HTML.Scrubber.Default do Meta.allow_tag_with_this_attribute_values("span", "class", ["h-card"]) Meta.allow_tag_with_these_attributes("span", []) - @allow_inline_images Keyword.get(@markup, :allow_inline_images) + @allow_inline_images Pleroma.Config.get([:markup, :allow_inline_images]) if @allow_inline_images do # restrict img tags to http/https only, because of MediaProxy. @@ -228,9 +224,7 @@ defmodule Pleroma.HTML.Scrubber.Default do ]) end - @allow_tables Keyword.get(@markup, :allow_tables) - - if @allow_tables do + if Pleroma.Config.get([:markup, :allow_tables]) do Meta.allow_tag_with_these_attributes("table", []) Meta.allow_tag_with_these_attributes("tbody", []) Meta.allow_tag_with_these_attributes("td", []) @@ -239,9 +233,7 @@ defmodule Pleroma.HTML.Scrubber.Default do Meta.allow_tag_with_these_attributes("tr", []) end - @allow_headings Keyword.get(@markup, :allow_headings) - - if @allow_headings do + if Pleroma.Config.get([:markup, :allow_headings]) do Meta.allow_tag_with_these_attributes("h1", []) Meta.allow_tag_with_these_attributes("h2", []) Meta.allow_tag_with_these_attributes("h3", []) @@ -249,9 +241,7 @@ defmodule Pleroma.HTML.Scrubber.Default do Meta.allow_tag_with_these_attributes("h5", []) end - @allow_fonts Keyword.get(@markup, :allow_fonts) - - if @allow_fonts do + if Pleroma.Config.get([:markup, :allow_fonts]) do Meta.allow_tag_with_these_attributes("font", ["face"]) end diff --git a/lib/pleroma/http/http.ex b/lib/pleroma/http/http.ex index c5f720bc9..c96ee7353 100644 --- a/lib/pleroma/http/http.ex +++ b/lib/pleroma/http/http.ex @@ -65,12 +65,9 @@ defmodule Pleroma.HTTP do end def process_request_options(options) do - config = Application.get_env(:pleroma, :http, []) - proxy = Keyword.get(config, :proxy_url, nil) - - case proxy do + case Pleroma.Config.get([:http, :proxy_url]) do nil -> options - _ -> options ++ [proxy: proxy] + proxy -> options ++ [proxy: proxy] end end diff --git a/lib/pleroma/plugs/federating_plug.ex b/lib/pleroma/plugs/federating_plug.ex index effc154bf..4dc4e9279 100644 --- a/lib/pleroma/plugs/federating_plug.ex +++ b/lib/pleroma/plugs/federating_plug.ex @@ -10,7 +10,7 @@ defmodule Pleroma.Web.FederatingPlug do end def call(conn, _opts) do - if Keyword.get(Application.get_env(:pleroma, :instance), :federating) do + if Pleroma.Config.get([:instance, :federating]) do conn else conn diff --git a/lib/pleroma/reverse_proxy.ex b/lib/pleroma/reverse_proxy.ex index 6e5feb4c3..983e156f5 100644 --- a/lib/pleroma/reverse_proxy.ex +++ b/lib/pleroma/reverse_proxy.ex @@ -61,7 +61,7 @@ defmodule Pleroma.ReverseProxy do * `http`: options for [hackney](https://github.com/benoitc/hackney). """ - @hackney Application.get_env(:pleroma, :hackney, :hackney) + @hackney Pleroma.Config.get(:hackney, :hackney) @default_hackney_options [] diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 653dec95f..5c91dc7d4 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -366,9 +366,7 @@ defmodule Pleroma.User do end def follow(%User{} = follower, %User{info: info} = followed) do - user_config = Application.get_env(:pleroma, :user) - deny_follow_blocked = Keyword.get(user_config, :deny_follow_blocked) - + deny_follow_blocked = Pleroma.Config.get([:user, :deny_follow_blocked]) ap_followers = followed.follower_address cond do diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index aa0229db7..bdd7e78d2 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -399,16 +399,12 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do end def block(blocker, blocked, activity_id \\ nil, local \\ true) do - ap_config = Application.get_env(:pleroma, :activitypub) - unfollow_blocked = Keyword.get(ap_config, :unfollow_blocked) - outgoing_blocks = Keyword.get(ap_config, :outgoing_blocks) + outgoing_blocks = Pleroma.Config.get([:activitypub, :outgoing_blocks]) + unfollow_blocked = Pleroma.Config.get([:activitypub, :unfollow_blocked]) - with true <- unfollow_blocked do + if unfollow_blocked do follow_activity = fetch_latest_follow(blocker, blocked) - - if follow_activity do - unfollow(blocker, blocked, nil, local) - end + if follow_activity, do: unfollow(blocker, blocked, nil, local) end with true <- outgoing_blocks, diff --git a/lib/pleroma/web/activity_pub/activity_pub_controller.ex b/lib/pleroma/web/activity_pub/activity_pub_controller.ex index ad2ca1e54..0182bda46 100644 --- a/lib/pleroma/web/activity_pub/activity_pub_controller.ex +++ b/lib/pleroma/web/activity_pub/activity_pub_controller.ex @@ -27,7 +27,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do plug(:relay_active? when action in [:relay]) def relay_active?(conn, _) do - if Keyword.get(Application.get_env(:pleroma, :instance), :allow_relay) do + if Pleroma.Config.get([:instance, :allow_relay]) do conn else conn diff --git a/lib/pleroma/web/activity_pub/mrf.ex b/lib/pleroma/web/activity_pub/mrf.ex index 1aaa20050..3bf7955f3 100644 --- a/lib/pleroma/web/activity_pub/mrf.ex +++ b/lib/pleroma/web/activity_pub/mrf.ex @@ -17,9 +17,7 @@ defmodule Pleroma.Web.ActivityPub.MRF do end def get_policies do - Application.get_env(:pleroma, :instance, []) - |> Keyword.get(:rewrite_policy, []) - |> get_policies() + Pleroma.Config.get([:instance, :rewrite_policy], []) |> get_policies() end defp get_policies(policy) when is_atom(policy), do: [policy] diff --git a/lib/pleroma/web/endpoint.ex b/lib/pleroma/web/endpoint.ex index 8cd7a2270..bd76e4295 100644 --- a/lib/pleroma/web/endpoint.ex +++ b/lib/pleroma/web/endpoint.ex @@ -66,7 +66,7 @@ defmodule Pleroma.Web.Endpoint do parsers: [:urlencoded, :multipart, :json], pass: ["*/*"], json_decoder: Jason, - length: Application.get_env(:pleroma, :instance) |> Keyword.get(:upload_limit), + length: Pleroma.Config.get([:instance, :upload_limit]), body_reader: {Pleroma.Web.Plugs.DigestPlug, :read_body, []} ) diff --git a/lib/pleroma/web/media_proxy/media_proxy.ex b/lib/pleroma/web/media_proxy/media_proxy.ex index 5762e767b..cee6d8481 100644 --- a/lib/pleroma/web/media_proxy/media_proxy.ex +++ b/lib/pleroma/web/media_proxy/media_proxy.ex @@ -12,25 +12,27 @@ defmodule Pleroma.Web.MediaProxy do def url("/" <> _ = url), do: url def url(url) do - config = Application.get_env(:pleroma, :media_proxy, []) - domain = URI.parse(url).host - - cond do - !Keyword.get(config, :enabled, false) or String.starts_with?(url, Pleroma.Web.base_url()) -> - url - - Enum.any?(Pleroma.Config.get([:media_proxy, :whitelist]), fn pattern -> - String.equivalent?(domain, pattern) - end) -> - url - - true -> - encode_url(url) + if !enabled?() or local?(url) or whitelisted?(url) do + url + else + encode_url(url) end end + defp enabled?, do: Pleroma.Config.get([:media_proxy, :enabled], false) + + defp local?(url), do: String.starts_with?(url, Pleroma.Web.base_url()) + + defp whitelisted?(url) do + %{host: domain} = URI.parse(url) + + Enum.any?(Pleroma.Config.get([:media_proxy, :whitelist]), fn pattern -> + String.equivalent?(domain, pattern) + end) + end + def encode_url(url) do - secret = Application.get_env(:pleroma, Pleroma.Web.Endpoint)[:secret_key_base] + secret = Pleroma.Config.get([Pleroma.Web.Endpoint, :secret_key_base]) # Must preserve `%2F` for compatibility with S3 # https://git.pleroma.social/pleroma/pleroma/issues/580 @@ -52,7 +54,7 @@ defmodule Pleroma.Web.MediaProxy do end def decode_url(sig, url) do - secret = Application.get_env(:pleroma, Pleroma.Web.Endpoint)[:secret_key_base] + secret = Pleroma.Config.get([Pleroma.Web.Endpoint, :secret_key_base]) sig = Base.url_decode64!(sig, @base64_opts) local_sig = :crypto.hmac(:sha, secret, url) diff --git a/lib/pleroma/web/nodeinfo/nodeinfo_controller.ex b/lib/pleroma/web/nodeinfo/nodeinfo_controller.ex index 45f90c579..59f3d4e11 100644 --- a/lib/pleroma/web/nodeinfo/nodeinfo_controller.ex +++ b/lib/pleroma/web/nodeinfo/nodeinfo_controller.ex @@ -32,20 +32,15 @@ defmodule Pleroma.Web.Nodeinfo.NodeinfoController do # returns a nodeinfo 2.0 map, since 2.1 just adds a repository field # under software. def raw_nodeinfo do - instance = Application.get_env(:pleroma, :instance) - media_proxy = Application.get_env(:pleroma, :media_proxy) - suggestions = Application.get_env(:pleroma, :suggestions) - chat = Application.get_env(:pleroma, :chat) - gopher = Application.get_env(:pleroma, :gopher) stats = Stats.get_stats() mrf_simple = - Application.get_env(:pleroma, :mrf_simple) + Config.get(:mrf_simple) |> Enum.into(%{}) # This horror is needed to convert regex sigils to strings mrf_keyword = - Application.get_env(:pleroma, :mrf_keyword, []) + Config.get(:mrf_keyword, []) |> Enum.map(fn {key, value} -> {key, Enum.map(value, fn @@ -74,14 +69,7 @@ defmodule Pleroma.Web.Nodeinfo.NodeinfoController do MRF.get_policies() |> Enum.map(fn policy -> to_string(policy) |> String.split(".") |> List.last() end) - quarantined = Keyword.get(instance, :quarantined_instances) - - quarantined = - if is_list(quarantined) do - quarantined - else - [] - end + quarantined = Config.get([:instance, :quarantined_instances], []) staff_accounts = User.all_superusers() @@ -92,7 +80,7 @@ defmodule Pleroma.Web.Nodeinfo.NodeinfoController do |> Enum.into(%{}, fn {k, v} -> {k, length(v)} end) federation_response = - if Keyword.get(instance, :mrf_transparency) do + if Config.get([:instance, :mrf_transparency]) do %{ mrf_policies: mrf_policies, mrf_simple: mrf_simple, @@ -109,22 +97,22 @@ defmodule Pleroma.Web.Nodeinfo.NodeinfoController do "pleroma_api", "mastodon_api", "mastodon_api_streaming", - if Keyword.get(media_proxy, :enabled) do + if Config.get([:media_proxy, :enabled]) do "media_proxy" end, - if Keyword.get(gopher, :enabled) do + if Config.get([:gopher, :enabled]) do "gopher" end, - if Keyword.get(chat, :enabled) do + if Config.get([:chat, :enabled]) do "chat" end, - if Keyword.get(suggestions, :enabled) do + if Config.get([:suggestions, :enabled]) do "suggestions" end, - if Keyword.get(instance, :allow_relay) do + if Config.get([:instance, :allow_relay]) do "relay" end, - if Keyword.get(instance, :safe_dm_mentions) do + if Config.get([:instance, :safe_dm_mentions]) do "safe_dm_mentions" end ] @@ -141,7 +129,7 @@ defmodule Pleroma.Web.Nodeinfo.NodeinfoController do inbound: [], outbound: [] }, - openRegistrations: Keyword.get(instance, :registrations_open), + openRegistrations: Config.get([:instance, :registrations_open]), usage: %{ users: %{ total: stats.user_count || 0 @@ -149,29 +137,29 @@ defmodule Pleroma.Web.Nodeinfo.NodeinfoController do localPosts: stats.status_count || 0 }, metadata: %{ - nodeName: Keyword.get(instance, :name), - nodeDescription: Keyword.get(instance, :description), - private: !Keyword.get(instance, :public, true), + nodeName: Config.get([:instance, :name]), + nodeDescription: Config.get([:instance, :description]), + private: !Config.get([:instance, :public], true), suggestions: %{ - enabled: Keyword.get(suggestions, :enabled, false), - thirdPartyEngine: Keyword.get(suggestions, :third_party_engine, ""), - timeout: Keyword.get(suggestions, :timeout, 5000), - limit: Keyword.get(suggestions, :limit, 23), - web: Keyword.get(suggestions, :web, "") + enabled: Config.get([:suggestions, :enabled], false), + thirdPartyEngine: Config.get([:suggestions, :third_party_engine], ""), + timeout: Config.get([:suggestions, :timeout], 5000), + limit: Config.get([:suggestions, :limit], 23), + web: Config.get([:suggestions, :web], "") }, staffAccounts: staff_accounts, federation: federation_response, - postFormats: Keyword.get(instance, :allowed_post_formats), + postFormats: Config.get([:instance, :allowed_post_formats]), uploadLimits: %{ - general: Keyword.get(instance, :upload_limit), - avatar: Keyword.get(instance, :avatar_upload_limit), - banner: Keyword.get(instance, :banner_upload_limit), - background: Keyword.get(instance, :background_upload_limit) + general: Config.get([:instance, :upload_limit]), + avatar: Config.get([:instance, :avatar_upload_limit]), + banner: Config.get([:instance, :banner_upload_limit]), + background: Config.get([:instance, :background_upload_limit]) }, - accountActivationRequired: Keyword.get(instance, :account_activation_required, false), - invitesEnabled: Keyword.get(instance, :invites_enabled, false), + accountActivationRequired: Config.get([:instance, :account_activation_required], false), + invitesEnabled: Config.get([:instance, :invites_enabled], false), features: features, - restrictedNicknames: Pleroma.Config.get([Pleroma.User, :restricted_nicknames]) + restrictedNicknames: Config.get([Pleroma.User, :restricted_nicknames]) } } end diff --git a/lib/pleroma/web/templates/layout/app.html.eex b/lib/pleroma/web/templates/layout/app.html.eex index 3389c91cc..85ec4d76c 100644 --- a/lib/pleroma/web/templates/layout/app.html.eex +++ b/lib/pleroma/web/templates/layout/app.html.eex @@ -4,7 +4,7 @@ - <%= Application.get_env(:pleroma, :instance)[:name] %> + <%= Pleroma.Config.get([:instance, :name]) %>