hj/pleroma
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Merge branch 'develop' of git.pleroma.social:pleroma/pleroma into update-validator

Browse source
This commit is contained in:
lain 2020-06-23 10:17:12 +02:00
commit 2c603f2009
71 changed files with 1965 additions and 370 deletions
Download patch file Download diff file Expand all files Collapse all files

3
lib/pleroma/application.ex
View file

@ -148,7 +148,8 @@ defmodule Pleroma.Application do
build_cachex("idempotency", expiration: idempotency_expiration(), limit: 2500),
build_cachex("web_resp", limit: 2500),
build_cachex("emoji_packs", expiration: emoji_packs_expiration(), limit: 10),
build_cachex("failed_proxy_url", limit: 2500)
build_cachex("failed_proxy_url", limit: 2500),
build_cachex("banned_urls", default_ttl: :timer.hours(24 * 30), limit: 5_000)
]
end

54
lib/pleroma/emoji/pack.ex
View file

@ -1,6 +1,7 @@
defmodule Pleroma.Emoji.Pack do
@derive {Jason.Encoder, only: [:files, :pack]}
@derive {Jason.Encoder, only: [:files, :pack, :files_count]}
defstruct files: %{},
files_count: 0,
pack_file: nil,
path: nil,
pack: %{},
@ -8,6 +9,7 @@ defmodule Pleroma.Emoji.Pack do
@type t() :: %__MODULE__{
files: %{String.t() => Path.t()},
files_count: non_neg_integer(),
pack_file: Path.t(),
path: Path.t(),
pack: map(),
@ -16,7 +18,7 @@ defmodule Pleroma.Emoji.Pack do
alias Pleroma.Emoji
@spec create(String.t()) :: :ok | {:error, File.posix()} | {:error, :empty_values}
@spec create(String.t()) :: {:ok, t()} | {:error, File.posix()} | {:error, :empty_values}
def create(name) do
with :ok <- validate_not_empty([name]),
dir <- Path.join(emoji_path(), name),
@ -26,10 +28,27 @@ defmodule Pleroma.Emoji.Pack do
end
end
@spec show(String.t()) :: {:ok, t()} | {:error, atom()}
def show(name) do
defp paginate(entities, 1, page_size), do: Enum.take(entities, page_size)
defp paginate(entities, page, page_size) do
entities
|> Enum.chunk_every(page_size)
|> Enum.at(page - 1)
end
@spec show(keyword()) :: {:ok, t()} | {:error, atom()}
def show(opts) do
name = opts[:name]
with :ok <- validate_not_empty([name]),
{:ok, pack} <- load_pack(name) do
shortcodes =
pack.files
|> Map.keys()
|> paginate(opts[:page], opts[:page_size])
pack = Map.put(pack, :files, Map.take(pack.files, shortcodes))
{:ok, validate_pack(pack)}
end
end
@ -120,10 +139,10 @@ defmodule Pleroma.Emoji.Pack do
end
end
@spec list_local() :: {:ok, map()}
def list_local do
@spec list_local(keyword()) :: {:ok, map(), non_neg_integer()}
def list_local(opts) do
with {:ok, results} <- list_packs_dir() do
packs =
all_packs =
results
|> Enum.map(fn name ->
case load_pack(name) do
@ -132,9 +151,13 @@ defmodule Pleroma.Emoji.Pack do
end
end)
|> Enum.reject(&is_nil/1)
packs =
all_packs
|> paginate(opts[:page], opts[:page_size])
|> Map.new(fn pack -> {pack.name, validate_pack(pack)} end)
{:ok, packs}
{:ok, packs, length(all_packs)}
end
end
@ -146,7 +169,7 @@ defmodule Pleroma.Emoji.Pack do
end
end
@spec download(String.t(), String.t(), String.t()) :: :ok | {:error, atom()}
@spec download(String.t(), String.t(), String.t()) :: {:ok, t()} | {:error, atom()}
def download(name, url, as) do
uri = url |> String.trim() |> URI.parse()
@ -197,7 +220,12 @@ defmodule Pleroma.Emoji.Pack do
|> Map.put(:path, Path.dirname(pack_file))
|> Map.put(:name, name)
{:ok, pack}
files_count =
pack.files
|> Map.keys()
|> length()
{:ok, Map.put(pack, :files_count, files_count)}
else
{:error, :not_found}
end
@ -296,7 +324,9 @@ defmodule Pleroma.Emoji.Pack do
# Otherwise, they'd have to download it from external-src
pack.pack["share-files"] &&
Enum.all?(pack.files, fn {_, file} ->
File.exists?(Path.join(pack.path, file))
pack.path
|> Path.join(file)
|> File.exists?()
end)
end
@ -440,7 +470,7 @@ defmodule Pleroma.Emoji.Pack do
# with the API so it should be sufficient
with {:create_dir, :ok} <- {:create_dir, File.mkdir_p(emoji_path)},
{:ls, {:ok, results}} <- {:ls, File.ls(emoji_path)} do
{:ok, results}
{:ok, Enum.sort(results)}
else
{:create_dir, {:error, e}} -> {:error, :create_dir, e}
{:ls, {:error, e}} -> {:error, :ls, e}

6
lib/pleroma/pagination.ex
View file

@ -64,6 +64,12 @@ defmodule Pleroma.Pagination do
@spec paginate(Ecto.Query.t(), map(), type(), atom() | nil) :: [Ecto.Schema.t()]
def paginate(query, options, method \\ :keyset, table_binding \\ nil)
def paginate(list, options, _method, _table_binding) when is_list(list) do
offset = options[:offset] || 0
limit = options[:limit] || 0
Enum.slice(list, offset, limit)
end
def paginate(query, options, :keyset, table_binding) do
query
|> restrict(:min_id, options, table_binding)

16
lib/pleroma/plugs/uploaded_media.ex
View file

@ -10,6 +10,8 @@ defmodule Pleroma.Plugs.UploadedMedia do
import Pleroma.Web.Gettext
require Logger
alias Pleroma.Web.MediaProxy
@behaviour Plug
# no slashes
@path "media"
@ -35,8 +37,7 @@ defmodule Pleroma.Plugs.UploadedMedia do
%{query_params: %{"name" => name}} = conn ->
name = String.replace(name, "\"", "\\\"")
conn
|> put_resp_header("content-disposition", "filename=\"#{name}\"")
put_resp_header(conn, "content-disposition", "filename=\"#{name}\"")
conn ->
conn
@ -47,7 +48,8 @@ defmodule Pleroma.Plugs.UploadedMedia do
with uploader <- Keyword.fetch!(config, :uploader),
proxy_remote = Keyword.get(config, :proxy_remote, false),
{:ok, get_method} <- uploader.get_file(file) do
{:ok, get_method} <- uploader.get_file(file),
false <- media_is_banned(conn, get_method) do
get_media(conn, get_method, proxy_remote, opts)
else
_ ->
@ -59,6 +61,14 @@ defmodule Pleroma.Plugs.UploadedMedia do
def call(conn, _opts), do: conn
defp media_is_banned(%{request_path: path} = _conn, {:static_dir, _}) do
MediaProxy.in_banned_urls(Pleroma.Web.base_url() <> path)
end
defp media_is_banned(_, {:url, url}), do: MediaProxy.in_banned_urls(url)
defp media_is_banned(_, _), do: false
defp get_media(conn, {:static_dir, directory}, _, opts) do
static_opts =
Map.get(opts, :static_plug_opts)

92
lib/pleroma/user.ex
View file

@ -263,37 +263,60 @@ defmodule Pleroma.User do
def account_status(%User{password_reset_pending: true}), do: :password_reset_pending
def account_status(%User{confirmation_pending: true}) do
case Config.get([:instance, :account_activation_required]) do
true -> :confirmation_pending
_ -> :active
if Config.get([:instance, :account_activation_required]) do
:confirmation_pending
else
:active
end
end
def account_status(%User{}), do: :active
@spec visible_for?(User.t(), User.t() | nil) :: boolean()
def visible_for?(user, for_user \\ nil)
@spec visible_for(User.t(), User.t() | nil) ::
:visible
| :invisible
| :restricted_unauthenticated
| :deactivated
| :confirmation_pending
def visible_for(user, for_user \\ nil)
def visible_for?(%User{invisible: true}, _), do: false
def visible_for(%User{invisible: true}, _), do: :invisible
def visible_for?(%User{id: user_id}, %User{id: user_id}), do: true
def visible_for(%User{id: user_id}, %User{id: user_id}), do: :visible
def visible_for?(%User{local: local} = user, nil) do
cfg_key =
if local,
do: :local,
else: :remote
if Config.get([:restrict_unauthenticated, :profiles, cfg_key]),
do: false,
else: account_status(user) == :active
def visible_for(%User{} = user, nil) do
if restrict_unauthenticated?(user) do
:restrict_unauthenticated
else
visible_account_status(user)
end
end
def visible_for?(%User{} = user, for_user) do
account_status(user) == :active || superuser?(for_user)
def visible_for(%User{} = user, for_user) do
if superuser?(for_user) do
:visible
else
visible_account_status(user)
end
end
def visible_for?(_, _), do: false
def visible_for(_, _), do: :invisible
defp restrict_unauthenticated?(%User{local: local}) do
config_key = if local, do: :local, else: :remote
Config.get([:restrict_unauthenticated, :profiles, config_key], false)
end
defp visible_account_status(user) do
status = account_status(user)
if status in [:active, :password_reset_pending] do
:visible
else
status
end
end
@spec superuser?(User.t()) :: boolean()
def superuser?(%User{local: true, is_admin: true}), do: true
@ -465,6 +488,7 @@ defmodule Pleroma.User do
|> validate_format(:nickname, local_nickname_regex())
|> validate_length(:bio, max: bio_limit)
|> validate_length(:name, min: 1, max: name_limit)
|> validate_inclusion(:actor_type, ["Person", "Service"])
|> put_fields()
|> put_emoji()
|> put_change_if_present(:bio, &{:ok, parse_bio(&1, struct)})
@ -758,7 +782,6 @@ defmodule Pleroma.User do
follower
|> update_following_count()
|> set_cache()
end
end
@ -787,7 +810,6 @@ defmodule Pleroma.User do
{:ok, follower} =
follower
|> update_following_count()
|> set_cache()
{:ok, follower, followed}
@ -1139,35 +1161,25 @@ defmodule Pleroma.User do
])
end
@spec update_follower_count(User.t()) :: {:ok, User.t()}
def update_follower_count(%User{} = user) do
if user.local or !Pleroma.Config.get([:instance, :external_user_synchronization]) do
follower_count_query =
User.Query.build(%{followers: user, deactivated: false})
|> select([u], %{count: count(u.id)})
follower_count = FollowingRelationship.follower_count(user)
User
|> where(id: ^user.id)
|> join(:inner, [u], s in subquery(follower_count_query))
|> update([u, s],
set: [follower_count: s.count]
)
|> select([u], u)
|> Repo.update_all([])
|> case do
{1, [user]} -> set_cache(user)
_ -> {:error, user}
end
user
|> follow_information_changeset(%{follower_count: follower_count})
|> update_and_set_cache
else
{:ok, maybe_fetch_follow_information(user)}
end
end
@spec update_following_count(User.t()) :: User.t()
@spec update_following_count(User.t()) :: {:ok, User.t()}
def update_following_count(%User{local: false} = user) do
if Pleroma.Config.get([:instance, :external_user_synchronization]) do
maybe_fetch_follow_information(user)
{:ok, maybe_fetch_follow_information(user)}
else
user
{:ok, user}
end
end
@ -1176,7 +1188,7 @@ defmodule Pleroma.User do
user
|> follow_information_changeset(%{following_count: following_count})
|> Repo.update!()
|> update_and_set_cache()
end
def set_unread_conversation_count(%User{local: true} = user) do

3
lib/pleroma/web/activity_pub/activity_pub.ex
View file

@ -812,7 +812,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
defp restrict_media(query, %{only_media: true}) do
from(
[_activity, object] in query,
[activity, object] in query,
where: fragment("(?)->>'type' = ?", activity.data, "Create"),
where: fragment("not (?)->'attachment' = (?)", object.data, ^[])
)
end

3
lib/pleroma/web/activity_pub/activity_pub_controller.ex
View file

@ -514,7 +514,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
{new_user, for_user}
end
# TODO: Add support for "object" field
@doc """
Endpoint based on <https://www.w3.org/wiki/SocialCG/ActivityPub/MediaUpload>
@ -525,6 +524,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
Response:
- HTTP Code: 201 Created
- HTTP Body: ActivityPub object to be inserted into another's `attachment` field
Note: Will not point to a URL with a `Location` header because no standalone Activity has been created.
"""
def upload_media(%{assigns: %{user: %User{} = user}} = conn, %{"file" => file} = data) do
with {:ok, object} <-

7
lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex
View file

@ -13,8 +13,10 @@ defmodule Pleroma.Web.ActivityPub.MRF.HellthreadPolicy do
defp delist_message(message, threshold) when threshold > 0 do
follower_collection = User.get_cached_by_ap_id(message["actor"]).follower_address
to = message["to"] || []
cc = message["cc"] || []
follower_collection? = Enum.member?(message["to"] ++ message["cc"], follower_collection)
follower_collection? = Enum.member?(to ++ cc, follower_collection)
message =
case get_recipient_count(message) do
@ -71,7 +73,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.HellthreadPolicy do
end
@impl true
def filter(%{"type" => "Create"} = message) do
def filter(%{"type" => "Create", "object" => %{"type" => object_type}} = message)
when object_type in ~w{Note Article} do
reject_threshold =
Pleroma.Config.get(
[:mrf_hellthread, :reject_threshold],

1
lib/pleroma/web/activity_pub/object_validators/note_validator.ex
View file

@ -41,7 +41,6 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.NoteValidator do
field(:announcements, {:array, :string}, default: [])
# see if needed
field(:conversation, :string)
field(:context_id, :string)
end

6
lib/pleroma/web/activity_pub/transmogrifier.ex
View file

@ -172,8 +172,8 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
object
|> Map.put("inReplyTo", replied_object.data["id"])
|> Map.put("inReplyToAtomUri", object["inReplyToAtomUri"] || in_reply_to_id)
|> Map.put("conversation", replied_object.data["context"] || object["conversation"])
|> Map.put("context", replied_object.data["context"] || object["conversation"])
|> Map.drop(["conversation"])
else
e ->
Logger.error("Couldn't fetch #{inspect(in_reply_to_id)}, error: #{inspect(e)}")
@ -207,7 +207,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
object
|> Map.put("context", context)
|> Map.put("conversation", context)
|> Map.drop(["conversation"])
end
def fix_attachments(%{"attachment" => attachment} = object) when is_list(attachment) do
@ -458,7 +458,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
to: data["to"],
object: object,
actor: user,
context: object["conversation"],
context: object["context"],
local: false,
published: data["published"],
additional:

29
lib/pleroma/web/admin_api/controllers/admin_api_controller.ex
View file

@ -111,8 +111,7 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
action: "delete"
})
conn
|> json(nicknames)
json(conn, nicknames)
end
def user_follow(%{assigns: %{user: admin}} = conn, %{
@ -131,8 +130,7 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
})
end
conn
|> json("ok")
json(conn, "ok")
end
def user_unfollow(%{assigns: %{user: admin}} = conn, %{
@ -151,8 +149,7 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
})
end
conn
|> json("ok")
json(conn, "ok")
end
def users_create(%{assigns: %{user: admin}} = conn, %{"users" => users}) do
@ -191,8 +188,7 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
action: "create"
})
conn
|> json(res)
json(conn, res)
{:error, id, changeset, _} ->
res =
@ -363,8 +359,8 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
filters
|> String.split(",")
|> Enum.filter(&Enum.member?(@filters, &1))
|> Enum.map(&String.to_atom(&1))
|> Enum.into(%{}, &{&1, true})
|> Enum.map(&String.to_atom/1)
|> Map.new(&{&1, true})
end
def right_add_multiple(%{assigns: %{user: admin}} = conn, %{
@ -568,10 +564,10 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
{:error, changeset} ->
errors = Map.new(changeset.errors, fn {key, {error, _}} -> {key, error} end)
json(conn, %{errors: errors})
{:errors, errors}
_ ->
json(conn, %{error: "Unable to update user."})
{:error, :not_found}
end
end
@ -616,7 +612,7 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
def reload_emoji(conn, _params) do
Pleroma.Emoji.reload()
conn |> json("ok")
json(conn, "ok")
end
def confirm_email(%{assigns: %{user: admin}} = conn, %{"nicknames" => nicknames}) do
@ -630,7 +626,7 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
action: "confirm_email"
})
conn |> json("")
json(conn, "")
end
def resend_confirmation_email(%{assigns: %{user: admin}} = conn, %{"nicknames" => nicknames}) do
@ -644,14 +640,13 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
action: "resend_confirmation_email"
})
conn |> json("")
json(conn, "")
end
def stats(conn, _) do
count = Stats.get_status_visibility_count()
conn
|> json(%{"status_visibility" => count})
json(conn, %{"status_visibility" => count})
end
defp page_params(params) do

6
lib/pleroma/web/admin_api/controllers/fallback_controller.ex
View file

@ -17,6 +17,12 @@ defmodule Pleroma.Web.AdminAPI.FallbackController do
|> json(%{error: reason})
end
def call(conn, {:errors, errors}) do
conn
|> put_status(:bad_request)
|> json(%{errors: errors})
end
def call(conn, {:param_cast, _}) do
conn
|> put_status(:bad_request)

63
lib/pleroma/web/admin_api/controllers/media_proxy_cache_controller.ex Normal file
View file

@ -0,0 +1,63 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.AdminAPI.MediaProxyCacheController do
use Pleroma.Web, :controller
alias Pleroma.Plugs.OAuthScopesPlug
alias Pleroma.Web.ApiSpec.Admin, as: Spec
alias Pleroma.Web.MediaProxy
plug(Pleroma.Web.ApiSpec.CastAndValidate)
plug(
OAuthScopesPlug,
%{scopes: ["read:media_proxy_caches"], admin: true} when action in [:index]
)
plug(
OAuthScopesPlug,
%{scopes: ["write:media_proxy_caches"], admin: true} when action in [:purge, :delete]
)
action_fallback(Pleroma.Web.AdminAPI.FallbackController)
defdelegate open_api_operation(action), to: Spec.MediaProxyCacheOperation
def index(%{assigns: %{user: _}} = conn, params) do
cursor =
:banned_urls_cache
|> :ets.table([{:traverse, {:select, Cachex.Query.create(true, :key)}}])
|> :qlc.cursor()
urls =
case params.page do
1 ->
:qlc.next_answers(cursor, params.page_size)
_ ->
:qlc.next_answers(cursor, (params.page - 1) * params.page_size)
:qlc.next_answers(cursor, params.page_size)
end
:qlc.delete_cursor(cursor)
render(conn, "index.json", urls: urls)
end
def delete(%{assigns: %{user: _}, body_params: %{urls: urls}} = conn, _) do
MediaProxy.remove_from_banned_urls(urls)
render(conn, "index.json", urls: urls)
end
def purge(%{assigns: %{user: _}, body_params: %{urls: urls, ban: ban}} = conn, _) do
MediaProxy.Invalidation.purge(urls)
if ban do
MediaProxy.put_in_banned_urls(urls)
end
render(conn, "index.json", urls: urls)
end
end

11
lib/pleroma/web/admin_api/views/media_proxy_cache_view.ex Normal file
View file

@ -0,0 +1,11 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.AdminAPI.MediaProxyCacheView do
use Pleroma.Web, :view
def render("index.json", %{urls: urls}) do
%{urls: urls}
end
end

6
lib/pleroma/web/api_spec/helpers.ex
View file

@ -39,6 +39,12 @@ defmodule Pleroma.Web.ApiSpec.Helpers do
:string,
"Return the newest items newer than this ID"
),
Operation.parameter(
:offset,
:query,
%Schema{type: :integer, default: 0},
"Return items past this number of items"
),
Operation.parameter(
:limit,
:query,

2
lib/pleroma/web/api_spec/operations/account_operation.ex
View file

@ -102,6 +102,7 @@ defmodule Pleroma.Web.ApiSpec.AccountOperation do
parameters: [%Reference{"$ref": "#/components/parameters/accountIdOrNickname"}],
responses: %{
200 => Operation.response("Account", "application/json", Account),
401 => Operation.response("Error", "application/json", ApiError),
404 => Operation.response("Error", "application/json", ApiError)
}
}
@ -142,6 +143,7 @@ defmodule Pleroma.Web.ApiSpec.AccountOperation do
] ++ pagination_params(),
responses: %{
200 => Operation.response("Statuses", "application/json", array_of_statuses()),
401 => Operation.response("Error", "application/json", ApiError),
404 => Operation.response("Error", "application/json", ApiError)
}
}

109
lib/pleroma/web/api_spec/operations/admin/media_proxy_cache_operation.ex Normal file
View file

@ -0,0 +1,109 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.ApiSpec.Admin.MediaProxyCacheOperation do
alias OpenApiSpex.Operation
alias OpenApiSpex.Schema
alias Pleroma.Web.ApiSpec.Schemas.ApiError
import Pleroma.Web.ApiSpec.Helpers
def open_api_operation(action) do
operation = String.to_existing_atom("#{action}_operation")
apply(__MODULE__, operation, [])
end
def index_operation do
%Operation{
tags: ["Admin", "MediaProxyCache"],
summary: "Fetch a paginated list of all banned MediaProxy URLs in Cachex",
operationId: "AdminAPI.MediaProxyCacheController.index",
security: [%{"oAuth" => ["read:media_proxy_caches"]}],
parameters: [
Operation.parameter(
:page,
:query,
%Schema{type: :integer, default: 1},
"Page"
),
Operation.parameter(
:page_size,
:query,
%Schema{type: :integer, default: 50},
"Number of statuses to return"
)
],
responses: %{
200 => success_response()
}
}
end
def delete_operation do
%Operation{
tags: ["Admin", "MediaProxyCache"],
summary: "Remove a banned MediaProxy URL from Cachex",
operationId: "AdminAPI.MediaProxyCacheController.delete",
security: [%{"oAuth" => ["write:media_proxy_caches"]}],
requestBody:
request_body(
"Parameters",
%Schema{
type: :object,
required: [:urls],
properties: %{
urls: %Schema{type: :array, items: %Schema{type: :string, format: :uri}}
}
},
required: true
),
responses: %{
200 => success_response(),
400 => Operation.response("Error", "application/json", ApiError)
}
}
end
def purge_operation do
%Operation{
tags: ["Admin", "MediaProxyCache"],
summary: "Purge and optionally ban a MediaProxy URL",
operationId: "AdminAPI.MediaProxyCacheController.purge",
security: [%{"oAuth" => ["write:media_proxy_caches"]}],
requestBody:
request_body(
"Parameters",
%Schema{
type: :object,
required: [:urls],
properties: %{
urls: %Schema{type: :array, items: %Schema{type: :string, format: :uri}},
ban: %Schema{type: :boolean, default: true}
}
},
required: true
),
responses: %{
200 => success_response(),
400 => Operation.response("Error", "application/json", ApiError)
}
}
end
defp success_response do
Operation.response("Array of banned MediaProxy URLs in Cachex", "application/json", %Schema{
type: :object,
properties: %{
urls: %Schema{
type: :array,
items: %Schema{
type: :string,
format: :uri,
description: "MediaProxy URLs"
}
}
}
})
end
end

10
lib/pleroma/web/api_spec/operations/notification_operation.ex
View file

@ -163,6 +163,13 @@ defmodule Pleroma.Web.ApiSpec.NotificationOperation do
description:
"Status that was the object of the notification, e.g. in mentions, reblogs, favourites, or polls.",
nullable: true
},
pleroma: %Schema{
type: :object,
properties: %{
is_seen: %Schema{type: :boolean},
is_muted: %Schema{type: :boolean}
}
}
},
example: %{
@ -170,7 +177,8 @@ defmodule Pleroma.Web.ApiSpec.NotificationOperation do
"type" => "mention",
"created_at" => "2019-11-23T07:49:02.064Z",
"account" => Account.schema().example,
"status" => Status.schema().example
"status" => Status.schema().example,
"pleroma" => %{"is_seen" => false, "is_muted" => false}
}
}
end

30
lib/pleroma/web/api_spec/operations/pleroma_emoji_pack_operation.ex
View file

@ -33,6 +33,20 @@ defmodule Pleroma.Web.ApiSpec.PleromaEmojiPackOperation do
tags: ["Emoji Packs"],
summary: "Lists local custom emoji packs",
operationId: "PleromaAPI.EmojiPackController.index",
parameters: [
Operation.parameter(
:page,
:query,
%Schema{type: :integer, default: 1},
"Page"
),
Operation.parameter(
:page_size,
:query,
%Schema{type: :integer, default: 50},
"Number of emoji packs to return"
)
],
responses: %{
200 => emoji_packs_response()
}
@ -44,7 +58,21 @@ defmodule Pleroma.Web.ApiSpec.PleromaEmojiPackOperation do
tags: ["Emoji Packs"],
summary: "Show emoji pack",
operationId: "PleromaAPI.EmojiPackController.show",
parameters: [name_param()],
parameters: [
name_param(),
Operation.parameter(
:page,
:query,
%Schema{type: :integer, default: 1},
"Page"
),
Operation.parameter(
:page_size,
:query,
%Schema{type: :integer, default: 30},
"Number of emoji to return"
)
],
responses: %{
200 => Operation.response("Emoji Pack", "application/json", emoji_pack()),
400 => Operation.response("Bad Request", "application/json", ApiError),

2
lib/pleroma/web/masto_fe_controller.ex
View file

@ -49,7 +49,7 @@ defmodule Pleroma.Web.MastoFEController do
|> render("manifest.json")
end
@doc "PUT /api/web/settings"
@doc "PUT /api/web/settings: Backend-obscure settings blob for MastoFE, don't parse/reuse elsewhere"
def put_settings(%{assigns: %{user: user}} = conn, %{"data" => settings} = _params) do
with {:ok, _} <- User.mastodon_settings_update(user, settings) do
json(conn, %{})

21
lib/pleroma/web/mastodon_api/controllers/account_controller.ex
View file

@ -179,6 +179,9 @@ defmodule Pleroma.Web.MastodonAPI.AccountController do
|> Maps.put_if_present(:pleroma_settings_store, params[:pleroma_settings_store])
|> Maps.put_if_present(:default_scope, params[:default_scope])
|> Maps.put_if_present(:default_scope, params["source"]["privacy"])
|> Maps.put_if_present(:actor_type, params[:bot], fn bot ->
if bot, do: {:ok, "Service"}, else: {:ok, "Person"}
end)
|> Maps.put_if_present(:actor_type, params[:actor_type])
# What happens here:
@ -238,17 +241,17 @@ defmodule Pleroma.Web.MastodonAPI.AccountController do
@doc "GET /api/v1/accounts/:id"
def show(%{assigns: %{user: for_user}} = conn, %{id: nickname_or_id}) do
with %User{} = user <- User.get_cached_by_nickname_or_id(nickname_or_id, for: for_user),
true <- User.visible_for?(user, for_user) do
:visible <- User.visible_for(user, for_user) do
render(conn, "show.json", user: user, for: for_user)
else
_e -> render_error(conn, :not_found, "Can't find user")
error -> user_visibility_error(conn, error)
end
end
@doc "GET /api/v1/accounts/:id/statuses"
def statuses(%{assigns: %{user: reading_user}} = conn, params) do
with %User{} = user <- User.get_cached_by_nickname_or_id(params.id, for: reading_user),
true <- User.visible_for?(user, reading_user) do
:visible <- User.visible_for(user, reading_user) do
params =
params
|> Map.delete(:tagged)
@ -265,7 +268,17 @@ defmodule Pleroma.Web.MastodonAPI.AccountController do
as: :activity
)
else
_e -> render_error(conn, :not_found, "Can't find user")
error -> user_visibility_error(conn, error)
end
end
defp user_visibility_error(conn, error) do
case error do
:restrict_unauthenticated ->
render_error(conn, :unauthorized, "This API requires an authenticated user")
_ ->
render_error(conn, :not_found, "Can't find user")
end
end

31
lib/pleroma/web/mastodon_api/controllers/search_controller.ex
View file

@ -107,21 +107,21 @@ defmodule Pleroma.Web.MastodonAPI.SearchController do
)
end
defp resource_search(:v2, "hashtags", query, _options) do
defp resource_search(:v2, "hashtags", query, options) do
tags_path = Web.base_url() <> "/tag/"
query
|> prepare_tags()
|> prepare_tags(options)
|> Enum.map(fn tag ->
%{name: tag, url: tags_path <> tag}
end)
end
defp resource_search(:v1, "hashtags", query, _options) do
prepare_tags(query)
defp resource_search(:v1, "hashtags", query, options) do
prepare_tags(query, options)
end
defp prepare_tags(query, add_joined_tag \\ true) do
defp prepare_tags(query, options) do
tags =
query
|> preprocess_uri_query()
@ -139,13 +139,20 @@ defmodule Pleroma.Web.MastodonAPI.SearchController do
tags = Enum.map(tags, fn tag -> String.trim_leading(tag, "#") end)
if Enum.empty?(explicit_tags) && add_joined_tag do
tags
|> Kernel.++([joined_tag(tags)])
|> Enum.uniq_by(&String.downcase/1)
else
tags
end
tags =
if Enum.empty?(explicit_tags) && !options[:skip_joined_tag] do
add_joined_tag(tags)
else
tags
end
Pleroma.Pagination.paginate(tags, options)
end
defp add_joined_tag(tags) do
tags
|> Kernel.++([joined_tag(tags)])
|> Enum.uniq_by(&String.downcase/1)
end
# If `query` is a URI, returns last component of its path, otherwise returns `query`

4
lib/pleroma/web/mastodon_api/views/account_view.ex
View file

@ -35,7 +35,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do
end
def render("show.json", %{user: user} = opts) do
if User.visible_for?(user, opts[:for]) do
if User.visible_for(user, opts[:for]) == :visible do
do_render("show.json", opts)
else
%{}
@ -179,7 +179,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do
0
end
bot = user.actor_type in ["Application", "Service"]
bot = user.actor_type == "Service"
emojis =
Enum.map(user.emoji, fn {shortcode, raw_url} ->

7
lib/pleroma/web/mastodon_api/views/instance_view.ex
View file

@ -23,7 +23,7 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do
streaming_api: Pleroma.Web.Endpoint.websocket_url()
},
stats: Pleroma.Stats.get_stats(),
thumbnail: instance_thumbnail(),
thumbnail: Keyword.get(instance, :instance_thumbnail),
languages: ["en"],
registrations: Keyword.get(instance, :registrations_open),
# Extra (not present in Mastodon):
@ -88,9 +88,4 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do
end
|> Map.put(:enabled, Config.get([:instance, :federating]))
end
defp instance_thumbnail do
Pleroma.Config.get([:instance, :instance_thumbnail]) ||
"#{Pleroma.Web.base_url()}/instance/thumbnail.jpeg"
end
end

8
lib/pleroma/web/mastodon_api/views/notification_view.ex
View file

@ -84,12 +84,7 @@ defmodule Pleroma.Web.MastodonAPI.NotificationView do
# Note: :relationships contain user mutes (needed for :muted flag in :status)
status_render_opts = %{relationships: opts[:relationships]}
account =
AccountView.render(
"show.json",
%{user: actor, for: reading_user}
)
account = AccountView.render("show.json", %{user: actor, for: reading_user})
response = %{
id: to_string(notification.id),
@ -97,6 +92,7 @@ defmodule Pleroma.Web.MastodonAPI.NotificationView do
created_at: CommonAPI.Utils.to_masto_date(notification.inserted_at),
account: account,
pleroma: %{
is_muted: User.mutes?(reading_user, actor),
is_seen: notification.seen
}
}

26
lib/pleroma/web/media_proxy/invalidation.ex
View file

@ -5,22 +5,34 @@
defmodule Pleroma.Web.MediaProxy.Invalidation do
@moduledoc false
@callback purge(list(String.t()), map()) :: {:ok, String.t()} | {:error, String.t()}
@callback purge(list(String.t()), Keyword.t()) :: {:ok, list(String.t())} | {:error, String.t()}
alias Pleroma.Config
alias Pleroma.Web.MediaProxy
@spec purge(list(String.t())) :: {:ok, String.t()} | {:error, String.t()}
@spec enabled?() :: boolean()
def enabled?, do: Config.get([:media_proxy, :invalidation, :enabled])
@spec purge(list(String.t()) | String.t()) :: {:ok, list(String.t())} | {:error, String.t()}
def purge(urls) do
[:media_proxy, :invalidation, :enabled]
|> Config.get()
|> do_purge(urls)
prepared_urls = prepare_urls(urls)
if enabled?() do
do_purge(prepared_urls)
else
{:ok, prepared_urls}
end
end
defp do_purge(true, urls) do
defp do_purge(urls) do
provider = Config.get([:media_proxy, :invalidation, :provider])
options = Config.get(provider)
provider.purge(urls, options)
end
defp do_purge(_, _), do: :ok
def prepare_urls(urls) do
urls
|> List.wrap()
|> Enum.map(&MediaProxy.url/1)
end
end

10
lib/pleroma/web/media_proxy/invalidations/http.ex
View file

@ -9,10 +9,10 @@ defmodule Pleroma.Web.MediaProxy.Invalidation.Http do
require Logger
@impl Pleroma.Web.MediaProxy.Invalidation
def purge(urls, opts) do
method = Map.get(opts, :method, :purge)
headers = Map.get(opts, :headers, [])
options = Map.get(opts, :options, [])
def purge(urls, opts \\ []) do
method = Keyword.get(opts, :method, :purge)
headers = Keyword.get(opts, :headers, [])
options = Keyword.get(opts, :options, [])
Logger.debug("Running cache purge: #{inspect(urls)}")
@ -22,7 +22,7 @@ defmodule Pleroma.Web.MediaProxy.Invalidation.Http do
end
end)
{:ok, "success"}
{:ok, urls}
end
defp do_purge(method, url, headers, options) do

36
lib/pleroma/web/media_proxy/invalidations/script.ex
View file

@ -10,32 +10,34 @@ defmodule Pleroma.Web.MediaProxy.Invalidation.Script do
require Logger
@impl Pleroma.Web.MediaProxy.Invalidation
def purge(urls, %{script_path: script_path} = _options) do
def purge(urls, opts \\ []) do
args =
urls
|> List.wrap()
|> Enum.uniq()
|> Enum.join(" ")
path = Path.expand(script_path)
Logger.debug("Running cache purge: #{inspect(urls)}, #{path}")
case do_purge(path, [args]) do
{result, exit_status} when exit_status > 0 ->
Logger.error("Error while cache purge: #{inspect(result)}")
{:error, inspect(result)}
_ ->
{:ok, "success"}
end
opts
|> Keyword.get(:script_path)
|> do_purge([args])
|> handle_result(urls)
end
def purge(_, _), do: {:error, "not found script path"}
defp do_purge(path, args) do
defp do_purge(script_path, args) when is_binary(script_path) do
path = Path.expand(script_path)
Logger.debug("Running cache purge: #{inspect(args)}, #{inspect(path)}")
System.cmd(path, args)
rescue
error -> {inspect(error), 1}
error -> error
end
defp do_purge(_, _), do: {:error, "not found script path"}
defp handle_result({_result, 0}, urls), do: {:ok, urls}
defp handle_result({:error, error}, urls), do: handle_result(error, urls)
defp handle_result(error, _) do
Logger.error("Error while cache purge: #{inspect(error)}")
{:error, inspect(error)}
end
end

35
lib/pleroma/web/media_proxy/media_proxy.ex
View file

@ -6,20 +6,53 @@ defmodule Pleroma.Web.MediaProxy do
alias Pleroma.Config
alias Pleroma.Upload
alias Pleroma.Web
alias Pleroma.Web.MediaProxy.Invalidation
@base64_opts [padding: false]
@spec in_banned_urls(String.t()) :: boolean()
def in_banned_urls(url), do: elem(Cachex.exists?(:banned_urls_cache, url(url)), 1)
def remove_from_banned_urls(urls) when is_list(urls) do
Cachex.execute!(:banned_urls_cache, fn cache ->
Enum.each(Invalidation.prepare_urls(urls), &Cachex.del(cache, &1))
end)
end
def remove_from_banned_urls(url) when is_binary(url) do
Cachex.del(:banned_urls_cache, url(url))
end
def put_in_banned_urls(urls) when is_list(urls) do
Cachex.execute!(:banned_urls_cache, fn cache ->
Enum.each(Invalidation.prepare_urls(urls), &Cachex.put(cache, &1, true))
end)
end
def put_in_banned_urls(url) when is_binary(url) do
Cachex.put(:banned_urls_cache, url(url), true)
end
def url(url) when is_nil(url) or url == "", do: nil
def url("/" <> _ = url), do: url
def url(url) do
if disabled?() or local?(url) or whitelisted?(url) do
if disabled?() or not url_proxiable?(url) do
url
else
encode_url(url)
end
end
@spec url_proxiable?(String.t()) :: boolean()
def url_proxiable?(url) do
if local?(url) or whitelisted?(url) do
false
else
true
end
end
defp disabled?, do: !Config.get([:media_proxy, :enabled], false)
defp local?(url), do: String.starts_with?(url, Pleroma.Web.base_url())

3
lib/pleroma/web/media_proxy/media_proxy_controller.ex
View file

@ -14,10 +14,11 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do
with config <- Pleroma.Config.get([:media_proxy], []),
true <- Keyword.get(config, :enabled, false),
{:ok, url} <- MediaProxy.decode_url(sig64, url64),
{_, false} <- {:in_banned_urls, MediaProxy.in_banned_urls(url)},
:ok <- filename_matches(params, conn.request_path, url) do
ReverseProxy.call(conn, url, Keyword.get(config, :proxy_opts, @default_proxy_opts))
else
false ->
error when error in [false, {:in_banned_urls, true}] ->
send_resp(conn, 404, Plug.Conn.Status.reason_phrase(404))
{:error, :invalid_signature} ->

10
lib/pleroma/web/pleroma_api/controllers/emoji_pack_controller.ex
View file

@ -37,14 +37,14 @@ defmodule Pleroma.Web.PleromaAPI.EmojiPackController do
end
end
def index(conn, _params) do
def index(conn, params) do
emoji_path =
[:instance, :static_dir]
|> Pleroma.Config.get!()
|> Path.join("emoji")
with {:ok, packs} <- Pack.list_local() do
json(conn, packs)
with {:ok, packs, count} <- Pack.list_local(page: params.page, page_size: params.page_size) do
json(conn, %{packs: packs, count: count})
else
{:error, :create_dir, e} ->
conn
@ -60,10 +60,10 @@ defmodule Pleroma.Web.PleromaAPI.EmojiPackController do
end
end
def show(conn, %{name: name}) do
def show(conn, %{name: name, page: page, page_size: page_size}) do
name = String.trim(name)
with {:ok, pack} <- Pack.show(name) do
with {:ok, pack} <- Pack.show(name: name, page: page, page_size: page_size) do
json(conn, pack)
else
{:error, :not_found} ->

5
lib/pleroma/web/router.ex
View file

@ -209,6 +209,10 @@ defmodule Pleroma.Web.Router do
post("/oauth_app", OAuthAppController, :create)
patch("/oauth_app/:id", OAuthAppController, :update)
delete("/oauth_app/:id", OAuthAppController, :delete)
get("/media_proxy_caches", MediaProxyCacheController, :index)
post("/media_proxy_caches/delete", MediaProxyCacheController, :delete)
post("/media_proxy_caches/purge", MediaProxyCacheController, :purge)
end
scope "/api/pleroma/emoji", Pleroma.Web.PleromaAPI do
@ -463,6 +467,7 @@ defmodule Pleroma.Web.Router do
scope "/api/web", Pleroma.Web do
pipe_through(:authenticated_api)
# Backend-obscure settings blob for MastoFE, don't parse/reuse elsewhere
put("/settings", MastoFEController, :put_settings)
end

140
lib/pleroma/workers/attachments_cleanup_worker.ex
View file

@ -18,13 +18,19 @@ defmodule Pleroma.Workers.AttachmentsCleanupWorker do
},
_job
) do
hrefs =
Enum.flat_map(attachments, fn attachment ->
Enum.map(attachment["url"], & &1["href"])
end)
attachments
|> Enum.flat_map(fn item -> Enum.map(item["url"], & &1["href"]) end)
|> fetch_objects
|> prepare_objects(actor, Enum.map(attachments, & &1["name"]))
|> filter_objects
|> do_clean
names = Enum.map(attachments, & &1["name"])
{:ok, :success}
end
def perform(%{"op" => "cleanup_attachments", "object" => _object}, _job), do: {:ok, :skip}
defp do_clean({object_ids, attachment_urls}) do
uploader = Pleroma.Config.get([Pleroma.Upload, :uploader])
prefix =
@ -39,68 +45,70 @@ defmodule Pleroma.Workers.AttachmentsCleanupWorker do
"/"
)
# find all objects for copies of the attachments, name and actor doesn't matter here
object_ids_and_hrefs =
from(o in Object,
where:
fragment(
"to_jsonb(array(select jsonb_array_elements((?)#>'{url}') ->> 'href' where jsonb_typeof((?)#>'{url}') = 'array'))::jsonb \\?| (?)",
o.data,
o.data,
^hrefs
)
)
# The query above can be time consumptive on large instances until we
# refactor how uploads are stored
|> Repo.all(timeout: :infinity)
# we should delete 1 object for any given attachment, but don't delete
# files if there are more than 1 object for it
|> Enum.reduce(%{}, fn %{
id: id,
data: %{
"url" => [%{"href" => href}],
"actor" => obj_actor,
"name" => name
}
},
acc ->
Map.update(acc, href, %{id: id, count: 1}, fn val ->
case obj_actor == actor and name in names do
true ->
# set id of the actor's object that will be deleted
%{val | id: id, count: val.count + 1}
Enum.each(attachment_urls, fn href ->
href
|> String.trim_leading("#{base_url}/#{prefix}")
|> uploader.delete_file()
end)
false ->
# another actor's object, just increase count to not delete file
%{val | count: val.count + 1}
end
end)
end)
|> Enum.map(fn {href, %{id: id, count: count}} ->
# only delete files that have single instance
with 1 <- count do
href
|> String.trim_leading("#{base_url}/#{prefix}")
|> uploader.delete_file()
{id, href}
else
_ -> {id, nil}
end
end)
object_ids = Enum.map(object_ids_and_hrefs, fn {id, _} -> id end)
from(o in Object, where: o.id in ^object_ids)
|> Repo.delete_all()
object_ids_and_hrefs
|> Enum.filter(fn {_, href} -> not is_nil(href) end)
|> Enum.map(&elem(&1, 1))
|> Pleroma.Web.MediaProxy.Invalidation.purge()
{:ok, :success}
delete_objects(object_ids)
end
def perform(%{"op" => "cleanup_attachments", "object" => _object}, _job), do: {:ok, :skip}
defp delete_objects([_ | _] = object_ids) do
Repo.delete_all(from(o in Object, where: o.id in ^object_ids))
end
defp delete_objects(_), do: :ok
# we should delete 1 object for any given attachment, but don't delete
# files if there are more than 1 object for it
defp filter_objects(objects) do
Enum.reduce(objects, {[], []}, fn {href, %{id: id, count: count}}, {ids, hrefs} ->
with 1 <- count do
{ids ++ [id], hrefs ++ [href]}
else
_ -> {ids ++ [id], hrefs}
end
end)
end
defp prepare_objects(objects, actor, names) do
objects
|> Enum.reduce(%{}, fn %{
id: id,
data: %{
"url" => [%{"href" => href}],
"actor" => obj_actor,
"name" => name
}
},
acc ->
Map.update(acc, href, %{id: id, count: 1}, fn val ->
case obj_actor == actor and name in names do
true ->
# set id of the actor's object that will be deleted
%{val | id: id, count: val.count + 1}
false ->
# another actor's object, just increase count to not delete file
%{val | count: val.count + 1}
end
end)
end)
end
defp fetch_objects(hrefs) do
from(o in Object,
where:
fragment(
"to_jsonb(array(select jsonb_array_elements((?)#>'{url}') ->> 'href' where jsonb_typeof((?)#>'{url}') = 'array'))::jsonb \\?| (?)",
o.data,
o.data,
^hrefs
)
)
# The query above can be time consumptive on large instances until we
# refactor how uploads are stored
|> Repo.all(timeout: :infinity)
end
end