hj/pleroma
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Merge branch 'develop' of git.pleroma.social:pleroma/pleroma into seanking/pleroma-fix_install_fe_bug

Browse source
This commit is contained in:
lain 2020-09-03 11:29:39 +02:00
commit f26b580e80
31 changed files with 283 additions and 157 deletions
Download patch file Download diff file Expand all files Collapse all files

4
lib/mix/tasks/pleroma/frontend.ex
View file

@ -124,7 +124,9 @@ defmodule Mix.Tasks.Pleroma.Frontend do
url = String.replace(frontend_info["build_url"], "${ref}", frontend_info["ref"])
with {:ok, %{status: 200, body: zip_body}} <-
Pleroma.HTTP.get(url, [], timeout: 120_000, recv_timeout: 120_000) do
Pleroma.HTTP.get(url, [],
adapter: [pool: :media, timeout: 120_000, recv_timeout: 120_000]
) do
unzip(zip_body, dest)
else
e -> {:error, e}

17
lib/pleroma/application.ex
View file

@ -22,13 +22,18 @@ defmodule Pleroma.Application do
def repository, do: @repository
def user_agent do
case Config.get([:http, :user_agent], :default) do
:default ->
info = "#{Pleroma.Web.base_url()} <#{Config.get([:instance, :email], "")}>"
named_version() <> "; " <> info
if Process.whereis(Pleroma.Web.Endpoint) do
case Config.get([:http, :user_agent], :default) do
:default ->
info = "#{Pleroma.Web.base_url()} <#{Config.get([:instance, :email], "")}>"
named_version() <> "; " <> info
custom ->
custom
custom ->
custom
end
else
# fallback, if endpoint is not started yet
"Pleroma Data Loader"
end
end

35
lib/pleroma/gun/connection_pool/worker.ex
View file

@ -83,17 +83,25 @@ defmodule Pleroma.Gun.ConnectionPool.Worker do
end)
{ref, state} = pop_in(state.client_monitors[client_pid])
Process.demonitor(ref)
timer =
if used_by == [] do
max_idle = Pleroma.Config.get([:connections_pool, :max_idle_time], 30_000)
Process.send_after(self(), :idle_close, max_idle)
# DOWN message can receive right after `remove_client` call and cause worker to terminate
state =
if is_nil(ref) do
state
else
nil
Process.demonitor(ref)
timer =
if used_by == [] do
max_idle = Pleroma.Config.get([:connections_pool, :max_idle_time], 30_000)
Process.send_after(self(), :idle_close, max_idle)
else
nil
end
%{state | timer: timer}
end
{:reply, :ok, %{state | timer: timer}, :hibernate}
{:reply, :ok, state, :hibernate}
end
@impl true
@ -103,16 +111,21 @@ defmodule Pleroma.Gun.ConnectionPool.Worker do
{:stop, :normal, state}
end
@impl true
def handle_info({:gun_up, _pid, _protocol}, state) do
{:noreply, state, :hibernate}
end
# Gracefully shutdown if the connection got closed without any streams left
@impl true
def handle_info({:gun_down, _pid, _protocol, _reason, []}, state) do
{:stop, :normal, state}
end
# Otherwise, shutdown with an error
# Otherwise, wait for retry
@impl true
def handle_info({:gun_down, _pid, _protocol, _reason, _killed_streams} = down_message, state) do
{:stop, {:error, down_message}, state}
def handle_info({:gun_down, _pid, _protocol, _reason, _killed_streams}, state) do
{:noreply, state, :hibernate}
end
@impl true

5
lib/pleroma/html.ex
View file

@ -109,8 +109,9 @@ defmodule Pleroma.HTML do
result =
content
|> Floki.parse_fragment!()
|> Floki.filter_out("a.mention,a.hashtag,a.attachment,a[rel~=\"tag\"]")
|> Floki.attribute("a", "href")
|> Floki.find("a:not(.mention,.hashtag,.attachment,[rel~=\"tag\"])")
|> Enum.take(1)
|> Floki.attribute("href")
|> Enum.at(0)
{:commit, {:ok, result}}

19
lib/pleroma/http/adapter_helper.ex
View file

@ -11,7 +11,6 @@ defmodule Pleroma.HTTP.AdapterHelper do
@type proxy_type() :: :socks4 | :socks5
@type host() :: charlist() | :inet.ip_address()
alias Pleroma.Config
alias Pleroma.HTTP.AdapterHelper
require Logger
@ -44,27 +43,13 @@ defmodule Pleroma.HTTP.AdapterHelper do
@spec options(URI.t(), keyword()) :: keyword()
def options(%URI{} = uri, opts \\ []) do
@defaults
|> put_timeout()
|> Keyword.merge(opts)
|> adapter_helper().options(uri)
end
# For Hackney, this is the time a connection can stay idle in the pool.
# For Gun, this is the timeout to receive a message from Gun.
defp put_timeout(opts) do
{config_key, default} =
if adapter() == Tesla.Adapter.Gun do
{:pools, Config.get([:pools, :default, :timeout], 5_000)}
else
{:hackney_pools, 10_000}
end
timeout = Config.get([config_key, opts[:pool], :timeout], default)
Keyword.merge(opts, timeout: timeout)
end
@spec get_conn(URI.t(), keyword()) :: {:ok, keyword()} | {:error, atom()}
def get_conn(uri, opts), do: adapter_helper().get_conn(uri, opts)
defp adapter, do: Application.get_env(:tesla, :adapter)
defp adapter_helper do

33
lib/pleroma/http/adapter_helper/gun.ex
View file

@ -5,6 +5,7 @@
defmodule Pleroma.HTTP.AdapterHelper.Gun do
@behaviour Pleroma.HTTP.AdapterHelper
alias Pleroma.Config
alias Pleroma.Gun.ConnectionPool
alias Pleroma.HTTP.AdapterHelper
@ -14,31 +15,46 @@ defmodule Pleroma.HTTP.AdapterHelper.Gun do
connect_timeout: 5_000,
domain_lookup_timeout: 5_000,
tls_handshake_timeout: 5_000,
retry: 0,
retry: 1,
retry_timeout: 1000,
await_up_timeout: 5_000
]
@type pool() :: :federation | :upload | :media | :default
@spec options(keyword(), URI.t()) :: keyword()
def options(incoming_opts \\ [], %URI{} = uri) do
proxy =
Pleroma.Config.get([:http, :proxy_url])
[:http, :proxy_url]
|> Config.get()
|> AdapterHelper.format_proxy()
config_opts = Pleroma.Config.get([:http, :adapter], [])
config_opts = Config.get([:http, :adapter], [])
@defaults
|> Keyword.merge(config_opts)
|> add_scheme_opts(uri)
|> AdapterHelper.maybe_add_proxy(proxy)
|> Keyword.merge(incoming_opts)
|> put_timeout()
end
defp add_scheme_opts(opts, %{scheme: "http"}), do: opts
defp add_scheme_opts(opts, %{scheme: "https"}) do
opts
|> Keyword.put(:certificates_verification, true)
Keyword.put(opts, :certificates_verification, true)
end
defp put_timeout(opts) do
# this is the timeout to receive a message from Gun
Keyword.put_new(opts, :timeout, pool_timeout(opts[:pool]))
end
@spec pool_timeout(pool()) :: non_neg_integer()
def pool_timeout(pool) do
default = Config.get([:pools, :default, :timeout], 5_000)
Config.get([:pools, pool, :timeout], default)
end
@spec get_conn(URI.t(), keyword()) :: {:ok, keyword()} | {:error, atom()}
@ -51,11 +67,11 @@ defmodule Pleroma.HTTP.AdapterHelper.Gun do
@prefix Pleroma.Gun.ConnectionPool
def limiter_setup do
wait = Pleroma.Config.get([:connections_pool, :connection_acquisition_wait])
retries = Pleroma.Config.get([:connections_pool, :connection_acquisition_retries])
wait = Config.get([:connections_pool, :connection_acquisition_wait])
retries = Config.get([:connections_pool, :connection_acquisition_retries])
:pools
|> Pleroma.Config.get([])
|> Config.get([])
|> Enum.each(fn {name, opts} ->
max_running = Keyword.get(opts, :size, 50)
max_waiting = Keyword.get(opts, :max_waiting, 10)
@ -69,7 +85,6 @@ defmodule Pleroma.HTTP.AdapterHelper.Gun do
case result do
:ok -> :ok
{:error, :existing} -> :ok
e -> raise e
end
end)

2
lib/pleroma/http/ex_aws.ex
View file

@ -11,6 +11,8 @@ defmodule Pleroma.HTTP.ExAws do
@impl true
def request(method, url, body \\ "", headers \\ [], http_opts \\ []) do
http_opts = Keyword.put_new(http_opts, :adapter, pool: :upload)
case HTTP.request(method, url, body, headers, http_opts) do
{:ok, env} ->
{:ok, %{status_code: env.status, headers: env.headers, body: env.body}}

4
lib/pleroma/http/tzdata.ex
View file

@ -11,6 +11,8 @@ defmodule Pleroma.HTTP.Tzdata do
@impl true
def get(url, headers, options) do
options = Keyword.put_new(options, :adapter, pool: :default)
with {:ok, %Tesla.Env{} = env} <- HTTP.get(url, headers, options) do
{:ok, {env.status, env.headers, env.body}}
end
@ -18,6 +20,8 @@ defmodule Pleroma.HTTP.Tzdata do
@impl true
def head(url, headers, options) do
options = Keyword.put_new(options, :adapter, pool: :default)
with {:ok, %Tesla.Env{} = env} <- HTTP.head(url, headers, options) do
{:ok, {env.status, env.headers}}
end

4
lib/pleroma/instances/instance.ex
View file

@ -150,7 +150,9 @@ defmodule Pleroma.Instances.Instance do
defp scrape_favicon(%URI{} = instance_uri) do
try do
with {:ok, %Tesla.Env{body: html}} <-
Pleroma.HTTP.get(to_string(instance_uri), [{:Accept, "text/html"}]),
Pleroma.HTTP.get(to_string(instance_uri), [{"accept", "text/html"}],
adapter: [pool: :media]
),
favicon_rel <-
html
|> Floki.parse_document!()

6
lib/pleroma/object/fetcher.ex
View file

@ -164,12 +164,12 @@ defmodule Pleroma.Object.Fetcher do
date: date
})
[{"signature", signature}]
{"signature", signature}
end
defp sign_fetch(headers, id, date) do
if Pleroma.Config.get([:activitypub, :sign_object_fetches]) do
headers ++ make_signature(id, date)
[make_signature(id, date) | headers]
else
headers
end
@ -177,7 +177,7 @@ defmodule Pleroma.Object.Fetcher do
defp maybe_date_fetch(headers, date) do
if Pleroma.Config.get([:activitypub, :sign_object_fetches]) do
headers ++ [{"date", date}]
[{"date", date} | headers]
else
headers
end

23
lib/pleroma/uploaders/s3.ex
View file

@ -46,12 +46,23 @@ defmodule Pleroma.Uploaders.S3 do
op =
if streaming do
upload.tempfile
|> ExAws.S3.Upload.stream_file()
|> ExAws.S3.upload(bucket, s3_name, [
{:acl, :public_read},
{:content_type, upload.content_type}
])
op =
upload.tempfile
|> ExAws.S3.Upload.stream_file()
|> ExAws.S3.upload(bucket, s3_name, [
{:acl, :public_read},
{:content_type, upload.content_type}
])
if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Gun do
# set s3 upload timeout to respect :upload pool timeout
# timeout should be slightly larger, so s3 can retry upload on fail
timeout = Pleroma.HTTP.AdapterHelper.Gun.pool_timeout(:upload) + 1_000
opts = Keyword.put(op.opts, :timeout, timeout)
Map.put(op, :opts, opts)
else
op
end
else
{:ok, file_data} = File.read(upload.tempfile)

2
lib/pleroma/user/search.ex
View file

@ -116,7 +116,7 @@ defmodule Pleroma.User.Search do
end
defp base_query(_user, false), do: User
defp base_query(user, true), do: User.get_followers_query(user)
defp base_query(user, true), do: User.get_friends_query(user)
defp filter_invisible_users(query) do
from(q in query, where: q.invisible == false)

20
lib/pleroma/web/api_spec/operations/list_operation.ex
View file

@ -114,7 +114,7 @@ defmodule Pleroma.Web.ApiSpec.ListOperation do
description: "Add accounts to the given list.",
operationId: "ListController.add_to_list",
parameters: [id_param()],
requestBody: add_remove_accounts_request(),
requestBody: add_remove_accounts_request(true),
security: [%{"oAuth" => ["write:lists"]}],
responses: %{
200 => Operation.response("Empty object", "application/json", %Schema{type: :object})
@ -127,8 +127,16 @@ defmodule Pleroma.Web.ApiSpec.ListOperation do
tags: ["Lists"],
summary: "Remove accounts from list",
operationId: "ListController.remove_from_list",
parameters: [id_param()],
requestBody: add_remove_accounts_request(),
parameters: [
id_param(),
Operation.parameter(
:account_ids,
:query,
%Schema{type: :array, items: %Schema{type: :string}},
"Array of account IDs"
)
],
requestBody: add_remove_accounts_request(false),
security: [%{"oAuth" => ["write:lists"]}],
responses: %{
200 => Operation.response("Empty object", "application/json", %Schema{type: :object})
@ -171,7 +179,7 @@ defmodule Pleroma.Web.ApiSpec.ListOperation do
)
end
defp add_remove_accounts_request do
defp add_remove_accounts_request(required) when is_boolean(required) do
request_body(
"Parameters",
%Schema{
@ -180,9 +188,9 @@ defmodule Pleroma.Web.ApiSpec.ListOperation do
properties: %{
account_ids: %Schema{type: :array, description: "Array of account IDs", items: FlakeID}
},
required: [:account_ids]
required: required && [:account_ids]
},
required: true
required: required
)
end
end

14
lib/pleroma/web/mastodon_api/controllers/auth_controller.ex
View file

@ -59,17 +59,11 @@ defmodule Pleroma.Web.MastodonAPI.AuthController do
def password_reset(conn, params) do
nickname_or_email = params["email"] || params["nickname"]
with {:ok, _} <- TwitterAPI.password_reset(nickname_or_email) do
conn
|> put_status(:no_content)
|> json("")
else
{:error, "unknown user"} ->
send_resp(conn, :not_found, "")
TwitterAPI.password_reset(nickname_or_email)
{:error, _} ->
send_resp(conn, :bad_request, "")
end
conn
|> put_status(:no_content)
|> json("")
end
defp local_mastodon_root_path(conn) do

6
lib/pleroma/web/mastodon_api/controllers/list_controller.ex
View file

@ -74,7 +74,7 @@ defmodule Pleroma.Web.MastodonAPI.ListController do
# DELETE /api/v1/lists/:id/accounts
def remove_from_list(
%{assigns: %{list: list}, body_params: %{account_ids: account_ids}} = conn,
%{assigns: %{list: list}, params: %{account_ids: account_ids}} = conn,
_
) do
Enum.each(account_ids, fn account_id ->
@ -86,6 +86,10 @@ defmodule Pleroma.Web.MastodonAPI.ListController do
json(conn, %{})
end
def remove_from_list(%{body_params: params} = conn, _) do
remove_from_list(%{conn | params: params}, %{})
end
defp list_by_id_and_user(%{assigns: %{user: user}, params: %{id: id}} = conn, _) do
case Pleroma.List.get(id, user) do
%Pleroma.List{} = list -> assign(conn, :list, list)

16
lib/pleroma/web/mastodon_api/views/status_view.ex
View file

@ -23,6 +23,17 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
import Pleroma.Web.ActivityPub.Visibility, only: [get_visibility: 1, visible_for_user?: 2]
# This is a naive way to do this, just spawning a process per activity
# to fetch the preview. However it should be fine considering
# pagination is restricted to 40 activities at a time
defp fetch_rich_media_for_activities(activities) do
Enum.each(activities, fn activity ->
spawn(fn ->
Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity)
end)
end)
end
# TODO: Add cached version.
defp get_replied_to_activities([]), do: %{}
@ -80,6 +91,11 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
# To do: check AdminAPIControllerTest on the reasons behind nil activities in the list
activities = Enum.filter(opts.activities, & &1)
# Start fetching rich media before doing anything else, so that later calls to get the cards
# only block for timeout in the worst case, as opposed to
# length(activities_with_links) * timeout
fetch_rich_media_for_activities(activities)
replied_to_activities = get_replied_to_activities(activities)
parent_activities =

2
lib/pleroma/web/rich_media/helpers.ex
View file

@ -96,6 +96,6 @@ defmodule Pleroma.Web.RichMedia.Helpers do
@rich_media_options
end
Pleroma.HTTP.get(url, headers, options)
Pleroma.HTTP.get(url, headers, adapter: options)
end
end

64
lib/pleroma/web/rich_media/parser.ex
View file

@ -3,6 +3,8 @@
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.RichMedia.Parser do
require Logger
defp parsers do
Pleroma.Config.get([:rich_media, :parsers])
end
@ -10,17 +12,29 @@ defmodule Pleroma.Web.RichMedia.Parser do
def parse(nil), do: {:error, "No URL provided"}
if Pleroma.Config.get(:env) == :test do
@spec parse(String.t()) :: {:ok, map()} | {:error, any()}
def parse(url), do: parse_url(url)
else
@spec parse(String.t()) :: {:ok, map()} | {:error, any()}
def parse(url) do
try do
Cachex.fetch!(:rich_media_cache, url, fn _ ->
{:commit, parse_url(url)}
end)
|> set_ttl_based_on_image(url)
rescue
e ->
{:error, "Cachex error: #{inspect(e)}"}
with {:ok, data} <- get_cached_or_parse(url),
{:ok, _} <- set_ttl_based_on_image(data, url) do
{:ok, data}
else
error ->
Logger.error(fn -> "Rich media error: #{inspect(error)}" end)
end
end
defp get_cached_or_parse(url) do
case Cachex.fetch!(:rich_media_cache, url, fn _ -> {:commit, parse_url(url)} end) do
{:ok, _data} = res ->
res
{:error, _} = e ->
ttl = Pleroma.Config.get([:rich_media, :failure_backoff], 60_000)
Cachex.expire(:rich_media_cache, url, ttl)
e
end
end
end
@ -47,19 +61,26 @@ defmodule Pleroma.Web.RichMedia.Parser do
config :pleroma, :rich_media,
ttl_setters: [MyModule]
"""
def set_ttl_based_on_image({:ok, data}, url) do
with {:ok, nil} <- Cachex.ttl(:rich_media_cache, url),
ttl when is_number(ttl) <- get_ttl_from_image(data, url) do
Cachex.expire_at(:rich_media_cache, url, ttl * 1000)
{:ok, data}
else
@spec set_ttl_based_on_image(map(), String.t()) ::
{:ok, Integer.t() | :noop} | {:error, :no_key}
def set_ttl_based_on_image(data, url) do
case get_ttl_from_image(data, url) do
{:ok, ttl} when is_number(ttl) ->
ttl = ttl * 1000
case Cachex.expire_at(:rich_media_cache, url, ttl) do
{:ok, true} -> {:ok, ttl}
{:ok, false} -> {:error, :no_key}
end
_ ->
{:ok, data}
{:ok, :noop}
end
end
defp get_ttl_from_image(data, url) do
Pleroma.Config.get([:rich_media, :ttl_setters])
[:rich_media, :ttl_setters]
|> Pleroma.Config.get()
|> Enum.reduce({:ok, nil}, fn
module, {:ok, _ttl} ->
module.ttl(data, url)
@ -70,23 +91,16 @@ defmodule Pleroma.Web.RichMedia.Parser do
end
defp parse_url(url) do
try do
{:ok, %Tesla.Env{body: html}} = Pleroma.Web.RichMedia.Helpers.rich_media_get(url)
with {:ok, %Tesla.Env{body: html}} <- Pleroma.Web.RichMedia.Helpers.rich_media_get(url),
{:ok, html} <- Floki.parse_document(html) do
html
|> parse_html()
|> maybe_parse()
|> Map.put("url", url)
|> clean_parsed_data()
|> check_parsed_data()
rescue
e ->
{:error, "Parsing error: #{inspect(e)} #{inspect(__STACKTRACE__)}"}
end
end
defp parse_html(html), do: Floki.parse_document!(html)
defp maybe_parse(html) do
Enum.reduce_while(parsers(), %{}, fn parser, acc ->
case parser.parse(html, acc) do

15
lib/pleroma/web/rich_media/parsers/ttl/aws_signed_url.ex
View file

@ -10,20 +10,15 @@ defmodule Pleroma.Web.RichMedia.Parser.TTL.AwsSignedUrl do
|> parse_query_params()
|> format_query_params()
|> get_expiration_timestamp()
else
{:error, "Not aws signed url #{inspect(image)}"}
end
end
defp is_aws_signed_url(""), do: nil
defp is_aws_signed_url(nil), do: nil
defp is_aws_signed_url(image) when is_binary(image) do
defp is_aws_signed_url(image) when is_binary(image) and image != "" do
%URI{host: host, query: query} = URI.parse(image)
if String.contains?(host, "amazonaws.com") and String.contains?(query, "X-Amz-Expires") do
image
else
nil
end
String.contains?(host, "amazonaws.com") and String.contains?(query, "X-Amz-Expires")
end
defp is_aws_signed_url(_), do: nil
@ -46,6 +41,6 @@ defmodule Pleroma.Web.RichMedia.Parser.TTL.AwsSignedUrl do
|> Map.get("X-Amz-Date")
|> Timex.parse("{ISO:Basic:Z}")
Timex.to_unix(date) + String.to_integer(Map.get(params, "X-Amz-Expires"))
{:ok, Timex.to_unix(date) + String.to_integer(Map.get(params, "X-Amz-Expires"))}
end
end

13
lib/pleroma/web/twitter_api/twitter_api.ex
View file

@ -72,7 +72,7 @@ defmodule Pleroma.Web.TwitterAPI.TwitterAPI do
def password_reset(nickname_or_email) do
with true <- is_binary(nickname_or_email),
%User{local: true, email: email} = user when is_binary(email) <-
%User{local: true, email: email, deactivated: false} = user when is_binary(email) <-
User.get_by_nickname_or_email(nickname_or_email),
{:ok, token_record} <- Pleroma.PasswordResetToken.create_token(user) do
user
@ -81,17 +81,8 @@ defmodule Pleroma.Web.TwitterAPI.TwitterAPI do
{:ok, :enqueued}
else
false ->
{:error, "bad user identifier"}
%User{local: true, email: nil} ->
_ ->
{:ok, :noop}
%User{local: false} ->
{:error, "remote user"}
nil ->
{:error, "unknown user"}
end
end

4
lib/pleroma/web/web_finger/web_finger.ex
View file

@ -136,12 +136,12 @@ defmodule Pleroma.Web.WebFinger do
def find_lrdd_template(domain) do
with {:ok, %{status: status, body: body}} when status in 200..299 <-
HTTP.get("http://#{domain}/.well-known/host-meta", []) do
HTTP.get("http://#{domain}/.well-known/host-meta") do
get_template_from_xml(body)
else
_ ->
with {:ok, %{body: body, status: status}} when status in 200..299 <-
HTTP.get("https://#{domain}/.well-known/host-meta", []) do
HTTP.get("https://#{domain}/.well-known/host-meta") do
get_template_from_xml(body)
else
e -> {:error, "Can't find LRDD template: #{inspect(e)}"}