Merge branch 'develop' of git.pleroma.social:pleroma/pleroma into last_status_at

lib/pleroma/application.ex

@@ -54,7 +54,6 @@ defmodule Pleroma.Application do
     Config.DeprecationWarnings.warn()
     Pleroma.Web.Plugs.HTTPSecurityPlug.warn_if_disabled()
     Pleroma.ApplicationRequirements.verify!()
-    setup_instrumenters()
     load_custom_modules()
     Pleroma.Docs.JSON.compile()
     limiters_setup()
@@ -91,6 +90,7 @@ defmodule Pleroma.Application do
     # Define workers and child supervisors to be supervised
     children =
       [
+        Pleroma.PromEx,
         Pleroma.Repo,
         Config.TransferTask,
         Pleroma.Emoji,
@@ -170,29 +170,6 @@ defmodule Pleroma.Application do
     end
   end
 
-  defp setup_instrumenters do
-    require Prometheus.Registry
-
-    if Application.get_env(:prometheus, Pleroma.Repo.Instrumenter) do
-      :ok =
-        :telemetry.attach(
-          "prometheus-ecto",
-          [:pleroma, :repo, :query],
-          &Pleroma.Repo.Instrumenter.handle_event/4,
-          %{}
-        )
-
-      Pleroma.Repo.Instrumenter.setup()
-    end
-
-    Pleroma.Web.Endpoint.MetricsExporter.setup()
-    Pleroma.Web.Endpoint.PipelineInstrumenter.setup()
-
-    # Note: disabled until prometheus-phx is integrated into prometheus-phoenix:
-    # Pleroma.Web.Endpoint.Instrumenter.setup()
-    PrometheusPhx.setup()
-  end
-
   defp cachex_children do
     [
       build_cachex("used_captcha", ttl_interval: seconds_valid_interval()),

lib/pleroma/config/transfer_task.ex

@@ -55,8 +55,7 @@ defmodule Pleroma.Config.TransferTask do
 
       started_applications = Application.started_applications()
 
-      # TODO: some problem with prometheus after restart!
-      reject = [nil, :prometheus, :postgrex]
+      reject = [nil, :postgrex]
 
       reject =
         if restart_pleroma? do

lib/pleroma/docs/generator.ex

@@ -17,6 +17,8 @@ defmodule Pleroma.Docs.Generator do
       # This shouldn't be needed as all modules are expected to have module_info/1,
       # but in test enviroments some transient modules `:elixir_compiler_XX`
       # are loaded for some reason (where XX is a random integer).
+      Code.ensure_loaded(module)
+
       if function_exported?(module, :module_info, 1) do
         module.module_info(:attributes)
         |> Keyword.get_values(:behaviour)

lib/pleroma/helpers/media_helper.ex

@@ -8,11 +8,12 @@ defmodule Pleroma.Helpers.MediaHelper do
   """
 
   alias Pleroma.HTTP
+  alias Vix.Vips.Operation
 
   require Logger
 
   def missing_dependencies do
-    Enum.reduce([imagemagick: "convert", ffmpeg: "ffmpeg"], [], fn {sym, executable}, acc ->
+    Enum.reduce([ffmpeg: "ffmpeg"], [], fn {sym, executable}, acc ->
       if Pleroma.Utils.command_available?(executable) do
         acc
       else
@@ -22,54 +23,22 @@ defmodule Pleroma.Helpers.MediaHelper do
   end
 
   def image_resize(url, options) do
-    with executable when is_binary(executable) <- System.find_executable("convert"),
-         {:ok, args} <- prepare_image_resize_args(options),
-         {:ok, env} <- HTTP.get(url, [], pool: :media),
-         {:ok, fifo_path} <- mkfifo() do
-      args = List.flatten([fifo_path, args])
-      run_fifo(fifo_path, env, executable, args)
+    with {:ok, env} <- HTTP.get(url, [], pool: :media),
+         {:ok, resized} <-
+           Operation.thumbnail_buffer(env.body, options.max_width,
+             height: options.max_height,
+             size: :VIPS_SIZE_DOWN
+           ) do
+      if options[:format] == "png" do
+        Operation.pngsave_buffer(resized, Q: options[:quality])
+      else
+        Operation.jpegsave_buffer(resized, Q: options[:quality], interlace: true)
+      end
     else
-      nil -> {:error, {:convert, :command_not_found}}
       {:error, _} = error -> error
     end
   end
 
-  defp prepare_image_resize_args(
-         %{max_width: max_width, max_height: max_height, format: "png"} = options
-       ) do
-    quality = options[:quality] || 85
-    resize = Enum.join([max_width, "x", max_height, ">"])
-
-    args = [
-      "-resize",
-      resize,
-      "-quality",
-      to_string(quality),
-      "png:-"
-    ]
-
-    {:ok, args}
-  end
-
-  defp prepare_image_resize_args(%{max_width: max_width, max_height: max_height} = options) do
-    quality = options[:quality] || 85
-    resize = Enum.join([max_width, "x", max_height, ">"])
-
-    args = [
-      "-interlace",
-      "Plane",
-      "-resize",
-      resize,
-      "-quality",
-      to_string(quality),
-      "jpg:-"
-    ]
-
-    {:ok, args}
-  end
-
-  defp prepare_image_resize_args(_), do: {:error, :missing_options}
-
   # Note: video thumbnail is intentionally not resized (always has original dimensions)
   def video_framegrab(url) do
     with executable when is_binary(executable) <- System.find_executable("ffmpeg"),

lib/pleroma/instances/instance.ex

@@ -97,13 +97,9 @@ defmodule Pleroma.Instances.Instance do
   def reachable?(url_or_host) when is_binary(url_or_host), do: true
 
   def set_reachable(url_or_host) when is_binary(url_or_host) do
-    with host <- host(url_or_host),
-         %Instance{} = existing_record <- Repo.get_by(Instance, %{host: host}) do
-      {:ok, _instance} =
-        existing_record
-        |> changeset(%{unreachable_since: nil})
-        |> Repo.update()
-    end
+    %Instance{host: host(url_or_host)}
+    |> changeset(%{unreachable_since: nil})
+    |> Repo.insert(on_conflict: {:replace, [:unreachable_since]}, conflict_target: :host)
   end
 
   def set_reachable(_), do: {:error, nil}

lib/pleroma/prom_ex.ex

@@ -0,0 +1,49 @@
+defmodule Pleroma.PromEx do
+  use PromEx, otp_app: :pleroma
+
+  alias PromEx.Plugins
+
+  @impl true
+  def plugins do
+    [
+      # PromEx built in plugins
+      Plugins.Application,
+      Plugins.Beam,
+      {Plugins.Phoenix, router: Pleroma.Web.Router, endpoint: Pleroma.Web.Endpoint},
+      Plugins.Ecto,
+      Plugins.Oban
+      # Plugins.PhoenixLiveView,
+      # Plugins.Absinthe,
+      # Plugins.Broadway,
+
+      # Add your own PromEx metrics plugins
+      # Pleroma.Users.PromExPlugin
+    ]
+  end
+
+  @impl true
+  def dashboard_assigns do
+    [
+      datasource_id: Pleroma.Config.get([Pleroma.PromEx, :datasource]),
+      default_selected_interval: "30s"
+    ]
+  end
+
+  @impl true
+  def dashboards do
+    [
+      # PromEx built in Grafana dashboards
+      {:prom_ex, "application.json"},
+      {:prom_ex, "beam.json"},
+      {:prom_ex, "phoenix.json"},
+      {:prom_ex, "ecto.json"},
+      {:prom_ex, "oban.json"}
+      # {:prom_ex, "phoenix_live_view.json"},
+      # {:prom_ex, "absinthe.json"},
+      # {:prom_ex, "broadway.json"},
+
+      # Add your dashboard definitions here with the format: {:otp_app, "path_in_priv"}
+      # {:pleroma, "/grafana_dashboards/user_metrics.json"}
+    ]
+  end
+end

lib/pleroma/repo.ex

@@ -11,8 +11,6 @@ defmodule Pleroma.Repo do
   import Ecto.Query
   require Logger
 
-  defmodule Instrumenter, do: use(Prometheus.EctoInstrumenter)
-
   @doc """
   Dynamically loads the repository url from the
   DATABASE_URL environment variable.

lib/pleroma/upload/filter/analyze_metadata.ex

@@ -8,22 +8,23 @@ defmodule Pleroma.Upload.Filter.AnalyzeMetadata do
   """
   require Logger
 
+  alias Vix.Vips.Image
+  alias Vix.Vips.Operation
+
   @behaviour Pleroma.Upload.Filter
 
   @spec filter(Pleroma.Upload.t()) ::
           {:ok, :filtered, Pleroma.Upload.t()} | {:ok, :noop} | {:error, String.t()}
   def filter(%Pleroma.Upload{tempfile: file, content_type: "image" <> _} = upload) do
     try do
-      image =
-        file
-        |> Mogrify.open()
-        |> Mogrify.verbose()
+      {:ok, image} = Image.new_from_file(file)
+      {width, height} = {Image.width(image), Image.height(image)}
 
       upload =
         upload
-        |> Map.put(:width, image.width)
-        |> Map.put(:height, image.height)
-        |> Map.put(:blurhash, get_blurhash(file))
+        |> Map.put(:width, width)
+        |> Map.put(:height, height)
+        |> Map.put(:blurhash, get_blurhash(image))
 
       {:ok, :filtered, upload}
     rescue
@@ -53,7 +54,7 @@ defmodule Pleroma.Upload.Filter.AnalyzeMetadata do
   def filter(_), do: {:ok, :noop}
 
   defp get_blurhash(file) do
-    with {:ok, blurhash} <- :eblurhash.magick(file) do
+    with {:ok, blurhash} <- vips_blurhash(file) do
       blurhash
     else
       _ -> nil
@@ -77,7 +78,28 @@ defmodule Pleroma.Upload.Filter.AnalyzeMetadata do
       %{width: width, height: height}
     else
       nil -> {:error, {:ffprobe, :command_not_found}}
-      {:error, _} = error -> error
+      error -> {:error, error}
+    end
+  end
+
+  defp vips_blurhash(%Vix.Vips.Image{} = image) do
+    with {:ok, resized_image} <- Operation.thumbnail_image(image, 100),
+         {height, width} <- {Image.height(resized_image), Image.width(resized_image)},
+         max <- max(height, width),
+         {x, y} <- {max(round(width * 5 / max), 1), max(round(height * 5 / max), 1)} do
+      {:ok, rgb} =
+        if Image.has_alpha?(resized_image) do
+          # remove alpha channel
+          resized_image
+          |> Operation.extract_band!(0, n: 3)
+          |> Image.write_to_binary()
+        else
+          Image.write_to_binary(resized_image)
+        end
+
+      Blurhash.encode(rgb, width, height, x, y)
+    else
+      _ -> nil
     end
   end
 end

lib/pleroma/web/activity_pub/mrf.ex

@@ -54,6 +54,8 @@ defmodule Pleroma.Web.ActivityPub.MRF do
   @required_description_keys [:key, :related_policy]
 
   def filter_one(policy, message) do
+    Code.ensure_loaded(policy)
+
     should_plug_history? =
       if function_exported?(policy, :history_awareness, 0) do
         policy.history_awareness()
@@ -188,6 +190,8 @@ defmodule Pleroma.Web.ActivityPub.MRF do
 
   def config_descriptions(policies) do
     Enum.reduce(policies, @mrf_config_descriptions, fn policy, acc ->
+      Code.ensure_loaded(policy)
+
       if function_exported?(policy, :config_description, 0) do
         description =
           @default_description

lib/pleroma/web/activity_pub/utils.ex

@@ -7,6 +7,7 @@ defmodule Pleroma.Web.ActivityPub.Utils do
   alias Ecto.UUID
   alias Pleroma.Activity
   alias Pleroma.Config
+  alias Pleroma.EctoType.ActivityPub.ObjectValidators.ObjectID
   alias Pleroma.Maps
   alias Pleroma.Notification
   alias Pleroma.Object
@@ -852,9 +853,11 @@ defmodule Pleroma.Web.ActivityPub.Utils do
     [actor | reported_activities] = activity.data["object"]
 
     stripped_activities =
-      Enum.map(reported_activities, fn
-        act when is_map(act) -> act["id"]
-        act when is_binary(act) -> act
+      Enum.reduce(reported_activities, [], fn act, acc ->
+        case ObjectID.cast(act) do
+          {:ok, act} -> [act | acc]
+          _ -> acc
+        end
       end)
 
     new_data = put_in(activity.data, ["object"], [actor | stripped_activities])

lib/pleroma/web/api_spec/operations/pleroma_scrobble_operation.ex

@@ -59,6 +59,7 @@ defmodule Pleroma.Web.ApiSpec.PleromaScrobbleOperation do
         album: %Schema{type: :string, description: "The album of the media playing"},
         artist: %Schema{type: :string, description: "The artist of the media playing"},
         length: %Schema{type: :integer, description: "The length of the media playing"},
+        externalLink: %Schema{type: :string, description: "A URL referencing the media playing"},
         visibility: %Schema{
           allOf: [VisibilityScope],
           default: "public",
@@ -69,7 +70,8 @@ defmodule Pleroma.Web.ApiSpec.PleromaScrobbleOperation do
         "title" => "Some Title",
         "artist" => "Some Artist",
         "album" => "Some Album",
-        "length" => 180_000
+        "length" => 180_000,
+        "externalLink" => "https://www.last.fm/music/Some+Artist/_/Some+Title"
       }
     }
   end
@@ -83,6 +85,7 @@ defmodule Pleroma.Web.ApiSpec.PleromaScrobbleOperation do
         title: %Schema{type: :string, description: "The title of the media playing"},
         album: %Schema{type: :string, description: "The album of the media playing"},
         artist: %Schema{type: :string, description: "The artist of the media playing"},
+        externalLink: %Schema{type: :string, description: "A URL referencing the media playing"},
         length: %Schema{
           type: :integer,
           description: "The length of the media playing",
@@ -97,6 +100,7 @@ defmodule Pleroma.Web.ApiSpec.PleromaScrobbleOperation do
         "artist" => "Some Artist",
         "album" => "Some Album",
         "length" => 180_000,
+        "externalLink" => "https://www.last.fm/music/Some+Artist/_/Some+Title",
         "created_at" => "2019-09-28T12:40:45.000Z"
       }
     }

lib/pleroma/web/common_api/activity_draft.ex

@@ -83,7 +83,7 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do
   defp listen_object(draft) do
     object =
       draft.params
-      |> Map.take([:album, :artist, :title, :length])
+      |> Map.take([:album, :artist, :title, :length, :externalLink])
       |> Map.new(fn {key, value} -> {to_string(key), value} end)
       |> Map.put("type", "Audio")
       |> Map.put("to", draft.to)

lib/pleroma/web/endpoint.ex

@@ -151,47 +151,6 @@ defmodule Pleroma.Web.Endpoint do
 
   plug(Pleroma.Web.Plugs.RemoteIp)
 
-  defmodule Instrumenter do
-    use Prometheus.PhoenixInstrumenter
-  end
-
-  defmodule PipelineInstrumenter do
-    use Prometheus.PlugPipelineInstrumenter
-  end
-
-  defmodule MetricsExporter do
-    use Prometheus.PlugExporter
-  end
-
-  defmodule MetricsExporterCaller do
-    @behaviour Plug
-
-    def init(opts), do: opts
-
-    def call(conn, opts) do
-      prometheus_config = Application.get_env(:prometheus, MetricsExporter, [])
-      ip_whitelist = List.wrap(prometheus_config[:ip_whitelist])
-
-      cond do
-        !prometheus_config[:enabled] ->
-          conn
-
-        ip_whitelist != [] and
-            !Enum.find(ip_whitelist, fn ip ->
-              Pleroma.Helpers.InetHelper.parse_address(ip) == {:ok, conn.remote_ip}
-            end) ->
-          conn
-
-        true ->
-          MetricsExporter.call(conn, opts)
-      end
-    end
-  end
-
-  plug(PipelineInstrumenter)
-
-  plug(MetricsExporterCaller)
-
   plug(Pleroma.Web.Router)
 
   @doc """

lib/pleroma/web/fallback/redirect_controller.ex

@@ -17,10 +17,28 @@ defmodule Pleroma.Web.Fallback.RedirectController do
     |> json(%{error: "Not implemented"})
   end
 
+  def add_generated_metadata(page_content, extra \\ "") do
+    title = "<title>#{Pleroma.Config.get([:instance, :name])}</title>"
+    favicon = "<link rel='icon' href='#{Pleroma.Config.get([:instance, :favicon])}'>"
+    manifest = "<link rel='manifest' href='/manifest.json'>"
+
+    page_content
+    |> String.replace(
+      "<!--server-generated-meta-->",
+      title <> favicon <> manifest <> extra
+    )
+  end
+
   def redirector(conn, _params, code \\ 200) do
+    {:ok, index_content} = File.read(index_file_path())
+
+    response =
+      index_content
+      |> add_generated_metadata()
+
     conn
     |> put_resp_content_type("text/html")
-    |> send_file(code, index_file_path())
+    |> send_resp(code, response)
   end
 
   def redirector_with_meta(conn, %{"maybe_nickname_or_id" => maybe_nickname_or_id} = params) do
@@ -34,14 +52,12 @@ defmodule Pleroma.Web.Fallback.RedirectController do
 
   def redirector_with_meta(conn, params) do
     {:ok, index_content} = File.read(index_file_path())
-
     tags = build_tags(conn, params)
     preloads = preload_data(conn, params)
-    title = "<title>#{Pleroma.Config.get([:instance, :name])}</title>"
 
     response =
       index_content
-      |> String.replace("<!--server-generated-meta-->", tags <> preloads <> title)
+      |> add_generated_metadata(tags <> preloads)
 
     conn
     |> put_resp_content_type("text/html")
@@ -55,11 +71,10 @@ defmodule Pleroma.Web.Fallback.RedirectController do
   def redirector_with_preload(conn, params) do
     {:ok, index_content} = File.read(index_file_path())
     preloads = preload_data(conn, params)
-    title = "<title>#{Pleroma.Config.get([:instance, :name])}</title>"
 
     response =
       index_content
-      |> String.replace("<!--server-generated-meta-->", preloads <> title)
+      |> add_generated_metadata(preloads)
 
     conn
     |> put_resp_content_type("text/html")

lib/pleroma/web/pleroma_api/views/scrobble_view.ex

@@ -27,6 +27,7 @@ defmodule Pleroma.Web.PleromaAPI.ScrobbleView do
       title: object.data["title"] |> HTML.strip_tags(),
       artist: object.data["artist"] |> HTML.strip_tags(),
       album: object.data["album"] |> HTML.strip_tags(),
+      externalLink: object.data["externalLink"],
       length: object.data["length"]
     }
   end

lib/pleroma/web/router.ex

@@ -224,6 +224,12 @@ defmodule Pleroma.Web.Router do
     post("/remote_interaction", UtilController, :remote_interaction)
   end
 
+  scope "/api/v1/pleroma", Pleroma.Web.PleromaAPI do
+    pipe_through(:pleroma_api)
+
+    get("/federation_status", InstancesController, :show)
+  end
+
   scope "/api/v1/pleroma", Pleroma.Web do
     pipe_through(:pleroma_api)
     post("/uploader_callback/:upload_path", UploaderController, :callback)
@@ -604,7 +610,6 @@ defmodule Pleroma.Web.Router do
   scope "/api/v1/pleroma", Pleroma.Web.PleromaAPI do
     pipe_through(:api)
     get("/accounts/:id/scrobbles", ScrobbleController, :index)
-    get("/federation_status", InstancesController, :show)
   end
 
   scope "/api/v2/pleroma", Pleroma.Web.PleromaAPI do