Merge branch 'issue/1509' into 'develop'

[#1509]  purge media from cache after delete

See merge request pleroma/pleroma!2539

lib/pleroma/object.ex

@@ -9,11 +9,13 @@ defmodule Pleroma.Object do
   import Ecto.Changeset
 
   alias Pleroma.Activity
+  alias Pleroma.Config
   alias Pleroma.Object
   alias Pleroma.Object.Fetcher
   alias Pleroma.ObjectTombstone
   alias Pleroma.Repo
   alias Pleroma.User
+  alias Pleroma.Workers.AttachmentsCleanupWorker
 
   require Logger
 
@@ -188,27 +190,37 @@ defmodule Pleroma.Object do
   def delete(%Object{data: %{"id" => id}} = object) do
     with {:ok, _obj} = swap_object_with_tombstone(object),
          deleted_activity = Activity.delete_all_by_object_ap_id(id),
-         {:ok, true} <- Cachex.del(:object_cache, "object:#{id}"),
-         {:ok, _} <- Cachex.del(:web_resp_cache, URI.parse(id).path) do
-      with true <- Pleroma.Config.get([:instance, :cleanup_attachments]) do
-        {:ok, _} =
-          Pleroma.Workers.AttachmentsCleanupWorker.enqueue("cleanup_attachments", %{
-            "object" => object
-          })
-      end
+         {:ok, _} <- invalid_object_cache(object) do
+      cleanup_attachments(
+        Config.get([:instance, :cleanup_attachments]),
+        %{"object" => object}
+      )
 
       {:ok, object, deleted_activity}
     end
   end
 
-  def prune(%Object{data: %{"id" => id}} = object) do
+  @spec cleanup_attachments(boolean(), %{required(:object) => map()}) ::
+          {:ok, Oban.Job.t() | nil}
+  def cleanup_attachments(true, %{"object" => _} = params) do
+    AttachmentsCleanupWorker.enqueue("cleanup_attachments", params)
+  end
+
+  def cleanup_attachments(_, _), do: {:ok, nil}
+
+  def prune(%Object{data: %{"id" => _id}} = object) do
     with {:ok, object} <- Repo.delete(object),
-         {:ok, true} <- Cachex.del(:object_cache, "object:#{id}"),
-         {:ok, _} <- Cachex.del(:web_resp_cache, URI.parse(id).path) do
+         {:ok, _} <- invalid_object_cache(object) do
       {:ok, object}
     end
   end
 
+  def invalid_object_cache(%Object{data: %{"id" => id}}) do
+    with {:ok, true} <- Cachex.del(:object_cache, "object:#{id}") do
+      Cachex.del(:web_resp_cache, URI.parse(id).path)
+    end
+  end
+
   def set_cache(%Object{data: %{"id" => ap_id}} = object) do
     Cachex.put(:object_cache, "object:#{ap_id}", object)
     {:ok, object}

lib/pleroma/web/media_proxy/invalidation.ex

@@ -0,0 +1,26 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.MediaProxy.Invalidation do
+  @moduledoc false
+
+  @callback purge(list(String.t()), map()) :: {:ok, String.t()} | {:error, String.t()}
+
+  alias Pleroma.Config
+
+  @spec purge(list(String.t())) :: {:ok, String.t()} | {:error, String.t()}
+  def purge(urls) do
+    [:media_proxy, :invalidation, :enabled]
+    |> Config.get()
+    |> do_purge(urls)
+  end
+
+  defp do_purge(true, urls) do
+    provider = Config.get([:media_proxy, :invalidation, :provider])
+    options = Config.get(provider)
+    provider.purge(urls, options)
+  end
+
+  defp do_purge(_, _), do: :ok
+end

lib/pleroma/web/media_proxy/invalidations/http.ex

@@ -0,0 +1,40 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.MediaProxy.Invalidation.Http do
+  @moduledoc false
+  @behaviour Pleroma.Web.MediaProxy.Invalidation
+
+  require Logger
+
+  @impl Pleroma.Web.MediaProxy.Invalidation
+  def purge(urls, opts) do
+    method = Map.get(opts, :method, :purge)
+    headers = Map.get(opts, :headers, [])
+    options = Map.get(opts, :options, [])
+
+    Logger.debug("Running cache purge: #{inspect(urls)}")
+
+    Enum.each(urls, fn url ->
+      with {:error, error} <- do_purge(method, url, headers, options) do
+        Logger.error("Error while cache purge: url - #{url}, error: #{inspect(error)}")
+      end
+    end)
+
+    {:ok, "success"}
+  end
+
+  defp do_purge(method, url, headers, options) do
+    case Pleroma.HTTP.request(method, url, "", headers, options) do
+      {:ok, %{status: status} = env} when 400 <= status and status < 500 ->
+        {:error, env}
+
+      {:error, error} = error ->
+        error
+
+      _ ->
+        {:ok, "success"}
+    end
+  end
+end

lib/pleroma/web/media_proxy/invalidations/script.ex

@@ -0,0 +1,41 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.MediaProxy.Invalidation.Script do
+  @moduledoc false
+
+  @behaviour Pleroma.Web.MediaProxy.Invalidation
+
+  require Logger
+
+  @impl Pleroma.Web.MediaProxy.Invalidation
+  def purge(urls, %{script_path: script_path} = _options) do
+    args =
+      urls
+      |> List.wrap()
+      |> Enum.uniq()
+      |> Enum.join(" ")
+
+    path = Path.expand(script_path)
+
+    Logger.debug("Running cache purge: #{inspect(urls)}, #{path}")
+
+    case do_purge(path, [args]) do
+      {result, exit_status} when exit_status > 0 ->
+        Logger.error("Error while cache purge: #{inspect(result)}")
+        {:error, inspect(result)}
+
+      _ ->
+        {:ok, "success"}
+    end
+  end
+
+  def purge(_, _), do: {:error, "not found script path"}
+
+  defp do_purge(path, args) do
+    System.cmd(path, args)
+  rescue
+    error -> {inspect(error), 1}
+  end
+end

lib/pleroma/workers/attachments_cleanup_worker.ex

@@ -27,8 +27,20 @@ defmodule Pleroma.Workers.AttachmentsCleanupWorker do
 
     uploader = Pleroma.Config.get([Pleroma.Upload, :uploader])
 
+    prefix =
+      case Pleroma.Config.get([Pleroma.Upload, :base_url]) do
+        nil -> "media"
+        _ -> ""
+      end
+
+    base_url =
+      String.trim_trailing(
+        Pleroma.Config.get([Pleroma.Upload, :base_url], Pleroma.Web.base_url()),
+        "/"
+      )
+
     # find all objects for copies of the attachments, name and actor doesn't matter here
-    delete_ids =
+    object_ids_and_hrefs =
       from(o in Object,
         where:
           fragment(
@@ -67,29 +79,28 @@ defmodule Pleroma.Workers.AttachmentsCleanupWorker do
       |> Enum.map(fn {href, %{id: id, count: count}} ->
         # only delete files that have single instance
         with 1 <- count do
-          prefix =
-            case Pleroma.Config.get([Pleroma.Upload, :base_url]) do
-              nil -> "media"
-              _ -> ""
-            end
+          href
+          |> String.trim_leading("#{base_url}/#{prefix}")
+          |> uploader.delete_file()
 
-          base_url =
-            String.trim_trailing(
-              Pleroma.Config.get([Pleroma.Upload, :base_url], Pleroma.Web.base_url()),
-              "/"
-            )
-
-          file_path = String.trim_leading(href, "#{base_url}/#{prefix}")
-
-          uploader.delete_file(file_path)
+          {id, href}
+        else
+          _ -> {id, nil}
         end
-
-        id
       end)
 
-    from(o in Object, where: o.id in ^delete_ids)
+    object_ids = Enum.map(object_ids_and_hrefs, fn {id, _} -> id end)
+
+    from(o in Object, where: o.id in ^object_ids)
     |> Repo.delete_all()
+
+    object_ids_and_hrefs
+    |> Enum.filter(fn {_, href} -> not is_nil(href) end)
+    |> Enum.map(&elem(&1, 1))
+    |> Pleroma.Web.MediaProxy.Invalidation.purge()
+
+    {:ok, :success}
   end
 
-  def perform(%{"op" => "cleanup_attachments", "object" => _object}, _job), do: :ok
+  def perform(%{"op" => "cleanup_attachments", "object" => _object}, _job), do: {:ok, :skip}
 end