Merge remote-tracking branch 'origin/media-management' into shigusegubu-new

It will return less than 50 for some reason

changelog.d/user-media-management.add

@@ -0,0 +1 @@
+Added information about duplicates and usage when fetching attachments and allow to delete said attachments via `DELETE /api/v1/media/:id`

lib/pleroma/upload.ex

@@ -35,8 +35,14 @@ defmodule Pleroma.Upload do
   """
   alias Ecto.UUID
   alias Pleroma.Maps
+  alias Pleroma.Object
+  alias Pleroma.Pagination
+  alias Pleroma.Repo
   alias Pleroma.Utils.URIEncoding
   alias Pleroma.Web.ActivityPub.Utils
+
+  import Ecto.Query
+
   require Logger
 
   @type source ::
@@ -301,4 +307,48 @@ defmodule Pleroma.Upload do
         public_endpoint || upload_base_url
     end
   end
+
+  def list_duplicate_attachments(%{"id" => id, "url" => [%{"href" => href}]}) do
+    clean_href = %URI{URI.parse(href) | query: nil} |> URI.to_string()
+
+    same_attachment_count =
+      from(
+        o in Object,
+        where:
+          fragment("STARTS_WITH((?)->'url'->0->>'href', (?))", o.data, ^clean_href) and
+            fragment("(?)->>'id' <> (?)", o.data, ^id)
+      )
+      |> Repo.all()
+  end
+
+  def list_objects_with_attachment(href) do
+    clean_href = %URI{URI.parse(href) | query: nil} |> URI.to_string()
+
+    # Ugly as sin, but I could not figure out a better way to do this in ecto
+    # This tries to find any objects with this attachment
+    # This DOES NOT handle chat messages and SKIPS them,
+    # because they have attachments as an object instead of an array
+    Repo.query!(
+      """
+          SELECT * FROM
+          (SELECT objects.data->>'id' AS id, jsonb_array_elements(objects.data->'attachment') as json_attachments FROM objects
+              WHERE jsonb_typeof(objects.data->'attachment') = 'array')
+          WHERE STARTS_WITH(json_attachments->'url'->0->>'href', $1)
+      """,
+      [clean_href]
+    ).rows
+    |> Enum.map(&List.first(&1))
+  end
+
+  def list_uploaded_by(user, offset, limit) do
+    from(obj in Object,
+      where:
+        fragment("(?)->>'type' = ?", obj.data, "Document") and
+          fragment("(?)->>'actor' = ?", obj.data, ^user.ap_id),
+      order_by: [desc: obj.id],
+      limit: ^limit,
+      offset: ^offset
+    )
+    |> Repo.all()
+  end
 end

lib/pleroma/web/api_spec/operations/media_operation.ex

@@ -8,6 +8,7 @@ defmodule Pleroma.Web.ApiSpec.MediaOperation do
   alias Pleroma.Web.ApiSpec.Helpers
   alias Pleroma.Web.ApiSpec.Schemas.ApiError
   alias Pleroma.Web.ApiSpec.Schemas.Attachment
+  alias Pleroma.Web.ApiSpec.Schemas.BooleanLike
 
   def open_api_operation(action) do
     operation = String.to_existing_atom("#{action}_operation")
@@ -73,6 +74,21 @@ defmodule Pleroma.Web.ApiSpec.MediaOperation do
     }
   end
 
+  def delete_operation do
+    %Operation{
+      tags: ["Media attachments"],
+      summary: "Delete attachment",
+      description: "Deletes an attachment.",
+      operationId: "MediaController.delete",
+      security: [%{"oAuth" => ["write:media"]}],
+      parameters: [id_param()],
+      responses: %{
+        200 => Operation.response("Media", "text/plain", nil),
+        400 => Operation.response("Media", "application/json", ApiError)
+      }
+    }
+  end
+
   defp update_request do
     %Schema{
       title: "MediaUpdateRequest",
@@ -101,7 +117,15 @@ defmodule Pleroma.Web.ApiSpec.MediaOperation do
       tags: ["Media attachments"],
       summary: "Attachment",
       operationId: "MediaController.show",
-      parameters: [id_param()],
+      parameters: [
+        id_param(),
+        Operation.parameter(
+          :attachment_usage,
+          :query,
+          %Schema{type: :boolean, default: false},
+          "Whether to include known usages of attachment"
+        )
+      ],
       security: [%{"oAuth" => ["read:media"]}],
       responses: %{
         200 => Operation.response("Media", "application/json", Attachment),

lib/pleroma/web/api_spec/operations/pleroma_util_operation.ex

@@ -6,6 +6,7 @@ defmodule Pleroma.Web.ApiSpec.PleromaUtilOperation do
   alias OpenApiSpex.Operation
   alias OpenApiSpex.Schema
   alias Pleroma.Web.ApiSpec.Schemas.ApiError
+  alias Pleroma.Web.ApiSpec.Schemas.Attachment
   alias Pleroma.Web.ApiSpec.Schemas.BooleanLike
 
   import Pleroma.Web.ApiSpec.Helpers
@@ -393,4 +394,38 @@ defmodule Pleroma.Web.ApiSpec.PleromaUtilOperation do
       }
     }
   end
+
+  def uploaded_media_operation do
+    %Operation{
+      tags: ["Media attachments"],
+      summary: "List media uploaded by the current user",
+      security: [%{"oAuth" => ["read:accounts"]}],
+      operationId: "PleromaAPI.UtilController.uploaded_media",
+      parameters: [
+        Operation.parameter(
+          :offset,
+          :query,
+          %Schema{type: :integer, default: 0},
+          "Offset for uploaded media pagination"
+        )
+      ],
+      responses: %{
+        200 =>
+          Operation.response("Success", "application/json", %Schema{
+            type: :array,
+            items: %Schema{
+              type: :object,
+              properties: %{
+                uploaded: %Schema{
+                  type: :string,
+                  description: "Date and time when the attachment was uploaded",
+                  format: :"date-time"
+                },
+                attachment: Attachment
+              }
+            }
+          })
+      }
+    }
+  end
 end

lib/pleroma/web/mastodon_api/controllers/media_controller.ex

@@ -6,6 +6,7 @@ defmodule Pleroma.Web.MastodonAPI.MediaController do
   use Pleroma.Web, :controller
 
   alias Pleroma.Object
+  alias Pleroma.Repo
   alias Pleroma.User
   alias Pleroma.Web.ActivityPub.ActivityPub
   alias Pleroma.Web.Plugs.OAuthScopesPlug
@@ -78,14 +79,67 @@ defmodule Pleroma.Web.MastodonAPI.MediaController do
     end
   end
 
+  @doc "DELETE /api/v1/media/:id"
+  def delete(
+        %{
+          assigns: %{user: user},
+          private: %{open_api_spex: %{params: %{id: id}}}
+        } = conn,
+        _
+      ) do
+    with %Object{} = object <- Object.get_by_id(id),
+         :ok <- Object.authorize_access(object, user),
+         href <- object.data["url"] |> List.first() |> Map.get("href"),
+         {:used, []} <- {:used, Pleroma.Upload.list_objects_with_attachment(href)},
+         {:ok, _, _} <- Object.delete(object) do
+      conn
+      |> put_resp_header("content-type", "text/plain")
+      |> send_resp(200, "")
+    else
+      {:used, list} ->
+        conn
+        |> put_status(:bad_request)
+        |> json(%{
+          error: "Refusing to delete attachment that is still used by #{Enum.count(list)} objects"
+        })
+    end
+  end
+
   def update(conn, data), do: show(conn, data)
 
   @doc "GET /api/v1/media/:id"
-  def show(%{assigns: %{user: user}, private: %{open_api_spex: %{params: %{id: id}}}} = conn, _) do
+  def show(
+        %{
+          assigns: %{user: user},
+          private: %{open_api_spex: %{params: %{id: id, attachment_usage: attachment_usage}}}
+        } = conn,
+        _
+      ) do
     with %Object{data: data, id: object_id} = object <- Object.get_by_id(id),
          :ok <- Object.authorize_access(object, user) do
       attachment_data = Map.put(data, "id", object_id)
 
+      attachment_data =
+        if attachment_usage do
+          href = data["url"] |> List.first() |> Map.get("href")
+          used_in_objects = Pleroma.Upload.list_objects_with_attachment(href)
+
+          attachment_duplicates =
+            Pleroma.Upload.list_duplicate_attachments(object.data)
+            |> Enum.group_by(&if &1.data["actor"] == user.ap_id, do: :own, else: :other)
+            |> Map.new(fn {key, val} -> {key, length(val)} end)
+
+          put_in(
+            attachment_data["usage"],
+            %{
+              objects: used_in_objects,
+              duplicates: attachment_duplicates
+            }
+          )
+        else
+          attachment_data
+        end
+
       render(conn, "attachment.json", %{attachment: attachment_data})
     end
   end

lib/pleroma/web/mastodon_api/views/instance_view.ex

@@ -220,7 +220,8 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do
       if Pleroma.Language.LanguageDetector.configured?() do
         "pleroma:language_detection"
       end,
-      "pleroma:block_expiration"
+      "pleroma:block_expiration",
+      "pleroma:user_media_management"
     ]
     |> Enum.filter(& &1)
   end

lib/pleroma/web/mastodon_api/views/status_view.ex

@@ -658,6 +658,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
     pleroma =
       %{mime_type: media_type}
       |> Maps.put_if_present(:name, name)
+      |> Maps.put_if_present(:usage, attachment["usage"])
 
     %{
       id: attachment_id,

lib/pleroma/web/pleroma_api/controllers/util_controller.ex

@@ -10,6 +10,7 @@ defmodule Pleroma.Web.PleromaAPI.UtilController do
   alias Pleroma.Config
   alias Pleroma.Emoji
   alias Pleroma.Healthcheck
+  alias Pleroma.Pagination
   alias Pleroma.User
   alias Pleroma.Utils.URIEncoding
   alias Pleroma.Web.ActivityPub.ActivityPub
@@ -38,7 +39,8 @@ defmodule Pleroma.Web.PleromaAPI.UtilController do
     OAuthScopesPlug,
     %{scopes: ["read:accounts"]}
     when action in [
-           :list_aliases
+           :list_aliases,
+           :uploaded_media
          ]
   )
 
@@ -274,4 +276,30 @@ defmodule Pleroma.Web.PleromaAPI.UtilController do
     |> put_status(:service_unavailable)
     |> json(info)
   end
+
+  def uploaded_media(
+        %{assigns: %{user: user}, private: %{open_api_spex: %{params: %{offset: offset}}}} = conn,
+        _
+      ) do
+    result =
+      Pleroma.Upload.list_uploaded_by(user, offset, 50)
+      |> Enum.map(fn %Pleroma.Object{
+                       id: id,
+                       data: %{"url" => [media_url], "name" => name},
+                       inserted_at: uploaded
+                     } ->
+        %{
+          uploaded: uploaded,
+          attachment: %{
+            description: name,
+            id: to_string(id),
+            url: media_url["href"],
+            pleroma: %{mime_type: media_url["mediaType"]},
+            type: "unknown"
+          }
+        }
+      end)
+
+    json(conn, result)
+  end
 end

lib/pleroma/web/router.ex

@@ -583,6 +583,8 @@ defmodule Pleroma.Web.Router do
     put("/aliases", UtilController, :add_alias)
     get("/aliases", UtilController, :list_aliases)
     delete("/aliases", UtilController, :delete_alias)
+
+    get("/uploaded_media", UtilController, :uploaded_media)
   end
 
   scope "/api/pleroma", Pleroma.Web.PleromaAPI do
@@ -803,6 +805,7 @@ defmodule Pleroma.Web.Router do
     post("/media", MediaController, :create)
     get("/media/:id", MediaController, :show)
     put("/media/:id", MediaController, :update)
+    delete("/media/:id", MediaController, :delete)
 
     get("/notifications", NotificationController, :index)
     get("/notifications/:id", NotificationController, :show)

lib/pleroma/workers/attachments_cleanup_worker.ex

@@ -29,6 +29,41 @@ defmodule Pleroma.Workers.AttachmentsCleanupWorker do
     {:ok, :success}
   end
 
+  def perform(%Job{
+        args: %{
+          "op" => "cleanup_attachments",
+          "object" =>
+            %{"data" => %{"type" => "Document", "url" => [%{"href" => href}]}} = object_data
+        }
+      }) do
+    if Pleroma.Config.get([:instance, :cleanup_attachments], false) do
+      clean_href = %URI{URI.parse(href) | query: nil} |> URI.to_string()
+
+      same_attachment_count = Pleroma.Upload.list_duplicate_attachments(object_data)
+
+      # No copies of this attachment in the database
+      if same_attachment_count == 0 do
+        attachment_post_ids = Pleroma.Upload.list_objects_with_attachment(clean_href)
+        # No posts with this attachment, either
+        if attachment_post_ids == [] do
+          base_url =
+            String.trim_trailing(
+              Pleroma.Upload.base_url(),
+              "/"
+            )
+
+          uploader = Pleroma.Config.get([Pleroma.Upload, :uploader])
+
+          clean_href
+          |> String.trim_leading("#{base_url}")
+          |> uploader.delete_file()
+        end
+      end
+    end
+
+    {:ok, :success}
+  end
+
   def perform(%Job{args: %{"op" => "cleanup_attachments", "object" => _object}}), do: {:ok, :skip}
 
   @impl true

test/pleroma/web/mastodon_api/views/instance_view_test.exs

@@ -117,7 +117,8 @@ defmodule Pleroma.Web.MastodonAPI.InstanceViewTest do
     "pleroma:get:main/ostatus",
     "pleroma:group_actors",
     "pleroma:bookmark_folders",
-    "pleroma:block_expiration"
+    "pleroma:block_expiration",
+    "pleroma:user_media_management"
   ]
 
   @configurable_features [