Insert text representation of hashtags into object["hashtags"]

Includes a new mix task: pleroma.database fill_old_hashtags

lib/mix/tasks/pleroma/database.ex

@@ -128,6 +128,49 @@ defmodule Mix.Tasks.Pleroma.Database do
     |> Stream.run()
   end
 
+  def run(["fill_old_hashtags"]) do
+    import Ecto.Query
+
+    start_pleroma()
+
+    from(
+      o in Object,
+      where: fragment("(?)->>'hashtags' is null", o.data),
+      where: fragment("(?)->>'tag' != '[]'", o.data),
+      select: %{id: o.id, tag: fragment("(?)->>'tag'", o.data)},
+      order_by: [:desc, o.id]
+    )
+    |> Pleroma.Repo.chunk_stream(200, :batches)
+    |> Stream.each(fn objects ->
+      Repo.transaction(fn ->
+        objects_first = objects |> List.first()
+        objects_last = objects |> List.last()
+
+        Logger.info(
+          "fill_old_hashtags: #{objects_first.id} (#{objects_first.inserted_at}) -- #{
+            objects_last.id
+          } (#{objects_last.inserted_at})"
+        )
+
+        objects
+        |> Enum.map(fn object ->
+          tags =
+            object.tag
+            |> Jason.decode!()
+            |> Enum.filter(&is_bitstring(&1))
+
+          Object
+          |> where([o], o.id == ^object.id)
+          |> update([o],
+            set: [data: fragment("safe_jsonb_set(?, '{hashtags}', ?, true)", o.data, ^tags)]
+          )
+          |> Repo.update_all([], timeout: :infinity)
+        end)
+      end)
+    end)
+    |> Stream.run()
+  end
+
   def run(["vacuum", args]) do
     start_pleroma()
 

lib/pleroma/activity/ir/topics.ex

@@ -48,6 +48,10 @@ defmodule Pleroma.Activity.Ir.Topics do
     tags
   end
 
+  defp hashtags_to_topics(%{data: %{"hashtags" => tags}}) do
+    Enum.map(tags, fn tag -> "hashtag:" <> tag end)
+  end
+
   defp hashtags_to_topics(%{data: %{"tag" => tags}}) do
     tags
     |> Enum.filter(&is_bitstring(&1))

lib/pleroma/constants.ex

@@ -18,7 +18,8 @@ defmodule Pleroma.Constants do
       "emoji",
       "context_id",
       "deleted_activity_id",
-      "pleroma_internal"
+      "pleroma_internal",
+      "hashtags"
     ]
   )
 

lib/pleroma/web/activity_pub/activity_pub.ex

@@ -666,7 +666,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
   defp restrict_tag_reject(query, %{tag_reject: [_ | _] = tag_reject}) do
     from(
       [_activity, object] in query,
-      where: fragment("not (?)->'tag' \\?| (?)", object.data, ^tag_reject)
+      where: fragment("not (?)->'hashtags' \\?| (?)", object.data, ^tag_reject)
     )
   end
 
@@ -679,7 +679,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
   defp restrict_tag_all(query, %{tag_all: [_ | _] = tag_all}) do
     from(
       [_activity, object] in query,
-      where: fragment("(?)->'tag' \\?& (?)", object.data, ^tag_all)
+      where: fragment("(?)->'hashtags' \\?& (?)", object.data, ^tag_all)
     )
   end
 
@@ -692,14 +692,14 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
   defp restrict_tag(query, %{tag: tag}) when is_list(tag) do
     from(
       [_activity, object] in query,
-      where: fragment("(?)->'tag' \\?| (?)", object.data, ^tag)
+      where: fragment("(?)->'hashtags' \\?| (?)", object.data, ^tag)
     )
   end
 
   defp restrict_tag(query, %{tag: tag}) when is_binary(tag) do
     from(
       [_activity, object] in query,
-      where: fragment("(?)->'tag' \\? (?)", object.data, ^tag)
+      where: fragment("(?)->'hashtags' \\? (?)", object.data, ^tag)
     )
   end
 

lib/pleroma/web/activity_pub/mrf/simple_policy.ex

@@ -74,9 +74,11 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do
 
     object =
       if MRF.subdomain_match?(media_nsfw, actor_host) do
-        tags = (child_object["tag"] || []) ++ ["nsfw"]
-        child_object = Map.put(child_object, "tag", tags)
-        child_object = Map.put(child_object, "sensitive", true)
+        child_object =
+          child_object
+          |> Map.put("hashtags", (child_object["hashtags"] || []) ++ ["nsfw"])
+          |> Map.put("sensitive", true)
+
         Map.put(object, "object", child_object)
       else
         object

lib/pleroma/web/activity_pub/transmogrifier.ex

@@ -312,16 +312,15 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
   def fix_emoji(object), do: object
 
   def fix_tag(%{"tag" => tag} = object) when is_list(tag) do
-    tags =
+    hashtags =
       tag
       |> Enum.filter(fn data -> data["type"] == "Hashtag" and data["name"] end)
-      |> Enum.map(fn %{"name" => name} ->
-        name
-        |> String.slice(1..-1)
-        |> String.downcase()
+      |> Enum.map(fn
+        %{"name" => "#" <> hashtag} -> String.downcase(hashtag)
+        %{"name" => hashtag} -> String.downcase(hashtag)
       end)
 
-    Map.put(object, "tag", tag ++ tags)
+    Map.put(object, "hashtags", hashtags)
   end
 
   def fix_tag(%{"tag" => %{} = tag} = object) do
@@ -865,7 +864,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
 
   def add_hashtags(object) do
     tags =
-      (object["tag"] || [])
+      ((object["hashtags"] || []) ++ (object["tag"] || []))
       |> Enum.map(fn
         # Expand internal representation tags into AS2 tags.
         tag when is_binary(tag) ->
@@ -936,7 +935,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
   end
 
   def set_sensitive(object) do
-    tags = object["tag"] || []
+    tags = object["hashtags"] || object["tag"] || []
     Map.put(object, "sensitive", "nsfw" in tags)
   end
 

lib/pleroma/web/common_api/utils.ex

@@ -310,7 +310,16 @@ defmodule Pleroma.Web.CommonAPI.Utils do
       "context" => draft.context,
       "attachment" => draft.attachments,
       "actor" => draft.user.ap_id,
-      "tag" => Keyword.values(draft.tags) |> Enum.uniq()
+      "tag" => Enum.filter(draft.tags, &is_map(&1)) |> Enum.uniq(),
+      "hashtags" =>
+        draft.tags
+        |> Enum.reduce([], fn
+          # Why so many formats
+          {:name, x}, acc -> if is_bitstring(x), do: [x | acc], else: acc
+          {"#" <> _, x}, acc -> if is_bitstring(x), do: [x | acc], else: acc
+          x, acc -> if is_bitstring(x), do: [x | acc], else: acc
+        end)
+        |> Enum.uniq()
     }
     |> add_in_reply_to(draft.in_reply_to)
     |> Map.merge(draft.extra)

lib/pleroma/web/mastodon_api/views/status_view.ex

@@ -347,7 +347,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
       media_attachments: attachments,
       poll: render(PollView, "show.json", object: object, for: opts[:for]),
       mentions: mentions,
-      tags: build_tags(tags),
+      tags: build_tags(object.data["hashtags"] || tags),
       application: %{
         name: "Web",
         website: nil

lib/pleroma/web/templates/feed/feed/_activity.atom.eex

@@ -22,8 +22,8 @@
     <link type="text/html" href='<%= @data["external_url"] %>' rel="alternate"/>
   <% end %>
 
-  <%= for tag <- @data["tag"] || [] do %>
-    <category term="<%= tag %>"></category>
+  <%= for hashtag <- @data["hashtags"] || [] do %>
+    <category term="<%= hashtag %>"></category>
   <% end %>
 
   <%= for attachment <- @data["attachment"] || [] do %>

lib/pleroma/web/templates/feed/feed/_activity.rss.eex

@@ -21,8 +21,8 @@
     <link><%= @data["external_url"] %></link>
   <% end %>
 
-  <%= for tag <- @data["tag"] || [] do %>
-    <category term="<%= tag %>"></category>
+  <%= for hashtag <- @data["hashtags"] || [] do %>
+    <category term="<%= hashtag %>"></category>
   <% end %>
 
   <%= for attachment <- @data["attachment"] || [] do %>

lib/pleroma/web/templates/feed/feed/_tag_activity.atom.eex

@@ -41,8 +41,8 @@
       <% end %>
     <% end %>
 
-    <%= for tag <- @data["tag"] || [] do %>
-      <category term="<%= tag %>"></category>
+    <%= for hashtag <- @data["hashtags"] || [] do %>
+      <category term="<%= hashtag %>"></category>
     <% end %>
 
     <%= for {emoji, file} <- @data["emoji"] || %{} do %>