Merge remote-tracking branch 'origin/develop' into shigusegubu

* origin/develop: (280 commits)
  Mark streaming feature for Apps in README.md
  Update README.md
  hide_followings was renamed to hide_followers in the FE, but never synced up in the BE
  tests: add a rich media card that contains all relevant fields
  test: add some regression tests for the rich media card rendering
  mastodon api: rich media: don't clobber %URI struct with a string
  adds a couple of explicit examples for ExSyslogger
  Fix if clause in activity_pub user_view
  rids the duplicate timestamp from default ExSyslogger config
  update frontend
  Allow to configure visibility for admin and moderator badges
  Add is_admin and is_moderator boolean fields to the user view
  rich media: parser: reject any data which cannot be explicitly encoded into JSON
  test: twitterapi: fix another possible test failure case
  test: twitterapi: fix the test breakage for real
  mastodon api: fix rendering of cards without image URLs (closes #597)
  Fix SQL
  ARGLE GARBLE
  html: don't attempt to parse nil content
  activitypub: transmogrifier: fix bare tags
  ...

lib/mix/tasks/pleroma/user.ex

@@ -52,6 +52,14 @@ defmodule Mix.Tasks.Pleroma.User do
   - `--locked`/`--no-locked` - whether the user's account is locked
   - `--moderator`/`--no-moderator` - whether the user is a moderator
   - `--admin`/`--no-admin` - whether the user is an admin
+
+  ## Add tags to a user.
+
+      mix pleroma.user tag NICKNAME TAGS
+
+  ## Delete tags from a user.
+
+      mix pleroma.user untag NICKNAME TAGS
   """
   def run(["new", nickname, email | rest]) do
     {options, [], []} =
@@ -249,6 +257,32 @@ defmodule Mix.Tasks.Pleroma.User do
     end
   end
 
+  def run(["tag", nickname | tags]) do
+    Common.start_pleroma()
+
+    with %User{} = user <- User.get_by_nickname(nickname) do
+      user = user |> User.tag(tags)
+
+      Mix.shell().info("Tags of #{user.nickname}: #{inspect(tags)}")
+    else
+      _ ->
+        Mix.shell().error("Could not change user tags for #{nickname}")
+    end
+  end
+
+  def run(["untag", nickname | tags]) do
+    Common.start_pleroma()
+
+    with %User{} = user <- User.get_by_nickname(nickname) do
+      user = user |> User.untag(tags)
+
+      Mix.shell().info("Tags of #{user.nickname}: #{inspect(tags)}")
+    else
+      _ ->
+        Mix.shell().error("Could not change user tags for #{nickname}")
+    end
+  end
+
   def run(["invite"]) do
     Common.start_pleroma()
 

lib/pleroma/PasswordResetToken.ex

@@ -10,7 +10,7 @@ defmodule Pleroma.PasswordResetToken do
   alias Pleroma.{User, PasswordResetToken, Repo}
 
   schema "password_reset_tokens" do
-    belongs_to(:user, User)
+    belongs_to(:user, User, type: Pleroma.FlakeId)
     field(:token, :string)
     field(:used, :boolean, default: false)
 

lib/pleroma/activity.ex

@@ -8,6 +8,7 @@ defmodule Pleroma.Activity do
   import Ecto.Query
 
   @type t :: %__MODULE__{}
+  @primary_key {:id, Pleroma.FlakeId, autogenerate: true}
 
   # https://github.com/tootsuite/mastodon/blob/master/app/models/notification.rb#L19
   @mastodon_notification_types %{
@@ -36,25 +37,11 @@ defmodule Pleroma.Activity do
     )
   end
 
-  # TODO:
-  # Go through these and fix them everywhere.
-  # Wrong name, only returns create activities
-  def all_by_object_ap_id_q(ap_id) do
-    from(
-      activity in Activity,
-      where:
-        fragment(
-          "coalesce((?)->'object'->>'id', (?)->>'object') = ?",
-          activity.data,
-          activity.data,
-          ^to_string(ap_id)
-        ),
-      where: fragment("(?)->>'type' = 'Create'", activity.data)
-    )
+  def get_by_id(id) do
+    Repo.get(Activity, id)
   end
 
-  # Wrong name, returns all.
-  def all_non_create_by_object_ap_id_q(ap_id) do
+  def by_object_ap_id(ap_id) do
     from(
       activity in Activity,
       where:
@@ -67,12 +54,7 @@ defmodule Pleroma.Activity do
     )
   end
 
-  # Wrong name plz fix thx
-  def all_by_object_ap_id(ap_id) do
-    Repo.all(all_by_object_ap_id_q(ap_id))
-  end
-
-  def create_activity_by_object_id_query(ap_ids) do
+  def create_by_object_ap_id(ap_ids) when is_list(ap_ids) do
     from(
       activity in Activity,
       where:
@@ -86,19 +68,37 @@ defmodule Pleroma.Activity do
     )
   end
 
-  def get_create_activity_by_object_ap_id(ap_id) when is_binary(ap_id) do
-    create_activity_by_object_id_query([ap_id])
+  def create_by_object_ap_id(ap_id) do
+    from(
+      activity in Activity,
+      where:
+        fragment(
+          "coalesce((?)->'object'->>'id', (?)->>'object') = ?",
+          activity.data,
+          activity.data,
+          ^to_string(ap_id)
+        ),
+      where: fragment("(?)->>'type' = 'Create'", activity.data)
+    )
+  end
+
+  def get_all_create_by_object_ap_id(ap_id) do
+    Repo.all(create_by_object_ap_id(ap_id))
+  end
+
+  def get_create_by_object_ap_id(ap_id) when is_binary(ap_id) do
+    create_by_object_ap_id(ap_id)
     |> Repo.one()
   end
 
-  def get_create_activity_by_object_ap_id(_), do: nil
+  def get_create_by_object_ap_id(_), do: nil
 
   def normalize(obj) when is_map(obj), do: Activity.get_by_ap_id(obj["id"])
   def normalize(ap_id) when is_binary(ap_id), do: Activity.get_by_ap_id(ap_id)
   def normalize(_), do: nil
 
   def get_in_reply_to_activity(%Activity{data: %{"object" => %{"inReplyTo" => ap_id}}}) do
-    get_create_activity_by_object_ap_id(ap_id)
+    get_create_by_object_ap_id(ap_id)
   end
 
   def get_in_reply_to_activity(_), do: nil

lib/pleroma/application.ex

@@ -6,11 +6,13 @@ defmodule Pleroma.Application do
   use Application
   import Supervisor.Spec
 
-  @name "Pleroma"
+  @name Mix.Project.config()[:name]
   @version Mix.Project.config()[:version]
+  @repository Mix.Project.config()[:source_url]
   def name, do: @name
   def version, do: @version
   def named_version(), do: @name <> " " <> @version
+  def repository, do: @repository
 
   def user_agent() do
     info = "#{Pleroma.Web.base_url()} <#{Pleroma.Config.get([:instance, :email], "")}>"
@@ -22,6 +24,8 @@ defmodule Pleroma.Application do
   def start(_type, _args) do
     import Cachex.Spec
 
+    Pleroma.Config.DeprecationWarnings.warn()
+
     # Define workers and child supervisors to be supervised
     children =
       [
@@ -99,11 +103,15 @@ defmodule Pleroma.Application do
           ],
           id: :cachex_idem
         ),
-        worker(Pleroma.Web.Federator.RetryQueue, []),
-        worker(Pleroma.Web.Federator, []),
-        worker(Pleroma.Stats, []),
-        worker(Pleroma.Web.Push, [])
+        worker(Pleroma.FlakeId, [])
       ] ++
+        hackney_pool_children() ++
+        [
+          worker(Pleroma.Web.Federator.RetryQueue, []),
+          worker(Pleroma.Web.Federator, []),
+          worker(Pleroma.Stats, []),
+          worker(Pleroma.Web.Push, [])
+        ] ++
         streamer_child() ++
         chat_child() ++
         [
@@ -118,6 +126,20 @@ defmodule Pleroma.Application do
     Supervisor.start_link(children, opts)
   end
 
+  def enabled_hackney_pools() do
+    [:media] ++
+      if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Hackney do
+        [:federation]
+      else
+        []
+      end ++
+      if Pleroma.Config.get([Pleroma.Uploader, :proxy_remote]) do
+        [:upload]
+      else
+        []
+      end
+  end
+
   if Mix.env() == :test do
     defp streamer_child(), do: []
     defp chat_child(), do: []
@@ -134,4 +156,11 @@ defmodule Pleroma.Application do
       end
     end
   end
+
+  defp hackney_pool_children() do
+    for pool <- enabled_hackney_pools() do
+      options = Pleroma.Config.get([:hackney_pools, pool])
+      :hackney_pool.child_spec(pool, options)
+    end
+  end
 end

lib/pleroma/clippy.ex

@@ -0,0 +1,155 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Clippy do
+  @moduledoc false
+  # No software is complete until they have a Clippy implementation.
+  # A ballmer peak _may_ be required to change this module.
+
+  def tip() do
+    tips()
+    |> Enum.random()
+    |> puts()
+  end
+
+  def tips() do
+    host = Pleroma.Config.get([Pleroma.Web.Endpoint, :url, :host])
+
+    [
+      "“πλήρωμα” is “pleroma” in greek",
+      "For an extended Pleroma Clippy Experience, use the “Redmond” themes in Pleroma FE settings",
+      "Staff accounts and MRF policies of Pleroma instances are disclosed on the NodeInfo endpoints for easy transparency!\n
+- https://catgirl.science/misc/nodeinfo.lua?#{host}
+- https://fediverse.network/#{host}/federation",
+      "Pleroma can federate to the Dark Web!\n
+- Tor: https://git.pleroma.social/pleroma/pleroma/wikis/Easy%20Onion%20Federation%20(Tor)
+- i2p: https://git.pleroma.social/pleroma/pleroma/wikis/I2p%20federation",
+      "Lists of Pleroma instances:\n\n- http://distsn.org/pleroma-instances.html\n- https://fediverse.network/pleroma\n- https://the-federation.info/pleroma",
+      "Pleroma uses the LitePub protocol - https://litepub.social",
+      "To receive more federated posts, subscribe to relays!\n
+- How-to: https://git.pleroma.social/pleroma/pleroma/wikis/Admin%20tasks#relay-managment
+- Relays: https://fediverse.network/activityrelay"
+    ]
+  end
+
+  @spec puts(String.t() | [[IO.ANSI.ansicode() | String.t(), ...], ...]) :: nil
+  def puts(text_or_lines) do
+    import IO.ANSI
+
+    lines =
+      if is_binary(text_or_lines) do
+        String.split(text_or_lines, ~r/\n/)
+      else
+        text_or_lines
+      end
+
+    longest_line_size =
+      lines
+      |> Enum.map(&charlist_count_text/1)
+      |> Enum.sort(&>=/2)
+      |> List.first()
+
+    pad_text = longest_line_size
+
+    pad =
+      for(_ <- 1..pad_text, do: "_")
+      |> Enum.join("")
+
+    pad_spaces =
+      for(_ <- 1..pad_text, do: " ")
+      |> Enum.join("")
+
+    spaces = "      "
+
+    pre_lines = [
+      "  /  \\#{spaces}  _#{pad}___",
+      "  |  |#{spaces} / #{pad_spaces}   \\"
+    ]
+
+    for l <- pre_lines do
+      IO.puts(l)
+    end
+
+    clippy_lines = [
+      "  #{bright()}@  @#{reset()}#{spaces} ",
+      "  || ||#{spaces}",
+      "  || ||   <--",
+      "  |\\_/|      ",
+      "  \\___/      "
+    ]
+
+    noclippy_line = "             "
+
+    env = %{
+      max_size: pad_text,
+      pad: pad,
+      pad_spaces: pad_spaces,
+      spaces: spaces,
+      pre_lines: pre_lines,
+      noclippy_line: noclippy_line
+    }
+
+    # surrond one/five line clippy with blank lines around to not fuck up the layout
+    #
+    # yes this fix sucks but it's good enough, have you ever seen a release of windows wihtout some butched
+    # features anyway?
+    lines =
+      if length(lines) == 1 or length(lines) == 5 do
+        [""] ++ lines ++ [""]
+      else
+        lines
+      end
+
+    clippy_line(lines, clippy_lines, env)
+  rescue
+    e ->
+      IO.puts("(Clippy crashed, sorry: #{inspect(e)})")
+      IO.puts(text_or_lines)
+  end
+
+  defp clippy_line([line | lines], [prefix | clippy_lines], env) do
+    IO.puts([prefix <> "| ", rpad_line(line, env.max_size)])
+    clippy_line(lines, clippy_lines, env)
+  end
+
+  # more text lines but clippy's complete
+  defp clippy_line([line | lines], [], env) do
+    IO.puts([env.noclippy_line, "| ", rpad_line(line, env.max_size)])
+
+    if lines == [] do
+      IO.puts(env.noclippy_line <> "\\_#{env.pad}___/")
+    end
+
+    clippy_line(lines, [], env)
+  end
+
+  # no more text lines but clippy's not complete
+  defp clippy_line([], [clippy | clippy_lines], env) do
+    if env.pad do
+      IO.puts(clippy <> "\\_#{env.pad}___/")
+      clippy_line([], clippy_lines, %{env | pad: nil})
+    else
+      IO.puts(clippy)
+      clippy_line([], clippy_lines, env)
+    end
+  end
+
+  defp clippy_line(_, _, _) do
+  end
+
+  defp rpad_line(line, max) do
+    pad = max - (charlist_count_text(line) - 2)
+    pads = Enum.join(for(_ <- 1..pad, do: " "))
+    [IO.ANSI.format(line), pads <> " |"]
+  end
+
+  defp charlist_count_text(line) do
+    if is_list(line) do
+      text = Enum.join(Enum.filter(line, &is_binary/1))
+      String.length(text)
+    else
+      String.length(line)
+    end
+  end
+end

lib/pleroma/config/deprecation_warnings.ex

@@ -0,0 +1,30 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Config.DeprecationWarnings do
+  require Logger
+
+  def check_frontend_config_mechanism() do
+    if Pleroma.Config.get(:fe) do
+      Logger.warn("""
+      !!!DEPRECATION WARNING!!!
+      You are using the old configuration mechanism for the frontend. Please check config.md.
+      """)
+    end
+  end
+
+  def check_hellthread_threshold do
+    if Pleroma.Config.get([:mrf_hellthread, :threshold]) do
+      Logger.warn("""
+      !!!DEPRECATION WARNING!!!
+      You are using the old configuration mechanism for the hellthread filter. Please check config.md.
+      """)
+    end
+  end
+
+  def warn do
+    check_frontend_config_mechanism()
+    check_hellthread_threshold()
+  end
+end

lib/pleroma/filter.ex

@@ -8,7 +8,7 @@ defmodule Pleroma.Filter do
   alias Pleroma.{User, Repo}
 
   schema "filters" do
-    belongs_to(:user, User)
+    belongs_to(:user, User, type: Pleroma.FlakeId)
     field(:filter_id, :integer)
     field(:hide, :boolean, default: false)
     field(:whole_word, :boolean, default: true)

lib/pleroma/flake_id.ex

@@ -0,0 +1,172 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.FlakeId do
+  @moduledoc """
+  Flake is a decentralized, k-ordered id generation service.
+
+  Adapted from:
+
+  * [flaky](https://github.com/nirvana/flaky), released under the terms of the Truly Free License,
+  * [Flake](https://github.com/boundary/flake), Copyright 2012, Boundary, Apache License, Version 2.0
+  """
+
+  @type t :: binary
+
+  @behaviour Ecto.Type
+  use GenServer
+  require Logger
+  alias __MODULE__
+  import Kernel, except: [to_string: 1]
+
+  defstruct node: nil, time: 0, sq: 0
+
+  @doc "Converts a binary Flake to a String"
+  def to_string(<<0::integer-size(64), id::integer-size(64)>>) do
+    Kernel.to_string(id)
+  end
+
+  def to_string(flake = <<_::integer-size(64), _::integer-size(48), _::integer-size(16)>>) do
+    encode_base62(flake)
+  end
+
+  def to_string(s), do: s
+
+  def from_string(int) when is_integer(int) do
+    from_string(Kernel.to_string(int))
+  end
+
+  for i <- [-1, 0] do
+    def from_string(unquote(i)), do: <<0::integer-size(128)>>
+    def from_string(unquote(Kernel.to_string(i))), do: <<0::integer-size(128)>>
+  end
+
+  def from_string(flake = <<_::integer-size(128)>>), do: flake
+
+  def from_string(string) when is_binary(string) and byte_size(string) < 18 do
+    case Integer.parse(string) do
+      {id, _} -> <<0::integer-size(64), id::integer-size(64)>>
+      _ -> nil
+    end
+  end
+
+  def from_string(string) do
+    string |> decode_base62 |> from_integer
+  end
+
+  def to_integer(<<integer::integer-size(128)>>), do: integer
+
+  def from_integer(integer) do
+    <<_time::integer-size(64), _node::integer-size(48), _seq::integer-size(16)>> =
+      <<integer::integer-size(128)>>
+  end
+
+  @doc "Generates a Flake"
+  @spec get :: binary
+  def get, do: to_string(:gen_server.call(:flake, :get))
+
+  # -- Ecto.Type API
+  @impl Ecto.Type
+  def type, do: :uuid
+
+  @impl Ecto.Type
+  def cast(value) do
+    {:ok, FlakeId.to_string(value)}
+  end
+
+  @impl Ecto.Type
+  def load(value) do
+    {:ok, FlakeId.to_string(value)}
+  end
+
+  @impl Ecto.Type
+  def dump(value) do
+    {:ok, FlakeId.from_string(value)}
+  end
+
+  def autogenerate(), do: get()
+
+  # -- GenServer API
+  def start_link do
+    :gen_server.start_link({:local, :flake}, __MODULE__, [], [])
+  end
+
+  @impl GenServer
+  def init([]) do
+    {:ok, %FlakeId{node: worker_id(), time: time()}}
+  end
+
+  @impl GenServer
+  def handle_call(:get, _from, state) do
+    {flake, new_state} = get(time(), state)
+    {:reply, flake, new_state}
+  end
+
+  # Matches when the calling time is the same as the state time. Incr. sq
+  defp get(time, %FlakeId{time: time, node: node, sq: seq}) do
+    new_state = %FlakeId{time: time, node: node, sq: seq + 1}
+    {gen_flake(new_state), new_state}
+  end
+
+  # Matches when the times are different, reset sq
+  defp get(newtime, %FlakeId{time: time, node: node}) when newtime > time do
+    new_state = %FlakeId{time: newtime, node: node, sq: 0}
+    {gen_flake(new_state), new_state}
+  end
+
+  # Error when clock is running backwards
+  defp get(newtime, %FlakeId{time: time}) when newtime < time do
+    {:error, :clock_running_backwards}
+  end
+
+  defp gen_flake(%FlakeId{time: time, node: node, sq: seq}) do
+    <<time::integer-size(64), node::integer-size(48), seq::integer-size(16)>>
+  end
+
+  defp nthchar_base62(n) when n <= 9, do: ?0 + n
+  defp nthchar_base62(n) when n <= 35, do: ?A + n - 10
+  defp nthchar_base62(n), do: ?a + n - 36
+
+  defp encode_base62(<<integer::integer-size(128)>>) do
+    integer
+    |> encode_base62([])
+    |> List.to_string()
+  end
+
+  defp encode_base62(int, acc) when int < 0, do: encode_base62(-int, acc)
+  defp encode_base62(int, []) when int == 0, do: '0'
+  defp encode_base62(int, acc) when int == 0, do: acc
+
+  defp encode_base62(int, acc) do
+    r = rem(int, 62)
+    id = div(int, 62)
+    acc = [nthchar_base62(r) | acc]
+    encode_base62(id, acc)
+  end
+
+  defp decode_base62(s) do
+    decode_base62(String.to_charlist(s), 0)
+  end
+
+  defp decode_base62([c | cs], acc) when c >= ?0 and c <= ?9,
+    do: decode_base62(cs, 62 * acc + (c - ?0))
+
+  defp decode_base62([c | cs], acc) when c >= ?A and c <= ?Z,
+    do: decode_base62(cs, 62 * acc + (c - ?A + 10))
+
+  defp decode_base62([c | cs], acc) when c >= ?a and c <= ?z,
+    do: decode_base62(cs, 62 * acc + (c - ?a + 36))
+
+  defp decode_base62([], acc), do: acc
+
+  defp time do
+    {mega_seconds, seconds, micro_seconds} = :erlang.timestamp()
+    1_000_000_000 * mega_seconds + seconds * 1000 + :erlang.trunc(micro_seconds / 1000)
+  end
+
+  defp worker_id() do
+    <<worker::integer-size(48)>> = :crypto.strong_rand_bytes(6)
+    worker
+  end
+end

lib/pleroma/formatter.ex

@@ -43,7 +43,7 @@ defmodule Pleroma.Formatter do
 
   def emojify(text, nil), do: text
 
-  def emojify(text, emoji) do
+  def emojify(text, emoji, strip \\ false) do
     Enum.reduce(emoji, text, fn {emoji, file}, text ->
       emoji = HTML.strip_tags(emoji)
       file = HTML.strip_tags(file)
@@ -51,14 +51,24 @@ defmodule Pleroma.Formatter do
       String.replace(
         text,
         ":#{emoji}:",
-        "<img height='32px' width='32px' alt='#{emoji}' title='#{emoji}' src='#{
-          MediaProxy.url(file)
-        }' />"
+        if not strip do
+          "<img height='32px' width='32px' alt='#{emoji}' title='#{emoji}' src='#{
+            MediaProxy.url(file)
+          }' />"
+        else
+          ""
+        end
       )
       |> HTML.filter_tags()
     end)
   end
 
+  def demojify(text) do
+    emojify(text, Emoji.get_all(), true)
+  end
+
+  def demojify(text, nil), do: text
+
   def get_emoji(text) when is_binary(text) do
     Enum.filter(Emoji.get_all(), fn {emoji, _} -> String.contains?(text, ":#{emoji}:") end)
   end
@@ -189,4 +199,16 @@ defmodule Pleroma.Formatter do
       String.replace(result_text, uuid, replacement)
     end)
   end
+
+  def truncate(text, max_length \\ 200, omission \\ "...") do
+    # Remove trailing whitespace
+    text = Regex.replace(~r/([^ \t\r\n])([ \t]+$)/u, text, "\\g{1}")
+
+    if String.length(text) < max_length do
+      text
+    else
+      length_with_omission = max_length - String.length(omission)
+      String.slice(text, 0, length_with_omission) <> omission
+    end
+  end
 end

lib/pleroma/html.ex

@@ -58,6 +58,22 @@ defmodule Pleroma.HTML do
       "#{signature}#{to_string(scrubber)}"
     end)
   end
+
+  def extract_first_external_url(_, nil), do: {:error, "No content"}
+
+  def extract_first_external_url(object, content) do
+    key = "URL|#{object.id}"
+
+    Cachex.fetch!(:scrubber_cache, key, fn _key ->
+      result =
+        content
+        |> Floki.filter_out("a.mention")
+        |> Floki.attribute("a", "href")
+        |> Enum.at(0)
+
+      {:commit, {:ok, result}}
+    end)
+  end
 end
 
 defmodule Pleroma.HTML.Scrubber.TwitterText do

lib/pleroma/http/connection.ex

@@ -10,7 +10,8 @@ defmodule Pleroma.HTTP.Connection do
   @hackney_options [
     timeout: 10000,
     recv_timeout: 20000,
-    follow_redirect: true
+    follow_redirect: true,
+    pool: :federation
   ]
   @adapter Application.get_env(:tesla, :adapter)
 

lib/pleroma/instances.ex

@@ -0,0 +1,36 @@
+defmodule Pleroma.Instances do
+  @moduledoc "Instances context."
+
+  @adapter Pleroma.Instances.Instance
+
+  defdelegate filter_reachable(urls_or_hosts), to: @adapter
+  defdelegate reachable?(url_or_host), to: @adapter
+  defdelegate set_reachable(url_or_host), to: @adapter
+  defdelegate set_unreachable(url_or_host, unreachable_since \\ nil), to: @adapter
+
+  def set_consistently_unreachable(url_or_host),
+    do: set_unreachable(url_or_host, reachability_datetime_threshold())
+
+  def reachability_datetime_threshold do
+    federation_reachability_timeout_days =
+      Pleroma.Config.get(:instance)[:federation_reachability_timeout_days] || 0
+
+    if federation_reachability_timeout_days > 0 do
+      NaiveDateTime.add(
+        NaiveDateTime.utc_now(),
+        -federation_reachability_timeout_days * 24 * 3600,
+        :second
+      )
+    else
+      ~N[0000-01-01 00:00:00]
+    end
+  end
+
+  def host(url_or_host) when is_binary(url_or_host) do
+    if url_or_host =~ ~r/^http/i do
+      URI.parse(url_or_host).host
+    else
+      url_or_host
+    end
+  end
+end

lib/pleroma/instances/instance.ex

@@ -0,0 +1,113 @@
+defmodule Pleroma.Instances.Instance do
+  @moduledoc "Instance."
+
+  alias Pleroma.Instances
+  alias Pleroma.Instances.Instance
+
+  use Ecto.Schema
+
+  import Ecto.{Query, Changeset}
+
+  alias Pleroma.Repo
+
+  schema "instances" do
+    field(:host, :string)
+    field(:unreachable_since, :naive_datetime)
+
+    timestamps()
+  end
+
+  defdelegate host(url_or_host), to: Instances
+
+  def changeset(struct, params \\ %{}) do
+    struct
+    |> cast(params, [:host, :unreachable_since])
+    |> validate_required([:host])
+    |> unique_constraint(:host)
+  end
+
+  def filter_reachable([]), do: %{}
+
+  def filter_reachable(urls_or_hosts) when is_list(urls_or_hosts) do
+    hosts =
+      urls_or_hosts
+      |> Enum.map(&(&1 && host(&1)))
+      |> Enum.filter(&(to_string(&1) != ""))
+
+    unreachable_since_by_host =
+      Repo.all(
+        from(i in Instance,
+          where: i.host in ^hosts,
+          select: {i.host, i.unreachable_since}
+        )
+      )
+      |> Map.new(& &1)
+
+    reachability_datetime_threshold = Instances.reachability_datetime_threshold()
+
+    for entry <- Enum.filter(urls_or_hosts, &is_binary/1) do
+      host = host(entry)
+      unreachable_since = unreachable_since_by_host[host]
+
+      if !unreachable_since ||
+           NaiveDateTime.compare(unreachable_since, reachability_datetime_threshold) == :gt do
+        {entry, unreachable_since}
+      end
+    end
+    |> Enum.filter(& &1)
+    |> Map.new(& &1)
+  end
+
+  def reachable?(url_or_host) when is_binary(url_or_host) do
+    !Repo.one(
+      from(i in Instance,
+        where:
+          i.host == ^host(url_or_host) and
+            i.unreachable_since <= ^Instances.reachability_datetime_threshold(),
+        select: true
+      )
+    )
+  end
+
+  def reachable?(_), do: true
+
+  def set_reachable(url_or_host) when is_binary(url_or_host) do
+    with host <- host(url_or_host),
+         %Instance{} = existing_record <- Repo.get_by(Instance, %{host: host}) do
+      {:ok, _instance} =
+        existing_record
+        |> changeset(%{unreachable_since: nil})
+        |> Repo.update()
+    end
+  end
+
+  def set_reachable(_), do: {:error, nil}
+
+  def set_unreachable(url_or_host, unreachable_since \\ nil)
+
+  def set_unreachable(url_or_host, unreachable_since) when is_binary(url_or_host) do
+    unreachable_since = unreachable_since || DateTime.utc_now()
+    host = host(url_or_host)
+    existing_record = Repo.get_by(Instance, %{host: host})
+
+    changes = %{unreachable_since: unreachable_since}
+
+    cond do
+      is_nil(existing_record) ->
+        %Instance{}
+        |> changeset(Map.put(changes, :host, host))
+        |> Repo.insert()
+
+      existing_record.unreachable_since &&
+          NaiveDateTime.compare(existing_record.unreachable_since, unreachable_since) != :gt ->
+        {:ok, existing_record}
+
+      true ->
+        existing_record
+        |> changeset(changes)
+        |> Repo.update()
+    end
+  end
+
+  def set_unreachable(_, _), do: {:error, nil}
+end

lib/pleroma/list.ex

@@ -8,7 +8,7 @@ defmodule Pleroma.List do
   alias Pleroma.{User, Repo, Activity}
 
   schema "lists" do
-    belongs_to(:user, Pleroma.User)
+    belongs_to(:user, User, type: Pleroma.FlakeId)
     field(:title, :string)
     field(:following, {:array, :string}, default: [])
 

lib/pleroma/mime.ex

@@ -102,10 +102,18 @@ defmodule Pleroma.MIME do
     "audio/ogg"
   end
 
-  defp check_mime_type(<<0x52, 0x49, 0x46, 0x46, _::binary>>) do
+  defp check_mime_type(<<"RIFF", _::binary-size(4), "WAVE", _::binary>>) do
     "audio/wav"
   end
 
+  defp check_mime_type(<<"RIFF", _::binary-size(4), "WEBP", _::binary>>) do
+    "image/webp"
+  end
+
+  defp check_mime_type(<<"RIFF", _::binary-size(4), "AVI.", _::binary>>) do
+    "video/avi"
+  end
+
   defp check_mime_type(_) do
     @default
   end

lib/pleroma/notification.ex

@@ -4,13 +4,14 @@
 
 defmodule Pleroma.Notification do
   use Ecto.Schema
-  alias Pleroma.{User, Activity, Notification, Repo, Object}
+  alias Pleroma.{User, Activity, Notification, Repo}
+  alias Pleroma.Web.CommonAPI.Utils
   import Ecto.Query
 
   schema "notifications" do
     field(:seen, :boolean, default: false)
-    belongs_to(:user, Pleroma.User)
-    belongs_to(:activity, Pleroma.Activity)
+    belongs_to(:user, User, type: Pleroma.FlakeId)
+    belongs_to(:activity, Activity, type: Pleroma.FlakeId)
 
     timestamps()
   end
@@ -34,7 +35,8 @@ defmodule Pleroma.Notification do
         n in Notification,
         where: n.user_id == ^user.id,
         order_by: [desc: n.id],
-        preload: [:activity],
+        join: activity in assoc(n, :activity),
+        preload: [activity: activity],
         limit: 20
       )
 
@@ -65,7 +67,8 @@ defmodule Pleroma.Notification do
       from(
         n in Notification,
         where: n.id == ^id,
-        preload: [:activity]
+        join: activity in assoc(n, :activity),
+        preload: [activity: activity]
       )
 
     notification = Repo.one(query)
@@ -96,7 +99,7 @@ defmodule Pleroma.Notification do
     end
   end
 
-  def create_notifications(%Activity{id: _, data: %{"to" => _, "type" => type}} = activity)
+  def create_notifications(%Activity{data: %{"to" => _, "type" => type}} = activity)
       when type in ["Create", "Like", "Announce", "Follow"] do
     users = get_notified_from_activity(activity)
 
@@ -132,54 +135,12 @@ defmodule Pleroma.Notification do
       when type in ["Create", "Like", "Announce", "Follow"] do
     recipients =
       []
-      |> maybe_notify_to_recipients(activity)
-      |> maybe_notify_mentioned_recipients(activity)
+      |> Utils.maybe_notify_to_recipients(activity)
+      |> Utils.maybe_notify_mentioned_recipients(activity)
       |> Enum.uniq()
 
     User.get_users_from_set(recipients, local_only)
   end
 
   def get_notified_from_activity(_, _local_only), do: []
-
-  defp maybe_notify_to_recipients(
-         recipients,
-         %Activity{data: %{"to" => to, "type" => _type}} = _activity
-       ) do
-    recipients ++ to
-  end
-
-  defp maybe_notify_mentioned_recipients(
-         recipients,
-         %Activity{data: %{"to" => _to, "type" => type} = data} = _activity
-       )
-       when type == "Create" do
-    object = Object.normalize(data["object"])
-
-    object_data =
-      cond do
-        !is_nil(object) ->
-          object.data
-
-        is_map(data["object"]) ->
-          data["object"]
-
-        true ->
-          %{}
-      end
-
-    tagged_mentions = maybe_extract_mentions(object_data)
-
-    recipients ++ tagged_mentions
-  end
-
-  defp maybe_notify_mentioned_recipients(recipients, _), do: recipients
-
-  defp maybe_extract_mentions(%{"tag" => tag}) do
-    tag
-    |> Enum.filter(fn x -> is_map(x) end)
-    |> Enum.filter(fn x -> x["type"] == "Mention" end)
-    |> Enum.map(fn x -> x["href"] end)
-  end
-
-  defp maybe_extract_mentions(_), do: []
 end

lib/pleroma/object.ex

@@ -31,8 +31,8 @@ defmodule Pleroma.Object do
     Repo.one(from(object in Object, where: fragment("(?)->>'id' = ?", object.data, ^ap_id)))
   end
 
-  def normalize(obj) when is_map(obj), do: Object.get_by_ap_id(obj["id"])
-  def normalize(ap_id) when is_binary(ap_id), do: Object.get_by_ap_id(ap_id)
+  def normalize(%{"id" => ap_id}), do: normalize(ap_id)
+  def normalize(ap_id) when is_binary(ap_id), do: get_cached_by_ap_id(ap_id)
   def normalize(_), do: nil
 
   # Owned objects can only be mutated by their owner
@@ -42,24 +42,18 @@ defmodule Pleroma.Object do
   # Legacy objects can be mutated by anybody
   def authorize_mutation(%Object{}, %User{}), do: true
 
-  if Mix.env() == :test do
-    def get_cached_by_ap_id(ap_id) do
-      get_by_ap_id(ap_id)
-    end
-  else
-    def get_cached_by_ap_id(ap_id) do
-      key = "object:#{ap_id}"
+  def get_cached_by_ap_id(ap_id) do
+    key = "object:#{ap_id}"
 
-      Cachex.fetch!(:object_cache, key, fn _ ->
-        object = get_by_ap_id(ap_id)
+    Cachex.fetch!(:object_cache, key, fn _ ->
+      object = get_by_ap_id(ap_id)
 
-        if object do
-          {:commit, object}
-        else
-          {:ignore, object}
-        end
-      end)
-    end
+      if object do
+        {:commit, object}
+      else
+        {:ignore, object}
+      end
+    end)
   end
 
   def context_mapping(context) do
@@ -85,9 +79,22 @@ defmodule Pleroma.Object do
 
   def delete(%Object{data: %{"id" => id}} = object) do
     with {:ok, _obj} = swap_object_with_tombstone(object),
-         Repo.delete_all(Activity.all_non_create_by_object_ap_id_q(id)),
+         Repo.delete_all(Activity.by_object_ap_id(id)),
          {:ok, true} <- Cachex.del(:object_cache, "object:#{id}") do
       {:ok, object}
     end
   end
+
+  def set_cache(%Object{data: %{"id" => ap_id}} = object) do
+    Cachex.put(:object_cache, "object:#{ap_id}", object)
+    {:ok, object}
+  end
+
+  def update_and_set_cache(changeset) do
+    with {:ok, object} <- Repo.update(changeset) do
+      set_cache(object)
+    else
+      e -> e
+    end
+  end
 end

lib/pleroma/plugs/instance_static.ex

@@ -17,11 +17,11 @@ defmodule Pleroma.Plugs.InstanceStatic do
     if File.exists?(instance_path) do
       instance_path
     else
-      Path.join(Application.app_dir(:pleroma, "priv/static/"), path)
+      Path.join(Application.app_dir(:pleroma, "priv_sid/static/"), path)
     end
   end
 
-  @only ~w(index.html static emoji packs sounds images instance favicon.png)
+  @only ~w(index.html static emoji packs sounds images instance favicon.png sw.js sw-pleroma.js)
 
   def init(opts) do
     opts

lib/pleroma/plugs/oauth_plug.ex

@@ -33,7 +33,12 @@ defmodule Pleroma.Plugs.OAuthPlug do
   #
   @spec fetch_user_and_token(String.t()) :: {:ok, User.t(), Token.t()} | nil
   defp fetch_user_and_token(token) do
-    query = from(q in Token, where: q.token == ^token, preload: [:user])
+    query =
+      from(t in Token,
+        where: t.token == ^token,
+        join: user in assoc(t, :user),
+        preload: [user: user]
+      )
 
     with %Token{user: %{info: %{deactivated: false} = _} = user} = token_record <- Repo.one(query) do
       {:ok, user, token_record}

lib/pleroma/reverse_proxy.ex

@@ -275,11 +275,24 @@ defmodule Pleroma.ReverseProxy do
 
   defp build_resp_cache_headers(headers, _opts) do
     has_cache? = Enum.any?(headers, fn {k, _} -> k in @resp_cache_headers end)
+    has_cache_control? = List.keymember?(headers, "cache-control", 0)
 
-    if has_cache? do
-      headers
-    else
-      List.keystore(headers, "cache-control", 0, {"cache-control", @default_cache_control_header})
+    cond do
+      has_cache? && has_cache_control? ->
+        headers
+
+      has_cache? ->
+        # There's caching header present but no cache-control -- we need to explicitely override it to public
+        # as Plug defaults to "max-age=0, private, must-revalidate"
+        List.keystore(headers, "cache-control", 0, {"cache-control", "public"})
+
+      true ->
+        List.keystore(
+          headers,
+          "cache-control",
+          0,
+          {"cache-control", @default_cache_control_header}
+        )
     end
   end
 

lib/pleroma/upload.ex

@@ -124,10 +124,10 @@ defmodule Pleroma.Upload do
 
           :pleroma, Pleroma.Upload, [filters: [Pleroma.Upload.Filter.Mogrify]]
 
-          :pleroma, Pleroma.Upload.Filter.Mogrify, args: "strip"
+          :pleroma, Pleroma.Upload.Filter.Mogrify, args: ["strip", "auto-orient"]
         """)
 
-        Pleroma.Config.put([Pleroma.Upload.Filter.Mogrify], args: "strip")
+        Pleroma.Config.put([Pleroma.Upload.Filter.Mogrify], args: ["strip", "auto-orient"])
         Map.put(opts, :filters, opts.filters ++ [Pleroma.Upload.Filter.Mogrify])
       else
         opts

lib/pleroma/uploaders/mdii.ex

@@ -24,7 +24,8 @@ defmodule Pleroma.Uploaders.MDII do
     extension = String.split(upload.name, ".") |> List.last()
     query = "#{cgi}?#{extension}"
 
-    with {:ok, %{status: 200, body: body}} <- @httpoison.post(query, file_data) do
+    with {:ok, %{status: 200, body: body}} <-
+           @httpoison.post(query, file_data, adapter: [pool: :default]) do
       remote_file_name = String.split(body) |> List.first()
       public_url = "#{files}/#{remote_file_name}.#{extension}"
       {:ok, {:url, public_url}}

lib/pleroma/uploaders/s3.ex

@@ -9,12 +9,20 @@ defmodule Pleroma.Uploaders.S3 do
   # The file name is re-encoded with S3's constraints here to comply with previous links with less strict filenames
   def get_file(file) do
     config = Pleroma.Config.get([__MODULE__])
+    bucket = Keyword.fetch!(config, :bucket)
+
+    bucket_with_namespace =
+      if namespace = Keyword.get(config, :bucket_namespace) do
+        namespace <> ":" <> bucket
+      else
+        bucket
+      end
 
     {:ok,
      {:url,
       Path.join([
         Keyword.fetch!(config, :public_endpoint),
-        Keyword.fetch!(config, :bucket),
+        bucket_with_namespace,
         strict_encode(URI.decode(file))
       ])}}
   end

lib/pleroma/uploaders/uploader.ex

@@ -27,18 +27,47 @@ defmodule Pleroma.Uploaders.Uploader do
     This allows to correctly proxy or redirect requests to the backend, while allowing to migrate backends without breaking any URL.
   * `{url, url :: String.t}` to bypass `get_file/2` and use the `url` directly in the activity.
   * `{:error, String.t}` error information if the file failed to be saved to the backend.
+  * `:wait_callback` will wait for an http post request at `/api/pleroma/upload_callback/:upload_path` and call the uploader's `http_callback/3` method.
 
 
   """
+  @type file_spec :: {:file | :url, String.t()}
   @callback put_file(Pleroma.Upload.t()) ::
-              :ok | {:ok, {:file | :url, String.t()}} | {:error, String.t()}
+              :ok | {:ok, file_spec()} | {:error, String.t()} | :wait_callback
+
+  @callback http_callback(Plug.Conn.t(), Map.t()) ::
+              {:ok, Plug.Conn.t()}
+              | {:ok, Plug.Conn.t(), file_spec()}
+              | {:error, Plug.Conn.t(), String.t()}
+  @optional_callbacks http_callback: 2
+
+  @spec put_file(module(), Pleroma.Upload.t()) :: {:ok, file_spec()} | {:error, String.t()}
 
-  @spec put_file(module(), Pleroma.Upload.t()) ::
-          {:ok, {:file | :url, String.t()}} | {:error, String.t()}
   def put_file(uploader, upload) do
     case uploader.put_file(upload) do
       :ok -> {:ok, {:file, upload.path}}
-      other -> other
+      :wait_callback -> handle_callback(uploader, upload)
+      {:ok, _} = ok -> ok
+      {:error, _} = error -> error
+    end
+  end
+
+  defp handle_callback(uploader, upload) do
+    :global.register_name({__MODULE__, upload.path}, self())
+
+    receive do
+      {__MODULE__, pid, conn, params} ->
+        case uploader.http_callback(conn, params) do
+          {:ok, conn, ok} ->
+            send(pid, {__MODULE__, conn})
+            {:ok, ok}
+
+          {:error, conn, error} ->
+            send(pid, {__MODULE__, conn})
+            {:error, error}
+        end
+    after
+      30_000 -> {:error, "Uploader callback timeout"}
     end
   end
 end

lib/pleroma/user.ex

@@ -17,6 +17,8 @@ defmodule Pleroma.User do
 
   @type t :: %__MODULE__{}
 
+  @primary_key {:id, Pleroma.FlakeId, autogenerate: true}
+
   @email_regex ~r/^[a-zA-Z0-9.!#$%&'*+\/=?^_`{|}~-]+@[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*$/
 
   @strict_local_nickname_regex ~r/^[a-zA-Z\d]+$/
@@ -37,6 +39,7 @@ defmodule Pleroma.User do
     field(:follower_address, :string)
     field(:search_rank, :float, virtual: true)
     field(:tags, {:array, :string}, default: [])
+    field(:bookmarks, {:array, :string}, default: [])
     field(:last_refreshed_at, :naive_datetime)
     has_many(:notifications, Notification)
     embeds_one(:info, Pleroma.User.Info)
@@ -307,20 +310,30 @@ defmodule Pleroma.User do
   @doc "A mass follow for local users. Ignores blocks and has no side effects"
   @spec follow_all(User.t(), list(User.t())) :: {atom(), User.t()}
   def follow_all(follower, followeds) do
-    following =
-      (follower.following ++ Enum.map(followeds, fn %{follower_address: fa} -> fa end))
-      |> Enum.uniq()
+    followed_addresses = Enum.map(followeds, fn %{follower_address: fa} -> fa end)
 
-    {:ok, follower} =
-      follower
-      |> follow_changeset(%{following: following})
-      |> update_and_set_cache
+    q =
+      from(u in User,
+        where: u.id == ^follower.id,
+        update: [
+          set: [
+            following:
+              fragment(
+                "array(select distinct unnest (array_cat(?, ?)))",
+                u.following,
+                ^followed_addresses
+              )
+          ]
+        ]
+      )
+
+    {1, [follower]} = Repo.update_all(q, [], returning: true)
 
     Enum.each(followeds, fn followed ->
       update_follower_count(followed)
     end)
 
-    {:ok, follower}
+    set_cache(follower)
   end
 
   def follow(%User{} = follower, %User{info: info} = followed) do
@@ -341,18 +354,17 @@ defmodule Pleroma.User do
           Websub.subscribe(follower, followed)
         end
 
-        following =
-          [ap_followers | follower.following]
-          |> Enum.uniq()
+        q =
+          from(u in User,
+            where: u.id == ^follower.id,
+            update: [push: [following: ^ap_followers]]
+          )
 
-        follower =
-          follower
-          |> follow_changeset(%{following: following})
-          |> update_and_set_cache
+        {1, [follower]} = Repo.update_all(q, [], returning: true)
 
         {:ok, _} = update_follower_count(followed)
 
-        follower
+        set_cache(follower)
     end
   end
 
@@ -360,17 +372,18 @@ defmodule Pleroma.User do
     ap_followers = followed.follower_address
 
     if following?(follower, followed) and follower.ap_id != followed.ap_id do
-      following =
-        follower.following
-        |> List.delete(ap_followers)
+      q =
+        from(u in User,
+          where: u.id == ^follower.id,
+          update: [pull: [following: ^ap_followers]]
+        )
 
-      {:ok, follower} =
-        follower
-        |> follow_changeset(%{following: following})
-        |> update_and_set_cache
+      {1, [follower]} = Repo.update_all(q, [], returning: true)
 
       {:ok, followed} = update_follower_count(followed)
 
+      set_cache(follower)
+
       {:ok, follower, Utils.fetch_latest_follow(follower, followed)}
     else
       {:error, "Not subscribed!"}
@@ -404,6 +417,10 @@ defmodule Pleroma.User do
     user.info.locked || false
   end
 
+  def get_by_id(id) do
+    Repo.get_by(User, id: id)
+  end
+
   def get_by_ap_id(ap_id) do
     Repo.get_by(User, ap_id: ap_id)
   end
@@ -417,12 +434,16 @@ defmodule Pleroma.User do
     get_by_nickname(nickname)
   end
 
+  def set_cache(user) do
+    Cachex.put(:user_cache, "ap_id:#{user.ap_id}", user)
+    Cachex.put(:user_cache, "nickname:#{user.nickname}", user)
+    Cachex.put(:user_cache, "user_info:#{user.id}", user_info(user))
+    {:ok, user}
+  end
+
   def update_and_set_cache(changeset) do
     with {:ok, user} <- Repo.update(changeset) do
-      Cachex.put(:user_cache, "ap_id:#{user.ap_id}", user)
-      Cachex.put(:user_cache, "nickname:#{user.nickname}", user)
-      Cachex.put(:user_cache, "user_info:#{user.id}", user_info(user))
-      {:ok, user}
+      set_cache(user)
     else
       e -> e
     end
@@ -439,11 +460,33 @@ defmodule Pleroma.User do
     Cachex.fetch!(:user_cache, key, fn _ -> get_by_ap_id(ap_id) end)
   end
 
+  def get_cached_by_id(id) do
+    key = "id:#{id}"
+
+    ap_id =
+      Cachex.fetch!(:user_cache, key, fn _ ->
+        user = get_by_id(id)
+
+        if user do
+          Cachex.put(:user_cache, "ap_id:#{user.ap_id}", user)
+          {:commit, user.ap_id}
+        else
+          {:ignore, ""}
+        end
+      end)
+
+    get_cached_by_ap_id(ap_id)
+  end
+
   def get_cached_by_nickname(nickname) do
     key = "nickname:#{nickname}"
     Cachex.fetch!(:user_cache, key, fn _ -> get_or_fetch_by_nickname(nickname) end)
   end
 
+  def get_cached_by_nickname_or_id(nickname_or_id) do
+    get_cached_by_id(nickname_or_id) || get_cached_by_nickname(nickname_or_id)
+  end
+
   def get_by_nickname(nickname) do
     Repo.get_by(User, nickname: nickname) ||
       if Regex.match?(~r(@#{Pleroma.Web.Endpoint.host()})i, nickname) do
@@ -901,7 +944,7 @@ defmodule Pleroma.User do
   def active_local_user_query do
     from(
       u in local_user_query(),
-      where: fragment("?->'deactivated' @> 'false'", u.info)
+      where: fragment("not (?->'deactivated' @> 'true')", u.info)
     )
   end
 
@@ -1128,6 +1171,22 @@ defmodule Pleroma.User do
     updated_user
   end
 
+  def bookmark(%User{} = user, status_id) do
+    bookmarks = Enum.uniq(user.bookmarks ++ [status_id])
+    update_bookmarks(user, bookmarks)
+  end
+
+  def unbookmark(%User{} = user, status_id) do
+    bookmarks = Enum.uniq(user.bookmarks -- [status_id])
+    update_bookmarks(user, bookmarks)
+  end
+
+  def update_bookmarks(%User{} = user, bookmarks) do
+    user
+    |> change(%{bookmarks: bookmarks})
+    |> update_and_set_cache
+  end
+
   defp normalize_tags(tags) do
     [tags]
     |> List.flatten()

lib/pleroma/user/info.ex

@@ -23,6 +23,7 @@ defmodule Pleroma.User.Info do
     field(:ap_enabled, :boolean, default: false)
     field(:is_moderator, :boolean, default: false)
     field(:is_admin, :boolean, default: false)
+    field(:show_role, :boolean, default: true)
     field(:keys, :string, default: nil)
     field(:settings, :map, default: nil)
     field(:magic_key, :string, default: nil)
@@ -30,8 +31,9 @@ defmodule Pleroma.User.Info do
     field(:topic, :string, default: nil)
     field(:hub, :string, default: nil)
     field(:salmon, :string, default: nil)
-    field(:hide_network, :boolean, default: false)
-    field(:pinned_activities, {:array, :integer}, default: [])
+    field(:hide_followers, :boolean, default: false)
+    field(:hide_follows, :boolean, default: false)
+    field(:pinned_activities, {:array, :string}, default: [])
 
     # Found in the wild
     # ap_id -> Where is this used?
@@ -143,8 +145,10 @@ defmodule Pleroma.User.Info do
       :no_rich_text,
       :default_scope,
       :banner,
-      :hide_network,
-      :background
+      :hide_follows,
+      :hide_followers,
+      :background,
+      :show_role
     ])
   end
 
@@ -194,7 +198,8 @@ defmodule Pleroma.User.Info do
     info
     |> cast(params, [
       :is_moderator,
-      :is_admin
+      :is_admin,
+      :show_role
     ])
   end
 

lib/pleroma/web/activity_pub/activity_pub.ex

@@ -3,7 +3,7 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.ActivityPub.ActivityPub do
-  alias Pleroma.{Activity, Repo, Object, Upload, User, Notification}
+  alias Pleroma.{Activity, Repo, Object, Upload, User, Notification, Instances}
   alias Pleroma.Web.ActivityPub.{Transmogrifier, MRF}
   alias Pleroma.Web.WebFinger
   alias Pleroma.Web.Federator
@@ -36,6 +36,14 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     {recipients, to, cc}
   end
 
+  defp get_recipients(%{"type" => "Create"} = data) do
+    to = data["to"] || []
+    cc = data["cc"] || []
+    actor = data["actor"] || []
+    recipients = (to ++ cc ++ [actor]) |> Enum.uniq()
+    {recipients, to, cc}
+  end
+
   defp get_recipients(data) do
     to = data["to"] || []
     cc = data["cc"] || []
@@ -56,7 +64,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     end
   end
 
-  defp check_remote_limit(%{"object" => %{"content" => content}}) do
+  defp check_remote_limit(%{"object" => %{"content" => content}}) when not is_nil(content) do
     limit = Pleroma.Config.get([:instance, :remote_limit])
     String.length(content) <= limit
   end
@@ -80,6 +88,10 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
           recipients: recipients
         })
 
+      Task.start(fn ->
+        Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity)
+      end)
+
       Notification.create_notifications(activity)
       stream_out(activity)
       {:ok, activity}
@@ -140,8 +152,9 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
              additional
            ),
          {:ok, activity} <- insert(create_data, local),
-         :ok <- maybe_federate(activity),
-         {:ok, _actor} <- User.increase_note_count(actor) do
+         # Changing note count prior to enqueuing federation task in order to avoid race conditions on updating user.info
+         {:ok, _actor} <- User.increase_note_count(actor),
+         :ok <- maybe_federate(activity) do
       {:ok, activity}
     end
   end
@@ -288,8 +301,9 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
 
     with {:ok, _} <- Object.delete(object),
          {:ok, activity} <- insert(data, local),
-         :ok <- maybe_federate(activity),
-         {:ok, _actor} <- User.decrease_note_count(user) do
+         # Changing note count prior to enqueuing federation task in order to avoid race conditions on updating user.info
+         {:ok, _actor} <- User.decrease_note_count(user),
+         :ok <- maybe_federate(activity) do
       {:ok, activity}
     end
   end
@@ -408,13 +422,42 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     |> Enum.reverse()
   end
 
+  defp restrict_since(query, %{"since_id" => ""}), do: query
+
   defp restrict_since(query, %{"since_id" => since_id}) do
     from(activity in query, where: activity.id > ^since_id)
   end
 
   defp restrict_since(query, _), do: query
 
-  defp restrict_tag(query, %{"tag" => tag}) do
+  defp restrict_tag_reject(query, %{"tag_reject" => tag_reject})
+       when is_list(tag_reject) and tag_reject != [] do
+    from(
+      activity in query,
+      where: fragment("(not (? #> '{\"object\",\"tag\"}') \\?| ?)", activity.data, ^tag_reject)
+    )
+  end
+
+  defp restrict_tag_reject(query, _), do: query
+
+  defp restrict_tag_all(query, %{"tag_all" => tag_all})
+       when is_list(tag_all) and tag_all != [] do
+    from(
+      activity in query,
+      where: fragment("(? #> '{\"object\",\"tag\"}') \\?& ?", activity.data, ^tag_all)
+    )
+  end
+
+  defp restrict_tag_all(query, _), do: query
+
+  defp restrict_tag(query, %{"tag" => tag}) when is_list(tag) do
+    from(
+      activity in query,
+      where: fragment("(? #> '{\"object\",\"tag\"}') \\?| ?", activity.data, ^tag)
+    )
+  end
+
+  defp restrict_tag(query, %{"tag" => tag}) when is_binary(tag) do
     from(
       activity in query,
       where: fragment("? <@ (? #> '{\"object\",\"tag\"}')", ^tag, activity.data)
@@ -463,6 +506,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
 
   defp restrict_local(query, _), do: query
 
+  defp restrict_max(query, %{"max_id" => ""}), do: query
+
   defp restrict_max(query, %{"max_id" => max_id}) do
     from(activity in query, where: activity.id < ^max_id)
   end
@@ -476,7 +521,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
   defp restrict_actor(query, _), do: query
 
   defp restrict_type(query, %{"type" => type}) when is_binary(type) do
-    restrict_type(query, %{"type" => [type]})
+    from(activity in query, where: fragment("?->>'type' = ?", activity.data, ^type))
   end
 
   defp restrict_type(query, %{"type" => type}) do
@@ -561,6 +606,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     base_query
     |> restrict_recipients(recipients, opts["user"])
     |> restrict_tag(opts)
+    |> restrict_tag_reject(opts)
+    |> restrict_tag_all(opts)
     |> restrict_since(opts)
     |> restrict_local(opts)
     |> restrict_limit(opts)
@@ -687,7 +734,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
   end
 
   def publish(actor, activity) do
-    followers =
+    remote_followers =
       if actor.follower_address in activity.recipients do
         {:ok, followers} = User.get_followers(actor)
         followers |> Enum.filter(&(!&1.local))
@@ -697,29 +744,31 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
 
     public = is_public?(activity)
 
-    remote_inboxes =
-      (Pleroma.Web.Salmon.remote_users(activity) ++ followers)
+    reachable_inboxes_metadata =
+      (Pleroma.Web.Salmon.remote_users(activity) ++ remote_followers)
       |> Enum.filter(fn user -> User.ap_enabled?(user) end)
       |> Enum.map(fn %{info: %{source_data: data}} ->
         (is_map(data["endpoints"]) && Map.get(data["endpoints"], "sharedInbox")) || data["inbox"]
       end)
       |> Enum.uniq()
       |> Enum.filter(fn inbox -> should_federate?(inbox, public) end)
+      |> Instances.filter_reachable()
 
     {:ok, data} = Transmogrifier.prepare_outgoing(activity.data)
     json = Jason.encode!(data)
 
-    Enum.each(remote_inboxes, fn inbox ->
+    Enum.each(reachable_inboxes_metadata, fn {inbox, unreachable_since} ->
       Federator.enqueue(:publish_single_ap, %{
         inbox: inbox,
         json: json,
         actor: actor,
-        id: activity.data["id"]
+        id: activity.data["id"],
+        unreachable_since: unreachable_since
       })
     end)
   end
 
-  def publish_one(%{inbox: inbox, json: json, actor: actor, id: id}) do
+  def publish_one(%{inbox: inbox, json: json, actor: actor, id: id} = params) do
     Logger.info("Federating #{id} to #{inbox}")
     host = URI.parse(inbox).host
 
@@ -732,15 +781,26 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
         digest: digest
       })
 
-    @httpoison.post(
-      inbox,
-      json,
-      [
-        {"Content-Type", "application/activity+json"},
-        {"signature", signature},
-        {"digest", digest}
-      ]
-    )
+    with {:ok, %{status: code}} when code in 200..299 <-
+           result =
+             @httpoison.post(
+               inbox,
+               json,
+               [
+                 {"Content-Type", "application/activity+json"},
+                 {"signature", signature},
+                 {"digest", digest}
+               ]
+             ) do
+      if !Map.has_key?(params, :unreachable_since) || params[:unreachable_since],
+        do: Instances.set_reachable(inbox)
+
+      result
+    else
+      {_post_result, response} ->
+        unless params[:unreachable_since], do: Instances.set_unreachable(inbox)
+        {:error, response}
+    end
   end
 
   # TODO:
@@ -802,11 +862,23 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
   def is_public?(%Object{data: %{"type" => "Tombstone"}}), do: false
   def is_public?(%Object{data: data}), do: is_public?(data)
   def is_public?(%Activity{data: data}), do: is_public?(data)
+  def is_public?(%{"directMessage" => true}), do: false
 
   def is_public?(data) do
     "https://www.w3.org/ns/activitystreams#Public" in (data["to"] ++ (data["cc"] || []))
   end
 
+  def is_private?(activity) do
+    !is_public?(activity) && Enum.any?(activity.data["to"], &String.contains?(&1, "/followers"))
+  end
+
+  def is_direct?(%Activity{data: %{"directMessage" => true}}), do: true
+  def is_direct?(%Object{data: %{"directMessage" => true}}), do: true
+
+  def is_direct?(activity) do
+    !is_public?(activity) && !is_private?(activity)
+  end
+
   def visible_for_user?(activity, nil) do
     is_public?(activity)
   end

lib/pleroma/web/activity_pub/activity_pub_controller.ex

@@ -4,6 +4,7 @@
 
 defmodule Pleroma.Web.ActivityPub.ActivityPubController do
   use Pleroma.Web, :controller
+
   alias Pleroma.{Activity, User, Object}
   alias Pleroma.Web.ActivityPub.{ObjectView, UserView}
   alias Pleroma.Web.ActivityPub.ActivityPub
@@ -17,6 +18,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
   action_fallback(:errors)
 
   plug(Pleroma.Web.FederatingPlug when action in [:inbox, :relay])
+  plug(:set_requester_reachable when action in [:inbox])
   plug(:relay_active? when action in [:relay])
 
   def relay_active?(conn, _) do
@@ -196,6 +198,14 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
     end
   end
 
+  def whoami(%{assigns: %{user: %User{} = user}} = conn, _params) do
+    conn
+    |> put_resp_header("content-type", "application/activity+json")
+    |> json(UserView.render("user.json", %{user: user}))
+  end
+
+  def whoami(_conn, _params), do: {:error, :not_found}
+
   def read_inbox(%{assigns: %{user: user}} = conn, %{"nickname" => nickname} = params) do
     if nickname == user.nickname do
       conn
@@ -289,4 +299,13 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
     |> put_status(500)
     |> json("error")
   end
+
+  defp set_requester_reachable(%Plug.Conn{} = conn, _) do
+    with actor <- conn.params["actor"],
+         true <- is_binary(actor) do
+      Pleroma.Instances.set_reachable(actor)
+    end
+
+    conn
+  end
 end

lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex

@@ -0,0 +1,57 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ActivityPub.MRF.AntiFollowbotPolicy do
+  alias Pleroma.User
+
+  @behaviour Pleroma.Web.ActivityPub.MRF
+
+  # XXX: this should become User.normalize_by_ap_id() or similar, really.
+  defp normalize_by_ap_id(%{"id" => id}), do: User.get_cached_by_ap_id(id)
+  defp normalize_by_ap_id(uri) when is_binary(uri), do: User.get_cached_by_ap_id(uri)
+  defp normalize_by_ap_id(_), do: nil
+
+  defp score_nickname("followbot@" <> _), do: 1.0
+  defp score_nickname("federationbot@" <> _), do: 1.0
+  defp score_nickname("federation_bot@" <> _), do: 1.0
+  defp score_nickname(_), do: 0.0
+
+  defp score_displayname("federation bot"), do: 1.0
+  defp score_displayname("federationbot"), do: 1.0
+  defp score_displayname("fedibot"), do: 1.0
+  defp score_displayname(_), do: 0.0
+
+  defp determine_if_followbot(%User{nickname: nickname, name: displayname}) do
+    nick_score =
+      nickname
+      |> String.downcase()
+      |> score_nickname()
+
+    name_score =
+      displayname
+      |> String.downcase()
+      |> score_displayname()
+
+    nick_score + name_score
+  end
+
+  defp determine_if_followbot(_), do: 0.0
+
+  @impl true
+  def filter(%{"type" => "Follow", "actor" => actor_id} = message) do
+    %User{} = actor = normalize_by_ap_id(actor_id)
+
+    score = determine_if_followbot(actor)
+
+    # TODO: scan biography data for keywords and score it somehow.
+    if score < 0.8 do
+      {:ok, message}
+    else
+      {:reject, nil}
+    end
+  end
+
+  @impl true
+  def filter(message), do: {:ok, message}
+end

lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex

@@ -3,20 +3,46 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.ActivityPub.MRF.HellthreadPolicy do
+  alias Pleroma.User
   @behaviour Pleroma.Web.ActivityPub.MRF
 
-  @impl true
-  def filter(%{"type" => "Create"} = object) do
-    threshold = Pleroma.Config.get([:mrf_hellthread, :threshold])
-    recipients = (object["to"] || []) ++ (object["cc"] || [])
+  defp delist_message(message) do
+    follower_collection = User.get_cached_by_ap_id(message["actor"]).follower_address
 
-    if length(recipients) > threshold do
-      {:reject, nil}
-    else
-      {:ok, object}
+    message
+    |> Map.put("to", [follower_collection])
+    |> Map.put("cc", ["https://www.w3.org/ns/activitystreams#Public"])
+  end
+
+  @impl true
+  def filter(%{"type" => "Create"} = message) do
+    delist_threshold = Pleroma.Config.get([:mrf_hellthread, :delist_threshold])
+
+    reject_threshold =
+      Pleroma.Config.get(
+        [:mrf_hellthread, :reject_threshold],
+        Pleroma.Config.get([:mrf_hellthread, :threshold])
+      )
+
+    recipients = (message["to"] || []) ++ (message["cc"] || [])
+
+    cond do
+      length(recipients) > reject_threshold and reject_threshold > 0 ->
+        {:reject, nil}
+
+      length(recipients) > delist_threshold and delist_threshold > 0 ->
+        if Enum.member?(message["to"], "https://www.w3.org/ns/activitystreams#Public") or
+             Enum.member?(message["cc"], "https://www.w3.org/ns/activitystreams#Public") do
+          {:ok, delist_message(message)}
+        else
+          {:ok, message}
+        end
+
+      true ->
+        {:ok, message}
     end
   end
 
   @impl true
-  def filter(object), do: {:ok, object}
+  def filter(message), do: {:ok, message}
 end

lib/pleroma/web/activity_pub/mrf/tag_policy.ex

@@ -0,0 +1,139 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ActivityPub.MRF.TagPolicy do
+  alias Pleroma.User
+  @behaviour Pleroma.Web.ActivityPub.MRF
+
+  defp get_tags(%User{tags: tags}) when is_list(tags), do: tags
+  defp get_tags(_), do: []
+
+  defp process_tag(
+         "mrf_tag:media-force-nsfw",
+         %{"type" => "Create", "object" => %{"attachment" => child_attachment} = object} = message
+       )
+       when length(child_attachment) > 0 do
+    tags = (object["tag"] || []) ++ ["nsfw"]
+
+    object =
+      object
+      |> Map.put("tags", tags)
+      |> Map.put("sensitive", true)
+
+    message = Map.put(message, "object", object)
+
+    {:ok, message}
+  end
+
+  defp process_tag(
+         "mrf_tag:media-strip",
+         %{"type" => "Create", "object" => %{"attachment" => child_attachment} = object} = message
+       )
+       when length(child_attachment) > 0 do
+    object = Map.delete(object, "attachment")
+    message = Map.put(message, "object", object)
+
+    {:ok, message}
+  end
+
+  defp process_tag(
+         "mrf_tag:force-unlisted",
+         %{"type" => "Create", "to" => to, "cc" => cc, "actor" => actor} = message
+       ) do
+    user = User.get_cached_by_ap_id(actor)
+
+    if Enum.member?(to, "https://www.w3.org/ns/activitystreams#Public") do
+      to =
+        List.delete(to, "https://www.w3.org/ns/activitystreams#Public") ++ [user.follower_address]
+
+      cc =
+        List.delete(cc, user.follower_address) ++ ["https://www.w3.org/ns/activitystreams#Public"]
+
+      object =
+        message["object"]
+        |> Map.put("to", to)
+        |> Map.put("cc", cc)
+
+      message =
+        message
+        |> Map.put("to", to)
+        |> Map.put("cc", cc)
+        |> Map.put("object", object)
+
+      {:ok, message}
+    else
+      {:ok, message}
+    end
+  end
+
+  defp process_tag(
+         "mrf_tag:sandbox",
+         %{"type" => "Create", "to" => to, "cc" => cc, "actor" => actor} = message
+       ) do
+    user = User.get_cached_by_ap_id(actor)
+
+    if Enum.member?(to, "https://www.w3.org/ns/activitystreams#Public") or
+         Enum.member?(cc, "https://www.w3.org/ns/activitystreams#Public") do
+      to =
+        List.delete(to, "https://www.w3.org/ns/activitystreams#Public") ++ [user.follower_address]
+
+      cc = List.delete(cc, "https://www.w3.org/ns/activitystreams#Public")
+
+      object =
+        message["object"]
+        |> Map.put("to", to)
+        |> Map.put("cc", cc)
+
+      message =
+        message
+        |> Map.put("to", to)
+        |> Map.put("cc", cc)
+        |> Map.put("object", object)
+
+      {:ok, message}
+    else
+      {:ok, message}
+    end
+  end
+
+  defp process_tag(
+         "mrf_tag:disable-remote-subscription",
+         %{"type" => "Follow", "actor" => actor} = message
+       ) do
+    user = User.get_cached_by_ap_id(actor)
+
+    if user.local == true do
+      {:ok, message}
+    else
+      {:reject, nil}
+    end
+  end
+
+  defp process_tag("mrf_tag:disable-any-subscription", %{"type" => "Follow"}), do: {:reject, nil}
+
+  defp process_tag(_, message), do: {:ok, message}
+
+  def filter_message(actor, message) do
+    User.get_cached_by_ap_id(actor)
+    |> get_tags()
+    |> Enum.reduce({:ok, message}, fn
+      tag, {:ok, message} ->
+        process_tag(tag, message)
+
+      _, error ->
+        error
+    end)
+  end
+
+  @impl true
+  def filter(%{"object" => target_actor, "type" => "Follow"} = message),
+    do: filter_message(target_actor, message)
+
+  @impl true
+  def filter(%{"actor" => actor, "type" => "Create"} = message),
+    do: filter_message(actor, message)
+
+  @impl true
+  def filter(message), do: {:ok, message}
+end

lib/pleroma/web/activity_pub/transmogrifier.ex

@@ -93,12 +93,47 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
     end
   end
 
-  def fix_addressing(map) do
-    map
+  def fix_explicit_addressing(%{"to" => to, "cc" => cc} = object, explicit_mentions) do
+    explicit_to =
+      to
+      |> Enum.filter(fn x -> x in explicit_mentions end)
+
+    explicit_cc =
+      to
+      |> Enum.filter(fn x -> x not in explicit_mentions end)
+
+    final_cc =
+      (cc ++ explicit_cc)
+      |> Enum.uniq()
+
+    object
+    |> Map.put("to", explicit_to)
+    |> Map.put("cc", final_cc)
+  end
+
+  def fix_explicit_addressing(object, _explicit_mentions), do: object
+
+  # if directMessage flag is set to true, leave the addressing alone
+  def fix_explicit_addressing(%{"directMessage" => true} = object), do: object
+
+  def fix_explicit_addressing(object) do
+    explicit_mentions =
+      object
+      |> Utils.determine_explicit_mentions()
+
+    explicit_mentions = explicit_mentions ++ ["https://www.w3.org/ns/activitystreams#Public"]
+
+    object
+    |> fix_explicit_addressing(explicit_mentions)
+  end
+
+  def fix_addressing(object) do
+    object
     |> fix_addressing_list("to")
     |> fix_addressing_list("cc")
     |> fix_addressing_list("bto")
     |> fix_addressing_list("bcc")
+    |> fix_explicit_addressing
   end
 
   def fix_actor(%{"attributedTo" => actor} = object) do
@@ -106,11 +141,11 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
     |> Map.put("actor", get_actor(%{"actor" => actor}))
   end
 
-  def fix_likes(%{"likes" => likes} = object)
-      when is_bitstring(likes) do
-    # Check for standardisation
-    # This is what Peertube does
-    # curl -H 'Accept: application/activity+json' $likes | jq .totalItems
+  # Check for standardisation
+  # This is what Peertube does
+  # curl -H 'Accept: application/activity+json' $likes | jq .totalItems
+  # Prismo returns only an integer (count) as "likes"
+  def fix_likes(%{"likes" => likes} = object) when not is_map(likes) do
     object
     |> Map.put("likes", [])
     |> Map.put("like_count", 0)
@@ -141,7 +176,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
     case fetch_obj_helper(in_reply_to_id) do
       {:ok, replied_object} ->
         with %Activity{} = activity <-
-               Activity.get_create_activity_by_object_ap_id(replied_object.data["id"]) do
+               Activity.get_create_by_object_ap_id(replied_object.data["id"]) do
           object
           |> Map.put("inReplyTo", replied_object.data["id"])
           |> Map.put("inReplyToAtomUri", object["inReplyToAtomUri"] || in_reply_to_id)
@@ -278,6 +313,8 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
     |> Map.put("tag", combined)
   end
 
+  def fix_tag(%{"tag" => %{} = tag} = object), do: Map.put(object, "tag", [tag])
+
   def fix_tag(object), do: object
 
   # content map usually only has one language so this will do for now.
@@ -334,7 +371,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
       Map.put(data, "actor", actor)
       |> fix_addressing
 
-    with nil <- Activity.get_create_activity_by_object_ap_id(object["id"]),
+    with nil <- Activity.get_create_by_object_ap_id(object["id"]),
          %User{} = user <- User.get_or_fetch_by_ap_id(data["actor"]) do
       object = fix_object(data["object"])
 
@@ -348,6 +385,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
         additional:
           Map.take(data, [
             "cc",
+            "directMessage",
             "id"
           ])
       }
@@ -417,9 +455,9 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
          {:ok, follow_activity} <- Utils.update_follow_state(follow_activity, "reject"),
          %User{local: true} = follower <- User.get_cached_by_ap_id(follow_activity.data["actor"]),
          {:ok, activity} <-
-           ActivityPub.accept(%{
+           ActivityPub.reject(%{
              to: follow_activity.data["to"],
-             type: "Accept",
+             type: "Reject",
              actor: followed.ap_id,
              object: follow_activity.data["id"],
              local: false
@@ -864,15 +902,10 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
 
     maybe_retire_websub(user.ap_id)
 
-    # Only do this for recent activties, don't go through the whole db.
-    # Only look at the last 1000 activities.
-    since = (Repo.aggregate(Activity, :max, :id) || 0) - 1_000
-
     q =
       from(
         a in Activity,
         where: ^old_follower_address in a.recipients,
-        where: a.id > ^since,
         update: [
           set: [
             recipients:

lib/pleroma/web/activity_pub/utils.ex

@@ -25,6 +25,20 @@ defmodule Pleroma.Web.ActivityPub.Utils do
     Map.put(params, "actor", get_ap_id(params["actor"]))
   end
 
+  def determine_explicit_mentions(%{"tag" => tag} = _object) when is_list(tag) do
+    tag
+    |> Enum.filter(fn x -> is_map(x) end)
+    |> Enum.filter(fn x -> x["type"] == "Mention" end)
+    |> Enum.map(fn x -> x["href"] end)
+  end
+
+  def determine_explicit_mentions(%{"tag" => tag} = object) when is_map(tag) do
+    Map.put(object, "tag", [tag])
+    |> determine_explicit_mentions()
+  end
+
+  def determine_explicit_mentions(_), do: []
+
   defp recipient_in_collection(ap_id, coll) when is_binary(coll), do: ap_id == coll
   defp recipient_in_collection(ap_id, coll) when is_list(coll), do: ap_id in coll
   defp recipient_in_collection(_, _), do: false
@@ -198,7 +212,7 @@ defmodule Pleroma.Web.ActivityPub.Utils do
     # Update activities that already had this. Could be done in a seperate process.
     # Alternatively, just don't do this and fetch the current object each time. Most
     # could probably be taken from cache.
-    relevant_activities = Activity.all_by_object_ap_id(id)
+    relevant_activities = Activity.get_all_create_by_object_ap_id(id)
 
     Enum.map(relevant_activities, fn activity ->
       new_activity_data = activity.data |> Map.put("object", object.data)
@@ -271,7 +285,7 @@ defmodule Pleroma.Web.ActivityPub.Utils do
            |> Map.put("#{property}_count", length(element))
            |> Map.put("#{property}s", element),
          changeset <- Changeset.change(object, data: new_data),
-         {:ok, object} <- Repo.update(changeset),
+         {:ok, object} <- Object.update_and_set_cache(changeset),
          _ <- update_object_in_activities(object) do
       {:ok, object}
     end
@@ -302,6 +316,25 @@ defmodule Pleroma.Web.ActivityPub.Utils do
   @doc """
   Updates a follow activity's state (for locked accounts).
   """
+  def update_follow_state(
+        %Activity{data: %{"actor" => actor, "object" => object, "state" => "pending"}} = activity,
+        state
+      ) do
+    try do
+      Ecto.Adapters.SQL.query!(
+        Repo,
+        "UPDATE activities SET data = jsonb_set(data, '{state}', $1) WHERE data->>'type' = 'Follow' AND data->>'actor' = $2 AND data->>'object' = $3 AND data->>'state' = 'pending'",
+        [state, actor, object]
+      )
+
+      activity = Repo.get(Activity, activity.id)
+      {:ok, activity}
+    rescue
+      e ->
+        {:error, e}
+    end
+  end
+
   def update_follow_state(%Activity{} = activity, state) do
     with new_data <-
            activity.data

lib/pleroma/web/activity_pub/views/user_view.ex

@@ -86,7 +86,7 @@ defmodule Pleroma.Web.ActivityPub.UserView do
     query = from(user in query, select: [:ap_id])
     following = Repo.all(query)
 
-    collection(following, "#{user.ap_id}/following", page, !user.info.hide_network)
+    collection(following, "#{user.ap_id}/following", page, !user.info.hide_follows)
     |> Map.merge(Utils.make_json_ld_header())
   end
 
@@ -99,7 +99,7 @@ defmodule Pleroma.Web.ActivityPub.UserView do
       "id" => "#{user.ap_id}/following",
       "type" => "OrderedCollection",
       "totalItems" => length(following),
-      "first" => collection(following, "#{user.ap_id}/following", 1, !user.info.hide_network)
+      "first" => collection(following, "#{user.ap_id}/following", 1, !user.info.hide_follows)
     }
     |> Map.merge(Utils.make_json_ld_header())
   end
@@ -109,7 +109,7 @@ defmodule Pleroma.Web.ActivityPub.UserView do
     query = from(user in query, select: [:ap_id])
     followers = Repo.all(query)
 
-    collection(followers, "#{user.ap_id}/followers", page, !user.info.hide_network)
+    collection(followers, "#{user.ap_id}/followers", page, !user.info.hide_followers)
     |> Map.merge(Utils.make_json_ld_header())
   end
 
@@ -122,7 +122,7 @@ defmodule Pleroma.Web.ActivityPub.UserView do
       "id" => "#{user.ap_id}/followers",
       "type" => "OrderedCollection",
       "totalItems" => length(followers),
-      "first" => collection(followers, "#{user.ap_id}/followers", 1, !user.info.hide_network)
+      "first" => collection(followers, "#{user.ap_id}/followers", 1, !user.info.hide_followers)
     }
     |> Map.merge(Utils.make_json_ld_header())
   end
@@ -160,7 +160,7 @@ defmodule Pleroma.Web.ActivityPub.UserView do
       "partOf" => iri,
       "totalItems" => info.note_count,
       "orderedItems" => collection,
-      "next" => "#{iri}?max_id=#{min_id - 1}"
+      "next" => "#{iri}?max_id=#{min_id}"
     }
 
     if max_qid == nil do
@@ -207,7 +207,7 @@ defmodule Pleroma.Web.ActivityPub.UserView do
       "partOf" => iri,
       "totalItems" => -1,
       "orderedItems" => collection,
-      "next" => "#{iri}?max_id=#{min_id - 1}"
+      "next" => "#{iri}?max_id=#{min_id}"
     }
 
     if max_qid == nil do
@@ -239,6 +239,8 @@ defmodule Pleroma.Web.ActivityPub.UserView do
 
     if offset < total do
       Map.put(map, "next", "#{iri}?page=#{page + 1}")
+    else
+      map
     end
   end
 end

lib/pleroma/web/common_api/common_api.ex

@@ -143,7 +143,7 @@ defmodule Pleroma.Web.CommonAPI do
           actor: user,
           context: context,
           object: object,
-          additional: %{"cc" => cc}
+          additional: %{"cc" => cc, "directMessage" => visibility == "direct"}
         })
 
       res

lib/pleroma/web/common_api/utils.ex

@@ -14,13 +14,13 @@ defmodule Pleroma.Web.CommonAPI.Utils do
 
   # This is a hack for twidere.
   def get_by_id_or_ap_id(id) do
-    activity = Repo.get(Activity, id) || Activity.get_create_activity_by_object_ap_id(id)
+    activity = Repo.get(Activity, id) || Activity.get_create_by_object_ap_id(id)
 
     activity &&
       if activity.data["type"] == "Create" do
         activity
       else
-        Activity.get_create_activity_by_object_ap_id(activity.data["object"])
+        Activity.get_create_by_object_ap_id(activity.data["object"])
       end
   end
 
@@ -261,4 +261,46 @@ defmodule Pleroma.Web.CommonAPI.Utils do
       }
     end)
   end
+
+  def maybe_notify_to_recipients(
+        recipients,
+        %Activity{data: %{"to" => to, "type" => _type}} = _activity
+      ) do
+    recipients ++ to
+  end
+
+  def maybe_notify_mentioned_recipients(
+        recipients,
+        %Activity{data: %{"to" => _to, "type" => type} = data} = _activity
+      )
+      when type == "Create" do
+    object = Object.normalize(data["object"])
+
+    object_data =
+      cond do
+        !is_nil(object) ->
+          object.data
+
+        is_map(data["object"]) ->
+          data["object"]
+
+        true ->
+          %{}
+      end
+
+    tagged_mentions = maybe_extract_mentions(object_data)
+
+    recipients ++ tagged_mentions
+  end
+
+  def maybe_notify_mentioned_recipients(recipients, _), do: recipients
+
+  def maybe_extract_mentions(%{"tag" => tag}) do
+    tag
+    |> Enum.filter(fn x -> is_map(x) end)
+    |> Enum.filter(fn x -> x["type"] == "Mention" end)
+    |> Enum.map(fn x -> x["href"] end)
+  end
+
+  def maybe_extract_mentions(_), do: []
 end

lib/pleroma/web/endpoint.ex

@@ -25,7 +25,7 @@ defmodule Pleroma.Web.Endpoint do
     at: "/",
     from: "priv_sid/static",
     only:
-      ~w(index.html static finmoji emoji packs sounds images instance sw.js favicon.png schemas doc)
+      ~w(index.html static finmoji emoji packs sounds images instance sw.js sw-pleroma.js favicon.png schemas doc)
   )
 
   # Code reloading can be explicitly enabled under the
@@ -82,4 +82,8 @@ defmodule Pleroma.Web.Endpoint do
     port = System.get_env("PORT") || raise "expected the PORT environment variable to be set"
     {:ok, Keyword.put(config, :http, [:inet6, port: port])}
   end
+
+  def websocket_url do
+    String.replace_leading(url(), "http", "ws")
+  end
 end

lib/pleroma/web/federator/federator.ex

@@ -6,7 +6,7 @@ defmodule Pleroma.Web.Federator do
   use GenServer
   alias Pleroma.User
   alias Pleroma.Activity
-  alias Pleroma.Web.{WebFinger, Websub}
+  alias Pleroma.Web.{WebFinger, Websub, Salmon}
   alias Pleroma.Web.Federator.RetryQueue
   alias Pleroma.Web.ActivityPub.ActivityPub
   alias Pleroma.Web.ActivityPub.Relay
@@ -124,6 +124,10 @@ defmodule Pleroma.Web.Federator do
     end
   end
 
+  def handle(:publish_single_salmon, params) do
+    Salmon.send_to_user(params)
+  end
+
   def handle(:publish_single_ap, params) do
     case ActivityPub.publish_one(params) do
       {:ok, _} ->

lib/pleroma/web/mastodon_api/mastodon_api_controller.ex

@@ -138,7 +138,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
       version: "#{@mastodon_api_level} (compatible; #{Pleroma.Application.named_version()})",
       email: Keyword.get(instance, :email),
       urls: %{
-        streaming_api: String.replace(Pleroma.Web.Endpoint.static_url(), "http", "ws")
+        streaming_api: Pleroma.Web.Endpoint.websocket_url()
       },
       stats: Stats.get_stats(),
       thumbnail: Web.base_url() <> "/instance/thumbnail.jpeg",
@@ -377,7 +377,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
 
   def unreblog_status(%{assigns: %{user: user}} = conn, %{"id" => ap_id_or_id}) do
     with {:ok, _unannounce, %{data: %{"id" => id}}} <- CommonAPI.unrepeat(ap_id_or_id, user),
-         %Activity{} = activity <- Activity.get_create_activity_by_object_ap_id(id) do
+         %Activity{} = activity <- Activity.get_create_by_object_ap_id(id) do
       conn
       |> put_view(StatusView)
       |> try_render("status.json", %{activity: activity, for: user, as: :activity})
@@ -386,7 +386,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
 
   def fav_status(%{assigns: %{user: user}} = conn, %{"id" => ap_id_or_id}) do
     with {:ok, _fav, %{data: %{"id" => id}}} <- CommonAPI.favorite(ap_id_or_id, user),
-         %Activity{} = activity <- Activity.get_create_activity_by_object_ap_id(id) do
+         %Activity{} = activity <- Activity.get_create_by_object_ap_id(id) do
       conn
       |> put_view(StatusView)
       |> try_render("status.json", %{activity: activity, for: user, as: :activity})
@@ -395,7 +395,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
 
   def unfav_status(%{assigns: %{user: user}} = conn, %{"id" => ap_id_or_id}) do
     with {:ok, _, _, %{data: %{"id" => id}}} <- CommonAPI.unfavorite(ap_id_or_id, user),
-         %Activity{} = activity <- Activity.get_create_activity_by_object_ap_id(id) do
+         %Activity{} = activity <- Activity.get_create_by_object_ap_id(id) do
       conn
       |> put_view(StatusView)
       |> try_render("status.json", %{activity: activity, for: user, as: :activity})
@@ -423,6 +423,28 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
     end
   end
 
+  def bookmark_status(%{assigns: %{user: user}} = conn, %{"id" => id}) do
+    with %Activity{} = activity <- Repo.get(Activity, id),
+         %User{} = user <- User.get_by_nickname(user.nickname),
+         true <- ActivityPub.visible_for_user?(activity, user),
+         {:ok, user} <- User.bookmark(user, activity.data["object"]["id"]) do
+      conn
+      |> put_view(StatusView)
+      |> try_render("status.json", %{activity: activity, for: user, as: :activity})
+    end
+  end
+
+  def unbookmark_status(%{assigns: %{user: user}} = conn, %{"id" => id}) do
+    with %Activity{} = activity <- Repo.get(Activity, id),
+         %User{} = user <- User.get_by_nickname(user.nickname),
+         true <- ActivityPub.visible_for_user?(activity, user),
+         {:ok, user} <- User.unbookmark(user, activity.data["object"]["id"]) do
+      conn
+      |> put_view(StatusView)
+      |> try_render("status.json", %{activity: activity, for: user, as: :activity})
+    end
+  end
+
   def notifications(%{assigns: %{user: user}} = conn, params) do
     notifications = Notification.for_user(user, params)
 
@@ -499,7 +521,8 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
 
   def upload(%{assigns: %{user: user}} = conn, %{"file" => file} = data) do
     with {:ok, object} <-
-           ActivityPub.upload(file,
+           ActivityPub.upload(
+             file,
              actor: User.ap_id(user),
              description: Map.get(data, "description")
            ) do
@@ -540,15 +563,34 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
   def hashtag_timeline(%{assigns: %{user: user}} = conn, params) do
     local_only = params["local"] in [true, "True", "true", "1"]
 
-    params =
+    tags =
+      [params["tag"], params["any"]]
+      |> List.flatten()
+      |> Enum.uniq()
+      |> Enum.filter(& &1)
+      |> Enum.map(&String.downcase(&1))
+
+    tag_all =
+      params["all"] ||
+        []
+        |> Enum.map(&String.downcase(&1))
+
+    tag_reject =
+      params["none"] ||
+        []
+        |> Enum.map(&String.downcase(&1))
+
+    query_params =
       params
       |> Map.put("type", "Create")
       |> Map.put("local_only", local_only)
       |> Map.put("blocking_user", user)
-      |> Map.put("tag", String.downcase(params["tag"]))
+      |> Map.put("tag", tags)
+      |> Map.put("tag_all", tag_all)
+      |> Map.put("tag_reject", tag_reject)
 
     activities =
-      ActivityPub.fetch_public_activities(params)
+      ActivityPub.fetch_public_activities(query_params)
       |> Enum.reverse()
 
     conn
@@ -563,7 +605,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
       followers =
         cond do
           for_user && user.id == for_user.id -> followers
-          user.info.hide_network -> []
+          user.info.hide_followers -> []
           true -> followers
         end
 
@@ -579,7 +621,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
       followers =
         cond do
           for_user && user.id == for_user.id -> followers
-          user.info.hide_network -> []
+          user.info.hide_follows -> []
           true -> followers
         end
 
@@ -743,8 +785,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
     fetched =
       if Regex.match?(~r/https?:/, query) do
         with {:ok, object} <- ActivityPub.fetch_object_from_id(query),
-             %Activity{} = activity <-
-               Activity.get_create_activity_by_object_ap_id(object.data["id"]),
+             %Activity{} = activity <- Activity.get_create_by_object_ap_id(object.data["id"]),
              true <- ActivityPub.visible_for_user?(activity, user) do
           [activity]
         else
@@ -840,6 +881,19 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
     |> render("index.json", %{activities: activities, for: user, as: :activity})
   end
 
+  def bookmarks(%{assigns: %{user: user}} = conn, _) do
+    user = Repo.get(User, user.id)
+
+    activities =
+      user.bookmarks
+      |> Enum.map(fn id -> Activity.get_create_by_object_ap_id(id) end)
+      |> Enum.reverse()
+
+    conn
+    |> put_view(StatusView)
+    |> render("index.json", %{activities: activities, for: user, as: :activity})
+  end
+
   def get_lists(%{assigns: %{user: user}} = conn, opts) do
     lists = Pleroma.List.for_user(user, opts)
     res = ListView.render("lists.json", lists: lists)
@@ -851,7 +905,10 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
       res = ListView.render("list.json", list: list)
       json(conn, res)
     else
-      _e -> json(conn, "error")
+      _e ->
+        conn
+        |> put_status(404)
+        |> json(%{error: "Record not found"})
     end
   end
 
@@ -1082,7 +1139,9 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
   def login(conn, _) do
     with {:ok, app} <- get_or_make_app() do
       path =
-        o_auth_path(conn, :authorize,
+        o_auth_path(
+          conn,
+          :authorize,
           response_type: "code",
           client_id: app.client_id,
           redirect_uri: ".",
@@ -1138,7 +1197,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
 
   def render_notification(user, %{id: id, activity: activity, inserted_at: created_at} = _params) do
     actor = User.get_cached_by_ap_id(activity.data["actor"])
-    parent_activity = Activity.get_create_activity_by_object_ap_id(activity.data["object"])
+    parent_activity = Activity.get_create_by_object_ap_id(activity.data["object"])
     mastodon_type = Activity.mastodon_notification_type(activity)
 
     response = %{
@@ -1290,7 +1349,8 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
                [],
                adapter: [
                  timeout: timeout,
-                 recv_timeout: timeout
+                 recv_timeout: timeout,
+                 pool: :default
                ]
              ),
            {:ok, data} <- Jason.decode(body) do
@@ -1323,6 +1383,22 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
     end
   end
 
+  def status_card(conn, %{"id" => status_id}) do
+    with %Activity{} = activity <- Repo.get(Activity, status_id),
+         true <- ActivityPub.is_public?(activity) do
+      data =
+        StatusView.render(
+          "card.json",
+          Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity)
+        )
+
+      json(conn, data)
+    else
+      _e ->
+        %{}
+    end
+  end
+
   def try_render(conn, target, params)
       when is_binary(target) do
     res = render(conn, target, params)

lib/pleroma/web/mastodon_api/views/account_view.ex

@@ -112,7 +112,9 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do
       # Pleroma extension
       pleroma: %{
         confirmation_pending: user_info.confirmation_pending,
-        tags: user.tags
+        tags: user.tags,
+        is_moderator: user.info.is_moderator,
+        is_admin: user.info.is_admin
       }
     }
   end

lib/pleroma/web/mastodon_api/views/status_view.ex

@@ -25,7 +25,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
         nil
     end)
     |> Enum.filter(& &1)
-    |> Activity.create_activity_by_object_id_query()
+    |> Activity.create_by_object_ap_id()
     |> Repo.all()
     |> Enum.reduce(%{}, fn activity, acc ->
       Map.put(acc, activity.data["object"]["id"], activity)
@@ -49,12 +49,11 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
     replied_to_activities = get_replied_to_activities(opts.activities)
 
     opts.activities
-    |> render_many(
+    |> safe_render_many(
       StatusView,
       "status.json",
       Map.put(opts, :replied_to_activities, replied_to_activities)
     )
-    |> Enum.filter(fn x -> not is_nil(x) end)
   end
 
   def render(
@@ -64,7 +63,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
     user = get_user(activity.data["actor"])
     created_at = Utils.to_masto_date(activity.data["published"])
 
-    reblogged = Activity.get_create_activity_by_object_ap_id(object)
+    reblogged = Activity.get_create_by_object_ap_id(object)
     reblogged = render("status.json", Map.put(opts, :activity, reblogged))
 
     mentions =
@@ -88,6 +87,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
       favourites_count: 0,
       reblogged: false,
       favourited: false,
+      bookmarked: false,
       muted: false,
       pinned: pinned?(activity, user),
       sensitive: false,
@@ -122,6 +122,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
 
     repeated = opts[:for] && opts[:for].ap_id in (object["announcements"] || [])
     favorited = opts[:for] && opts[:for].ap_id in (object["likes"] || [])
+    bookmarked = opts[:for] && object["id"] in opts[:for].bookmarks
 
     attachment_data = object["attachment"] || []
     attachments = render_many(attachment_data, StatusView, "attachment.json", as: :attachment)
@@ -140,6 +141,8 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
         __MODULE__
       )
 
+    card = render("card.json", Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity))
+
     %{
       id: to_string(activity.id),
       uri: object["id"],
@@ -148,6 +151,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
       in_reply_to_id: reply_to && to_string(reply_to.id),
       in_reply_to_account_id: reply_to_user && to_string(reply_to_user.id),
       reblog: nil,
+      card: card,
       content: content,
       created_at: created_at,
       reblogs_count: announcement_count,
@@ -155,6 +159,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
       favourites_count: like_count,
       reblogged: present?(repeated),
       favourited: present?(favorited),
+      bookmarked: present?(bookmarked),
       muted: false,
       pinned: pinned?(activity, user),
       sensitive: sensitive,
@@ -176,6 +181,46 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
     nil
   end
 
+  def render("card.json", %{rich_media: rich_media, page_url: page_url}) do
+    page_url_data = URI.parse(page_url)
+
+    page_url_data =
+      if rich_media[:url] != nil do
+        URI.merge(page_url_data, URI.parse(rich_media[:url]))
+      else
+        page_url_data
+      end
+
+    page_url = page_url_data |> to_string
+
+    image_url =
+      if rich_media[:image] != nil do
+        URI.merge(page_url_data, URI.parse(rich_media[:image]))
+        |> to_string
+      else
+        nil
+      end
+
+    site_name = rich_media[:site_name] || page_url_data.host
+
+    %{
+      type: "link",
+      provider_name: site_name,
+      provider_url: page_url_data.scheme <> "://" <> page_url_data.host,
+      url: page_url,
+      image: image_url |> MediaProxy.url(),
+      title: rich_media[:title],
+      description: rich_media[:description],
+      pleroma: %{
+        opengraph: rich_media
+      }
+    }
+  end
+
+  def render("card.json", _) do
+    nil
+  end
+
   def render("attachment.json", %{attachment: attachment}) do
     [attachment_url | _] = attachment["url"]
     media_type = attachment_url["mediaType"] || attachment_url["mimeType"] || "image"
@@ -209,7 +254,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
 
   def get_reply_to(%{data: %{"object" => object}}, _) do
     if object["inReplyTo"] && object["inReplyTo"] != "" do
-      Activity.get_create_activity_by_object_ap_id(object["inReplyTo"])
+      Activity.get_create_by_object_ap_id(object["inReplyTo"])
     else
       nil
     end
@@ -231,6 +276,9 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
       Enum.any?(to, &String.contains?(&1, "/followers")) ->
         "private"
 
+      length(cc) > 0 ->
+        "private"
+
       true ->
         "direct"
     end

lib/pleroma/web/metadata.ex

@@ -0,0 +1,40 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.Metadata do
+  alias Phoenix.HTML
+
+  def build_tags(params) do
+    Enum.reduce(Pleroma.Config.get([__MODULE__, :providers], []), "", fn parser, acc ->
+      rendered_html =
+        params
+        |> parser.build_tags()
+        |> Enum.map(&to_tag/1)
+        |> Enum.map(&HTML.safe_to_string/1)
+        |> Enum.join()
+
+      acc <> rendered_html
+    end)
+  end
+
+  def to_tag(data) do
+    with {name, attrs, _content = []} <- data do
+      HTML.Tag.tag(name, attrs)
+    else
+      {name, attrs, content} ->
+        HTML.Tag.content_tag(name, content, attrs)
+
+      _ ->
+        raise ArgumentError, message: "make_tag invalid args"
+    end
+  end
+
+  def activity_nsfw?(%{data: %{"sensitive" => sensitive}}) do
+    Pleroma.Config.get([__MODULE__, :unfurl_nsfw], false) == false and sensitive
+  end
+
+  def activity_nsfw?(_) do
+    false
+  end
+end

lib/pleroma/web/metadata/opengraph.ex

@@ -0,0 +1,154 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.Metadata.Providers.OpenGraph do
+  alias Pleroma.Web.Metadata.Providers.Provider
+  alias Pleroma.Web.Metadata
+  alias Pleroma.{HTML, Formatter, User}
+  alias Pleroma.Web.MediaProxy
+
+  @behaviour Provider
+
+  @impl Provider
+  def build_tags(%{
+        object: object,
+        url: url,
+        user: user
+      }) do
+    attachments = build_attachments(object)
+    scrubbed_content = scrub_html_and_truncate(object)
+    # Zero width space
+    content =
+      if scrubbed_content != "" and scrubbed_content != "\u200B" do
+        ": “" <> scrubbed_content <> "”"
+      else
+        ""
+      end
+
+    # Most previews only show og:title which is inconvenient. Instagram
+    # hacks this by putting the description in the title and making the
+    # description longer prefixed by how many likes and shares the post
+    # has. Here we use the descriptive nickname in the title, and expand
+    # the full account & nickname in the description. We also use the cute^Wevil
+    # smart quotes around the status text like Instagram, too.
+    [
+      {:meta,
+       [
+         property: "og:title",
+         content: "#{user.name}" <> content
+       ], []},
+      {:meta, [property: "og:url", content: url], []},
+      {:meta,
+       [
+         property: "og:description",
+         content: "#{user_name_string(user)}" <> content
+       ], []},
+      {:meta, [property: "og:type", content: "website"], []}
+    ] ++
+      if attachments == [] or Metadata.activity_nsfw?(object) do
+        [
+          {:meta, [property: "og:image", content: attachment_url(User.avatar_url(user))], []},
+          {:meta, [property: "og:image:width", content: 150], []},
+          {:meta, [property: "og:image:height", content: 150], []}
+        ]
+      else
+        attachments
+      end
+  end
+
+  @impl Provider
+  def build_tags(%{user: user}) do
+    with truncated_bio = scrub_html_and_truncate(user.bio || "") do
+      [
+        {:meta,
+         [
+           property: "og:title",
+           content: user_name_string(user)
+         ], []},
+        {:meta, [property: "og:url", content: User.profile_url(user)], []},
+        {:meta, [property: "og:description", content: truncated_bio], []},
+        {:meta, [property: "og:type", content: "website"], []},
+        {:meta, [property: "og:image", content: attachment_url(User.avatar_url(user))], []},
+        {:meta, [property: "og:image:width", content: 150], []},
+        {:meta, [property: "og:image:height", content: 150], []}
+      ]
+    end
+  end
+
+  defp build_attachments(%{data: %{"attachment" => attachments}}) do
+    Enum.reduce(attachments, [], fn attachment, acc ->
+      rendered_tags =
+        Enum.reduce(attachment["url"], [], fn url, acc ->
+          media_type =
+            Enum.find(["image", "audio", "video"], fn media_type ->
+              String.starts_with?(url["mediaType"], media_type)
+            end)
+
+          # TODO: Add additional properties to objects when we have the data available.
+          # Also, Whatsapp only wants JPEG or PNGs. It seems that if we add a second og:image
+          # object when a Video or GIF is attached it will display that in the Whatsapp Rich Preview.
+          case media_type do
+            "audio" ->
+              [
+                {:meta, [property: "og:" <> media_type, content: attachment_url(url["href"])], []}
+                | acc
+              ]
+
+            "image" ->
+              [
+                {:meta, [property: "og:" <> media_type, content: attachment_url(url["href"])],
+                 []},
+                {:meta, [property: "og:image:width", content: 150], []},
+                {:meta, [property: "og:image:height", content: 150], []}
+                | acc
+              ]
+
+            "video" ->
+              [
+                {:meta, [property: "og:" <> media_type, content: attachment_url(url["href"])], []}
+                | acc
+              ]
+
+            _ ->
+              acc
+          end
+        end)
+
+      acc ++ rendered_tags
+    end)
+  end
+
+  defp scrub_html_and_truncate(%{data: %{"content" => content}} = object) do
+    content
+    # html content comes from DB already encoded, decode first and scrub after
+    |> HtmlEntities.decode()
+    |> String.replace(~r/<br\s?\/?>/, " ")
+    |> HTML.get_cached_stripped_html_for_object(object, __MODULE__)
+    |> Formatter.demojify()
+    |> Formatter.truncate()
+  end
+
+  defp scrub_html_and_truncate(content) when is_binary(content) do
+    content
+    # html content comes from DB already encoded, decode first and scrub after
+    |> HtmlEntities.decode()
+    |> String.replace(~r/<br\s?\/?>/, " ")
+    |> HTML.strip_tags()
+    |> Formatter.demojify()
+    |> Formatter.truncate()
+  end
+
+  defp attachment_url(url) do
+    MediaProxy.url(url)
+  end
+
+  defp user_name_string(user) do
+    "#{user.name} " <>
+      if user.local do
+        "(@#{user.nickname}@#{Pleroma.Web.Endpoint.host()})"
+      else
+        "(@#{user.nickname})"
+      end
+  end
+end

lib/pleroma/web/metadata/provider.ex

@@ -0,0 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.Metadata.Providers.Provider do
+  @callback build_tags(map()) :: list()
+end

lib/pleroma/web/metadata/twitter_card.ex

@@ -0,0 +1,46 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.Metadata.Providers.TwitterCard do
+  alias Pleroma.Web.Metadata.Providers.Provider
+  alias Pleroma.Web.Metadata
+
+  @behaviour Provider
+
+  @impl Provider
+  def build_tags(%{object: object}) do
+    if Metadata.activity_nsfw?(object) or object.data["attachment"] == [] do
+      build_tags(nil)
+    else
+      case find_first_acceptable_media_type(object) do
+        "image" ->
+          [{:meta, [property: "twitter:card", content: "summary_large_image"], []}]
+
+        "audio" ->
+          [{:meta, [property: "twitter:card", content: "player"], []}]
+
+        "video" ->
+          [{:meta, [property: "twitter:card", content: "player"], []}]
+
+        _ ->
+          build_tags(nil)
+      end
+    end
+  end
+
+  @impl Provider
+  def build_tags(_) do
+    [{:meta, [property: "twitter:card", content: "summary"], []}]
+  end
+
+  def find_first_acceptable_media_type(%{data: %{"attachment" => attachment}}) do
+    Enum.find_value(attachment, fn attachment ->
+      Enum.find_value(attachment["url"], fn url ->
+        Enum.find(["image", "audio", "video"], fn media_type ->
+          String.starts_with?(url["mediaType"], media_type)
+        end)
+      end)
+    end)
+  end
+end

lib/pleroma/web/nodeinfo/nodeinfo_controller.ex

@@ -19,6 +19,10 @@ defmodule Pleroma.Web.Nodeinfo.NodeinfoController do
         %{
           rel: "http://nodeinfo.diaspora.software/ns/schema/2.0",
           href: Web.base_url() <> "/nodeinfo/2.0.json"
+        },
+        %{
+          rel: "http://nodeinfo.diaspora.software/ns/schema/2.1",
+          href: Web.base_url() <> "/nodeinfo/2.1.json"
         }
       ]
     }
@@ -26,8 +30,9 @@ defmodule Pleroma.Web.Nodeinfo.NodeinfoController do
     json(conn, response)
   end
 
-  # Schema definition: https://github.com/jhass/nodeinfo/blob/master/schemas/2.0/schema.json
-  def nodeinfo(conn, %{"version" => "2.0"}) do
+  # returns a nodeinfo 2.0 map, since 2.1 just adds a repository field
+  # under software.
+  def raw_nodeinfo() do
     instance = Application.get_env(:pleroma, :instance)
     media_proxy = Application.get_env(:pleroma, :media_proxy)
     suggestions = Application.get_env(:pleroma, :suggestions)
@@ -98,10 +103,10 @@ defmodule Pleroma.Web.Nodeinfo.NodeinfoController do
       ]
       |> Enum.filter(& &1)
 
-    response = %{
+    %{
       version: "2.0",
       software: %{
-        name: Pleroma.Application.name(),
+        name: Pleroma.Application.name() |> String.downcase(),
         version: Pleroma.Application.version()
       },
       protocols: ["ostatus", "activitypub"],
@@ -142,12 +147,37 @@ defmodule Pleroma.Web.Nodeinfo.NodeinfoController do
         restrictedNicknames: Pleroma.Config.get([Pleroma.User, :restricted_nicknames])
       }
     }
+  end
 
+  # Schema definition: https://github.com/jhass/nodeinfo/blob/master/schemas/2.0/schema.json
+  # and https://github.com/jhass/nodeinfo/blob/master/schemas/2.1/schema.json
+  def nodeinfo(conn, %{"version" => "2.0"}) do
     conn
     |> put_resp_header(
       "content-type",
       "application/json; profile=http://nodeinfo.diaspora.software/ns/schema/2.0#; charset=utf-8"
     )
+    |> json(raw_nodeinfo())
+  end
+
+  def nodeinfo(conn, %{"version" => "2.1"}) do
+    raw_response = raw_nodeinfo()
+
+    updated_software =
+      raw_response
+      |> Map.get(:software)
+      |> Map.put(:repository, Pleroma.Application.repository())
+
+    response =
+      raw_response
+      |> Map.put(:software, updated_software)
+      |> Map.put(:version, "2.1")
+
+    conn
+    |> put_resp_header(
+      "content-type",
+      "application/json; profile=http://nodeinfo.diaspora.software/ns/schema/2.1#; charset=utf-8"
+    )
     |> json(response)
   end
 

lib/pleroma/web/oauth/authorization.ex

@@ -14,7 +14,7 @@ defmodule Pleroma.Web.OAuth.Authorization do
     field(:token, :string)
     field(:valid_until, :naive_datetime)
     field(:used, :boolean, default: false)
-    belongs_to(:user, Pleroma.User)
+    belongs_to(:user, Pleroma.User, type: Pleroma.FlakeId)
     belongs_to(:app, App)
 
     timestamps()

lib/pleroma/web/oauth/fallback_controller.ex

@@ -9,7 +9,8 @@ defmodule Pleroma.Web.OAuth.FallbackController do
   # No user/password
   def call(conn, _) do
     conn
+    |> put_status(:unauthorized)
     |> put_flash(:error, "Invalid Username/Password")
-    |> OAuthController.authorize(conn.params)
+    |> OAuthController.authorize(conn.params["authorization"])
   end
 end

lib/pleroma/web/oauth/token.ex

@@ -14,7 +14,7 @@ defmodule Pleroma.Web.OAuth.Token do
     field(:token, :string)
     field(:refresh_token, :string)
     field(:valid_until, :naive_datetime)
-    belongs_to(:user, Pleroma.User)
+    belongs_to(:user, Pleroma.User, type: Pleroma.FlakeId)
     belongs_to(:app, App)
 
     timestamps()

lib/pleroma/web/ostatus/activity_representer.ex

@@ -183,7 +183,7 @@ defmodule Pleroma.Web.OStatus.ActivityRepresenter do
     _in_reply_to = get_in_reply_to(activity.data)
     author = if with_author, do: [{:author, UserRepresenter.to_simple_form(user)}], else: []
 
-    retweeted_activity = Activity.get_create_activity_by_object_ap_id(activity.data["object"])
+    retweeted_activity = Activity.get_create_by_object_ap_id(activity.data["object"])
     retweeted_user = User.get_cached_by_ap_id(retweeted_activity.data["actor"])
 
     retweeted_xml = to_simple_form(retweeted_activity, retweeted_user, true)

lib/pleroma/web/ostatus/handlers/note_handler.ex

@@ -86,7 +86,7 @@ defmodule Pleroma.Web.OStatus.NoteHandler do
   end
 
   def fetch_replied_to_activity(entry, inReplyTo) do
-    with %Activity{} = activity <- Activity.get_create_activity_by_object_ap_id(inReplyTo) do
+    with %Activity{} = activity <- Activity.get_create_by_object_ap_id(inReplyTo) do
       activity
     else
       _e ->
@@ -103,7 +103,7 @@ defmodule Pleroma.Web.OStatus.NoteHandler do
   # TODO: Clean this up a bit.
   def handle_note(entry, doc \\ nil) do
     with id <- XML.string_from_xpath("//id", entry),
-         activity when is_nil(activity) <- Activity.get_create_activity_by_object_ap_id(id),
+         activity when is_nil(activity) <- Activity.get_create_by_object_ap_id(id),
          [author] <- :xmerl_xpath.string('//author[1]', doc),
          {:ok, actor} <- OStatus.find_make_or_update_user(author),
          content_html <- OStatus.get_content(entry),

lib/pleroma/web/ostatus/ostatus.ex

@@ -48,6 +48,9 @@ defmodule Pleroma.Web.OStatus do
 
   def handle_incoming(xml_string) do
     with doc when doc != :error <- parse_document(xml_string) do
+      with {:ok, actor_user} <- find_make_or_update_user(doc),
+           do: Pleroma.Instances.set_reachable(actor_user.ap_id)
+
       entries = :xmerl_xpath.string('//entry', doc)
 
       activities =
@@ -148,7 +151,7 @@ defmodule Pleroma.Web.OStatus do
     Logger.debug("Trying to get entry from db")
 
     with id when not is_nil(id) <- string_from_xpath("//activity:object[1]/id", entry),
-         %Activity{} = activity <- Activity.get_create_activity_by_object_ap_id(id) do
+         %Activity{} = activity <- Activity.get_create_by_object_ap_id(id) do
       {:ok, activity}
     else
       _ ->

lib/pleroma/web/ostatus/ostatus_controller.ex

@@ -7,7 +7,6 @@ defmodule Pleroma.Web.OStatus.OStatusController do
 
   alias Pleroma.{User, Activity, Object}
   alias Pleroma.Web.OStatus.{FeedRepresenter, ActivityRepresenter}
-  alias Pleroma.Repo
   alias Pleroma.Web.{OStatus, Federator}
   alias Pleroma.Web.XML
   alias Pleroma.Web.ActivityPub.ObjectView
@@ -15,12 +14,17 @@ defmodule Pleroma.Web.OStatus.OStatusController do
   alias Pleroma.Web.ActivityPub.ActivityPub
 
   plug(Pleroma.Web.FederatingPlug when action in [:salmon_incoming])
+
   action_fallback(:errors)
 
   def feed_redirect(conn, %{"nickname" => nickname}) do
     case get_format(conn) do
       "html" ->
-        Fallback.RedirectController.redirector(conn, nil)
+        with %User{} = user <- User.get_cached_by_nickname_or_id(nickname) do
+          Fallback.RedirectController.redirector_with_meta(conn, %{user: user})
+        else
+          nil -> {:error, :not_found}
+        end
 
       "activity+json" ->
         ActivityPubController.call(conn, :user)
@@ -90,8 +94,7 @@ defmodule Pleroma.Web.OStatus.OStatusController do
       ActivityPubController.call(conn, :object)
     else
       with id <- o_status_url(conn, :object, uuid),
-           {_, %Activity{} = activity} <-
-             {:activity, Activity.get_create_activity_by_object_ap_id(id)},
+           {_, %Activity{} = activity} <- {:activity, Activity.get_create_by_object_ap_id(id)},
            {_, true} <- {:public?, ActivityPub.is_public?(activity)},
            %User{} = user <- User.get_cached_by_ap_id(activity.data["actor"]) do
         case get_format(conn) do
@@ -137,24 +140,40 @@ defmodule Pleroma.Web.OStatus.OStatusController do
   end
 
   def notice(conn, %{"id" => id}) do
-    with {_, %Activity{} = activity} <- {:activity, Repo.get(Activity, id)},
+    with {_, %Activity{} = activity} <- {:activity, Activity.get_by_id(id)},
          {_, true} <- {:public?, ActivityPub.is_public?(activity)},
          %User{} = user <- User.get_cached_by_ap_id(activity.data["actor"]) do
       case format = get_format(conn) do
         "html" ->
-          conn
-          |> put_resp_content_type("text/html")
-          |> send_file(200, Application.app_dir(:pleroma, "priv_sid/static/index.html"))
+          if activity.data["type"] == "Create" do
+            %Object{} = object = Object.normalize(activity.data["object"])
+
+            Fallback.RedirectController.redirector_with_meta(conn, %{
+              object: object,
+              url:
+                Pleroma.Web.Router.Helpers.o_status_url(
+                  Pleroma.Web.Endpoint,
+                  :notice,
+                  activity.id
+                ),
+              user: user
+            })
+          else
+            Fallback.RedirectController.redirector(conn, nil)
+          end
 
         _ ->
           represent_activity(conn, format, activity, user)
       end
     else
       {:public?, false} ->
-        {:error, :not_found}
+        conn
+        |> put_status(404)
+        |> Fallback.RedirectController.redirector(nil, 404)
 
       {:activity, nil} ->
-        {:error, :not_found}
+        conn
+        |> Fallback.RedirectController.redirector(nil, 404)
 
       e ->
         e

lib/pleroma/web/push/subscription.ex

@@ -10,7 +10,7 @@ defmodule Pleroma.Web.Push.Subscription do
   alias Pleroma.Web.Push.Subscription
 
   schema "push_subscriptions" do
-    belongs_to(:user, User)
+    belongs_to(:user, User, type: Pleroma.FlakeId)
     belongs_to(:token, Token)
     field(:endpoint, :string)
     field(:key_p256dh, :string)

lib/pleroma/web/rich_media/controllers/rich_media_controller.ex

@@ -1,17 +0,0 @@
-defmodule Pleroma.Web.RichMedia.RichMediaController do
-  use Pleroma.Web, :controller
-
-  import Pleroma.Web.ControllerHelper, only: [json_response: 3]
-
-  def parse(conn, %{"url" => url}) do
-    case Pleroma.Web.RichMedia.Parser.parse(url) do
-      {:ok, data} ->
-        conn
-        |> json_response(200, data)
-
-      {:error, msg} ->
-        conn
-        |> json_response(404, msg)
-    end
-  end
-end

lib/pleroma/web/rich_media/helpers.ex

@@ -0,0 +1,19 @@
+# Pleroma: A lightweight social networking server
+# Copyright _ 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.RichMedia.Helpers do
+  alias Pleroma.{Activity, Object, HTML}
+  alias Pleroma.Web.RichMedia.Parser
+
+  def fetch_data_for_activity(%Activity{} = activity) do
+    with true <- Pleroma.Config.get([:rich_media, :enabled]),
+         %Object{} = object <- Object.normalize(activity.data["object"]),
+         {:ok, page_url} <- HTML.extract_first_external_url(object, object.data["content"]),
+         {:ok, rich_media} <- Parser.parse(page_url) do
+      %{page_url: page_url, rich_media: rich_media}
+    else
+      _ -> %{}
+    end
+  end
+end

lib/pleroma/web/rich_media/parser.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.Web.RichMedia.Parser do
   @parsers [
     Pleroma.Web.RichMedia.Parsers.OGP,
@@ -5,17 +9,32 @@ defmodule Pleroma.Web.RichMedia.Parser do
     Pleroma.Web.RichMedia.Parsers.OEmbed
   ]
 
+  def parse(nil), do: {:error, "No URL provided"}
+
   if Mix.env() == :test do
     def parse(url), do: parse_url(url)
   else
-    def parse(url),
-      do: Cachex.fetch!(:rich_media_cache, url, fn _ -> parse_url(url) end)
+    def parse(url) do
+      try do
+        Cachex.fetch!(:rich_media_cache, url, fn _ ->
+          {:commit, parse_url(url)}
+        end)
+      rescue
+        e ->
+          {:error, "Cachex error: #{inspect(e)}"}
+      end
+    end
   end
 
   defp parse_url(url) do
-    {:ok, %Tesla.Env{body: html}} = Pleroma.HTTP.get(url)
+    try do
+      {:ok, %Tesla.Env{body: html}} = Pleroma.HTTP.get(url, [], adapter: [pool: :media])
 
-    html |> maybe_parse() |> get_parsed_data()
+      html |> maybe_parse() |> clean_parsed_data() |> check_parsed_data()
+    rescue
+      e ->
+        {:error, "Parsing error: #{inspect(e)}"}
+    end
   end
 
   defp maybe_parse(html) do
@@ -27,11 +46,23 @@ defmodule Pleroma.Web.RichMedia.Parser do
     end)
   end
 
-  defp get_parsed_data(data) when data == %{} do
-    {:error, "No metadata found"}
-  end
-
-  defp get_parsed_data(data) do
+  defp check_parsed_data(%{title: title} = data) when is_binary(title) and byte_size(title) > 0 do
     {:ok, data}
   end
+
+  defp check_parsed_data(data) do
+    {:error, "Found metadata was invalid or incomplete: #{inspect(data)}"}
+  end
+
+  defp clean_parsed_data(data) do
+    data
+    |> Enum.reject(fn {key, val} ->
+      with {:ok, _} <- Jason.encode(%{key => val}) do
+        false
+      else
+        _ -> true
+      end
+    end)
+    |> Map.new()
+  end
 end

lib/pleroma/web/rich_media/parsers/oembed_parser.ex

@@ -20,8 +20,12 @@ defmodule Pleroma.Web.RichMedia.Parsers.OEmbed do
   end
 
   defp get_oembed_data(url) do
-    {:ok, %Tesla.Env{body: json}} = Pleroma.HTTP.get(url)
+    {:ok, %Tesla.Env{body: json}} = Pleroma.HTTP.get(url, [], adapter: [pool: :media])
 
-    {:ok, Poison.decode!(json)}
+    {:ok, data} = Jason.decode(json)
+
+    data = data |> Map.new(fn {k, v} -> {String.to_atom(k), v} end)
+
+    {:ok, data}
   end
 end

lib/pleroma/web/router.ex

@@ -107,6 +107,11 @@ defmodule Pleroma.Web.Router do
     get("/captcha", UtilController, :captcha)
   end
 
+  scope "/api/pleroma", Pleroma.Web do
+    pipe_through(:pleroma_api)
+    post("/uploader_callback/:upload_path", UploaderController, :callback)
+  end
+
   scope "/api/pleroma/admin", Pleroma.Web.AdminAPI do
     pipe_through(:admin_api)
     delete("/user", AdminAPIController, :user_delete)
@@ -180,6 +185,7 @@ defmodule Pleroma.Web.Router do
     get("/timelines/direct", MastodonAPIController, :dm_timeline)
 
     get("/favourites", MastodonAPIController, :favourites)
+    get("/bookmarks", MastodonAPIController, :bookmarks)
 
     post("/statuses", MastodonAPIController, :post_status)
     delete("/statuses/:id", MastodonAPIController, :delete_status)
@@ -190,6 +196,8 @@ defmodule Pleroma.Web.Router do
     post("/statuses/:id/unfavourite", MastodonAPIController, :unfav_status)
     post("/statuses/:id/pin", MastodonAPIController, :pin_status)
     post("/statuses/:id/unpin", MastodonAPIController, :unpin_status)
+    post("/statuses/:id/bookmark", MastodonAPIController, :bookmark_status)
+    post("/statuses/:id/unbookmark", MastodonAPIController, :unbookmark_status)
 
     post("/notifications/clear", MastodonAPIController, :clear_notifications)
     post("/notifications/dismiss", MastodonAPIController, :dismiss_notification)
@@ -234,12 +242,6 @@ defmodule Pleroma.Web.Router do
     put("/settings", MastodonAPIController, :put_settings)
   end
 
-  scope "/api", Pleroma.Web.RichMedia do
-    pipe_through(:authenticated_api)
-
-    get("/rich_media/parse", RichMediaController, :parse)
-  end
-
   scope "/api/v1", Pleroma.Web.MastodonAPI do
     pipe_through(:api)
     get("/instance", MastodonAPIController, :masto_instance)
@@ -253,7 +255,7 @@ defmodule Pleroma.Web.Router do
 
     get("/statuses/:id", MastodonAPIController, :get_status)
     get("/statuses/:id/context", MastodonAPIController, :get_context)
-    get("/statuses/:id/card", MastodonAPIController, :empty_object)
+    get("/statuses/:id/card", MastodonAPIController, :status_card)
     get("/statuses/:id/favourited_by", MastodonAPIController, :favourited_by)
     get("/statuses/:id/reblogged_by", MastodonAPIController, :reblogged_by)
 
@@ -279,6 +281,7 @@ defmodule Pleroma.Web.Router do
     post("/help/test", TwitterAPI.UtilController, :help_test)
     get("/statusnet/config", TwitterAPI.UtilController, :config)
     get("/statusnet/version", TwitterAPI.UtilController, :version)
+    get("/pleroma/frontend_configurations", TwitterAPI.UtilController, :frontend_configurations)
   end
 
   scope "/api", Pleroma.Web do
@@ -391,7 +394,11 @@ defmodule Pleroma.Web.Router do
   end
 
   pipeline :ostatus do
-    plug(:accepts, ["xml", "atom", "html", "activity+json"])
+    plug(:accepts, ["html", "xml", "atom", "activity+json"])
+  end
+
+  pipeline :oembed do
+    plug(:accepts, ["json", "xml"])
   end
 
   scope "/", Pleroma.Web do
@@ -409,6 +416,12 @@ defmodule Pleroma.Web.Router do
     post("/push/subscriptions/:id", Websub.WebsubController, :websub_incoming)
   end
 
+  scope "/", Pleroma.Web do
+    pipe_through(:oembed)
+
+    get("/oembed", OEmbed.OEmbedController, :url)
+  end
+
   pipeline :activitypub do
     plug(:accepts, ["activity+json"])
     plug(Pleroma.Web.Plugs.HTTPSignaturePlug)
@@ -441,6 +454,7 @@ defmodule Pleroma.Web.Router do
   scope "/", Pleroma.Web.ActivityPub do
     pipe_through([:activitypub_client])
 
+    get("/api/ap/whoami", ActivityPubController, :whoami)
     get("/users/:nickname/inbox", ActivityPubController, :read_inbox)
     post("/users/:nickname/outbox", ActivityPubController, :update_outbox)
   end
@@ -496,6 +510,7 @@ defmodule Pleroma.Web.Router do
 
   scope "/", Fallback do
     get("/registration/:token", RedirectController, :registration_page)
+    get("/:maybe_nickname_or_id", RedirectController, :redirector_with_meta)
     get("/*path", RedirectController, :redirector)
 
     options("/*path", RedirectController, :empty)
@@ -504,11 +519,36 @@ end
 
 defmodule Fallback.RedirectController do
   use Pleroma.Web, :controller
+  alias Pleroma.Web.Metadata
+  alias Pleroma.User
 
-  def redirector(conn, _params) do
+  def redirector(conn, _params, code \\ 200) do
     conn
     |> put_resp_content_type("text/html")
-    |> send_file(200, Application.app_dir(:pleroma, "priv_sid/static/index.html"))
+    |> send_file(code, index_file_path())
+  end
+
+  def redirector_with_meta(conn, %{"maybe_nickname_or_id" => maybe_nickname_or_id} = params) do
+    with %User{} = user <- User.get_cached_by_nickname_or_id(maybe_nickname_or_id) do
+      redirector_with_meta(conn, %{user: user})
+    else
+      nil ->
+        redirector(conn, params)
+    end
+  end
+
+  def redirector_with_meta(conn, params) do
+    {:ok, index_content} = File.read(index_file_path())
+    tags = Metadata.build_tags(params)
+    response = String.replace(index_content, "<!--server-generated-meta-->", tags)
+
+    conn
+    |> put_resp_content_type("text/html")
+    |> send_resp(200, response)
+  end
+
+  def index_file_path do
+    Pleroma.Plugs.InstanceStatic.file_path("index.html")
   end
 
   def registration_page(conn, params) do

lib/pleroma/web/salmon/salmon.ex

@@ -6,6 +6,7 @@ defmodule Pleroma.Web.Salmon do
   @httpoison Application.get_env(:pleroma, :httpoison)
 
   use Bitwise
+  alias Pleroma.Instances
   alias Pleroma.Web.XML
   alias Pleroma.Web.OStatus.ActivityRepresenter
   alias Pleroma.User
@@ -161,25 +162,31 @@ defmodule Pleroma.Web.Salmon do
     |> Enum.filter(fn user -> user && !user.local end)
   end
 
-  # push an activity to remote accounts
-  #
-  defp send_to_user(%{info: %{salmon: salmon}}, feed, poster),
-    do: send_to_user(salmon, feed, poster)
+  @doc "Pushes an activity to remote account."
+  def send_to_user(%{recipient: %{info: %{salmon: salmon}}} = params),
+    do: send_to_user(Map.put(params, :recipient, salmon))
 
-  defp send_to_user(url, feed, poster) when is_binary(url) do
-    with {:ok, %{status: code}} <-
+  def send_to_user(%{recipient: url, feed: feed, poster: poster} = params) when is_binary(url) do
+    with {:ok, %{status: code}} when code in 200..299 <-
            poster.(
              url,
              feed,
              [{"Content-Type", "application/magic-envelope+xml"}]
            ) do
+      if !Map.has_key?(params, :unreachable_since) || params[:unreachable_since],
+        do: Instances.set_reachable(url)
+
       Logger.debug(fn -> "Pushed to #{url}, code #{code}" end)
+      :ok
     else
-      e -> Logger.debug(fn -> "Pushing Salmon to #{url} failed, #{inspect(e)}" end)
+      e ->
+        unless params[:unreachable_since], do: Instances.set_reachable(url)
+        Logger.debug(fn -> "Pushing Salmon to #{url} failed, #{inspect(e)}" end)
+        :error
     end
   end
 
-  defp send_to_user(_, _, _), do: nil
+  def send_to_user(_), do: :noop
 
   @supported_activities [
     "Create",
@@ -209,12 +216,23 @@ defmodule Pleroma.Web.Salmon do
       {:ok, private, _} = keys_from_pem(keys)
       {:ok, feed} = encode(private, feed)
 
-      remote_users(activity)
+      remote_users = remote_users(activity)
+
+      salmon_urls = Enum.map(remote_users, & &1.info.salmon)
+      reachable_urls_metadata = Instances.filter_reachable(salmon_urls)
+      reachable_urls = Map.keys(reachable_urls_metadata)
+
+      remote_users
+      |> Enum.filter(&(&1.info.salmon in reachable_urls))
       |> Enum.each(fn remote_user ->
-        Task.start(fn ->
-          Logger.debug(fn -> "Sending Salmon to #{remote_user.ap_id}" end)
-          send_to_user(remote_user, feed, poster)
-        end)
+        Logger.debug(fn -> "Sending Salmon to #{remote_user.ap_id}" end)
+
+        Pleroma.Web.Federator.enqueue(:publish_single_salmon, %{
+          recipient: remote_user,
+          feed: feed,
+          poster: poster,
+          unreachable_since: reachable_urls_metadata[remote_user.info.salmon]
+        })
       end)
     end
   end

lib/pleroma/web/templates/layout/app.html.eex

@@ -1,7 +1,8 @@
 <!DOCTYPE html>
 <html>
   <head>
-    <meta charset=utf-8 />
+    <meta charset="utf-8" />
+    <meta name="viewport" content="width=device-width,initial-scale=1,minimal-ui" />
     <title>
     <%= Application.get_env(:pleroma, :instance)[:name] %>
     </title>

lib/pleroma/web/templates/mastodon_api/mastodon/index.html.eex

@@ -1,23 +1,28 @@
 <!DOCTYPE html>
 <html lang='en'>
 <head>
+<meta charset='utf-8'>
+<meta content='width=device-width, initial-scale=1' name='viewport'>
 <title>
 <%= Application.get_env(:pleroma, :instance)[:name] %>
 </title>
-<meta charset='utf-8'>
-<meta content='width=device-width, initial-scale=1' name='viewport'>
 <link rel="icon" type="image/png" href="/favicon.png"/>
-<link rel="stylesheet" media="all" href="/packs/common.css" />
-<link rel="stylesheet" media="all" href="/packs/default.css" />
+<script crossorigin='anonymous' src="/packs/locales.js"></script>
+<script crossorigin='anonymous' src="/packs/locales/glitch/en.js"></script>
 
-<script src="/packs/common.js"></script>
-<script src="/packs/locale_en.js"></script>
-<link as='script' crossorigin='anonymous' href='/packs/features/getting_started.js' rel='preload'>
-<link as='script' crossorigin='anonymous' href='/packs/features/compose.js' rel='preload'>
-<link as='script' crossorigin='anonymous' href='/packs/features/home_timeline.js' rel='preload'>
-<link as='script' crossorigin='anonymous' href='/packs/features/notifications.js' rel='preload'>
+<link rel='preload' as='script' crossorigin='anonymous' href='/packs/features/getting_started.js'>
+<link rel='preload' as='script' crossorigin='anonymous' href='/packs/features/compose.js'>
+<link rel='preload' as='script' crossorigin='anonymous' href='/packs/features/home_timeline.js'>
+<link rel='preload' as='script' crossorigin='anonymous' href='/packs/features/notifications.js'>
 <script id='initial-state' type='application/json'><%= raw @initial_state %></script>
-<script src="/packs/application.js"></script>
+
+<script src="/packs/core/common.js"></script>
+<link rel="stylesheet" media="all" href="/packs/core/common.css" />
+
+<script src="/packs/flavours/glitch/common.js"></script>
+<link rel="stylesheet" media="all" href="/packs/flavours/glitch/common.css" />
+
+<script src="/packs/flavours/glitch/home.js"></script>
 </head>
 <body class='app-body no-reduce-motion system-font'>
   <div class='app-holder' data-props='{&quot;locale&quot;:&quot;en&quot;}' id='mastodon'>

lib/pleroma/web/twitter_api/controllers/util_controller.ex

@@ -183,25 +183,31 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do
           invitesEnabled: if(Keyword.get(instance, :invites_enabled, false), do: "1", else: "0")
         }
 
-        pleroma_fe = %{
-          theme: Keyword.get(instance_fe, :theme),
-          background: Keyword.get(instance_fe, :background),
-          logo: Keyword.get(instance_fe, :logo),
-          logoMask: Keyword.get(instance_fe, :logo_mask),
-          logoMargin: Keyword.get(instance_fe, :logo_margin),
-          redirectRootNoLogin: Keyword.get(instance_fe, :redirect_root_no_login),
-          redirectRootLogin: Keyword.get(instance_fe, :redirect_root_login),
-          chatDisabled: !Keyword.get(instance_chat, :enabled),
-          showInstanceSpecificPanel: Keyword.get(instance_fe, :show_instance_panel),
-          scopeOptionsEnabled: Keyword.get(instance_fe, :scope_options_enabled),
-          formattingOptionsEnabled: Keyword.get(instance_fe, :formatting_options_enabled),
-          collapseMessageWithSubject: Keyword.get(instance_fe, :collapse_message_with_subject),
-          hidePostStats: Keyword.get(instance_fe, :hide_post_stats),
-          hideUserStats: Keyword.get(instance_fe, :hide_user_stats),
-          scopeCopy: Keyword.get(instance_fe, :scope_copy),
-          subjectLineBehavior: Keyword.get(instance_fe, :subject_line_behavior),
-          alwaysShowSubjectInput: Keyword.get(instance_fe, :always_show_subject_input)
-        }
+        pleroma_fe =
+          if instance_fe do
+            %{
+              theme: Keyword.get(instance_fe, :theme),
+              background: Keyword.get(instance_fe, :background),
+              logo: Keyword.get(instance_fe, :logo),
+              logoMask: Keyword.get(instance_fe, :logo_mask),
+              logoMargin: Keyword.get(instance_fe, :logo_margin),
+              redirectRootNoLogin: Keyword.get(instance_fe, :redirect_root_no_login),
+              redirectRootLogin: Keyword.get(instance_fe, :redirect_root_login),
+              chatDisabled: !Keyword.get(instance_chat, :enabled),
+              showInstanceSpecificPanel: Keyword.get(instance_fe, :show_instance_panel),
+              scopeOptionsEnabled: Keyword.get(instance_fe, :scope_options_enabled),
+              formattingOptionsEnabled: Keyword.get(instance_fe, :formatting_options_enabled),
+              collapseMessageWithSubject:
+                Keyword.get(instance_fe, :collapse_message_with_subject),
+              hidePostStats: Keyword.get(instance_fe, :hide_post_stats),
+              hideUserStats: Keyword.get(instance_fe, :hide_user_stats),
+              scopeCopy: Keyword.get(instance_fe, :scope_copy),
+              subjectLineBehavior: Keyword.get(instance_fe, :subject_line_behavior),
+              alwaysShowSubjectInput: Keyword.get(instance_fe, :always_show_subject_input)
+            }
+          else
+            Pleroma.Config.get([:frontend_configurations, :pleroma_fe])
+          end
 
         managed_config = Keyword.get(instance, :managed_config)
 
@@ -216,6 +222,14 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do
     end
   end
 
+  def frontend_configurations(conn, _params) do
+    config =
+      Pleroma.Config.get(:frontend_configurations, %{})
+      |> Enum.into(%{})
+
+    json(conn, config)
+  end
+
   def version(conn, _params) do
     version = Pleroma.Application.named_version()
 

lib/pleroma/web/twitter_api/representers/activity_representer.ex

@@ -12,6 +12,7 @@ defmodule Pleroma.Web.TwitterAPI.Representers.ActivityRepresenter do
   alias Pleroma.Web.CommonAPI.Utils
   alias Pleroma.Formatter
   alias Pleroma.HTML
+  alias Pleroma.Web.MastodonAPI.StatusView
 
   defp user_by_ap_id(user_list, ap_id) do
     Enum.find(user_list, fn %{ap_id: user_id} -> ap_id == user_id end)
@@ -158,7 +159,9 @@ defmodule Pleroma.Web.TwitterAPI.Representers.ActivityRepresenter do
     mentions = opts[:mentioned] || []
 
     attentions =
-      activity.recipients
+      []
+      |> Utils.maybe_notify_to_recipients(activity)
+      |> Utils.maybe_notify_mentioned_recipients(activity)
       |> Enum.map(fn ap_id -> Enum.find(mentions, fn user -> ap_id == user.ap_id end) end)
       |> Enum.filter(& &1)
       |> Enum.map(fn user -> UserView.render("show.json", %{user: user, for: opts[:for]}) end)
@@ -184,6 +187,12 @@ defmodule Pleroma.Web.TwitterAPI.Representers.ActivityRepresenter do
 
     summary = HTML.strip_tags(object["summary"])
 
+    card =
+      StatusView.render(
+        "card.json",
+        Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity)
+      )
+
     %{
       "id" => activity.id,
       "uri" => activity.data["object"]["id"],
@@ -212,7 +221,8 @@ defmodule Pleroma.Web.TwitterAPI.Representers.ActivityRepresenter do
       "possibly_sensitive" => possibly_sensitive,
       "visibility" => Pleroma.Web.MastodonAPI.StatusView.get_visibility(object),
       "summary" => summary,
-      "summary_html" => summary |> Formatter.emojify(object["emoji"])
+      "summary_html" => summary |> Formatter.emojify(object["emoji"]),
+      "card" => card
     }
   end
 

lib/pleroma/web/twitter_api/twitter_api.ex

@@ -70,14 +70,14 @@ defmodule Pleroma.Web.TwitterAPI.TwitterAPI do
 
   def repeat(%User{} = user, ap_id_or_id) do
     with {:ok, _announce, %{data: %{"id" => id}}} <- CommonAPI.repeat(ap_id_or_id, user),
-         %Activity{} = activity <- Activity.get_create_activity_by_object_ap_id(id) do
+         %Activity{} = activity <- Activity.get_create_by_object_ap_id(id) do
       {:ok, activity}
     end
   end
 
   def unrepeat(%User{} = user, ap_id_or_id) do
     with {:ok, _unannounce, %{data: %{"id" => id}}} <- CommonAPI.unrepeat(ap_id_or_id, user),
-         %Activity{} = activity <- Activity.get_create_activity_by_object_ap_id(id) do
+         %Activity{} = activity <- Activity.get_create_by_object_ap_id(id) do
       {:ok, activity}
     end
   end
@@ -92,14 +92,14 @@ defmodule Pleroma.Web.TwitterAPI.TwitterAPI do
 
   def fav(%User{} = user, ap_id_or_id) do
     with {:ok, _fav, %{data: %{"id" => id}}} <- CommonAPI.favorite(ap_id_or_id, user),
-         %Activity{} = activity <- Activity.get_create_activity_by_object_ap_id(id) do
+         %Activity{} = activity <- Activity.get_create_by_object_ap_id(id) do
       {:ok, activity}
     end
   end
 
   def unfav(%User{} = user, ap_id_or_id) do
     with {:ok, _unfav, _fav, %{data: %{"id" => id}}} <- CommonAPI.unfavorite(ap_id_or_id, user),
-         %Activity{} = activity <- Activity.get_create_activity_by_object_ap_id(id) do
+         %Activity{} = activity <- Activity.get_create_by_object_ap_id(id) do
       {:ok, activity}
     end
   end

lib/pleroma/web/twitter_api/twitter_api_controller.ex

@@ -24,7 +24,7 @@ defmodule Pleroma.Web.TwitterAPI.Controller do
 
     conn
     |> put_view(UserView)
-    |> render("show.json", %{user: user, token: token})
+    |> render("show.json", %{user: user, token: token, for: user})
   end
 
   def status_update(%{assigns: %{user: user}} = conn, %{"status" => _} = status_data) do
@@ -265,8 +265,6 @@ defmodule Pleroma.Web.TwitterAPI.Controller do
   end
 
   def fetch_conversation(%{assigns: %{user: user}} = conn, %{"id" => id}) do
-    id = String.to_integer(id)
-
     with context when is_binary(context) <- TwitterAPI.conversation_id_to_context(id),
          activities <-
            ActivityPub.fetch_activities_for_context(context, %{
@@ -330,54 +328,57 @@ defmodule Pleroma.Web.TwitterAPI.Controller do
   end
 
   def get_by_id_or_ap_id(id) do
-    activity = Repo.get(Activity, id) || Activity.get_create_activity_by_object_ap_id(id)
+    activity = Repo.get(Activity, id) || Activity.get_create_by_object_ap_id(id)
 
     if activity.data["type"] == "Create" do
       activity
     else
-      Activity.get_create_activity_by_object_ap_id(activity.data["object"])
+      Activity.get_create_by_object_ap_id(activity.data["object"])
     end
   end
 
   def favorite(%{assigns: %{user: user}} = conn, %{"id" => id}) do
-    with {_, {:ok, id}} <- {:param_cast, Ecto.Type.cast(:integer, id)},
-         {:ok, activity} <- TwitterAPI.fav(user, id) do
+    with {:ok, activity} <- TwitterAPI.fav(user, id) do
       conn
       |> put_view(ActivityView)
       |> render("activity.json", %{activity: activity, for: user})
+    else
+      _ -> json_reply(conn, 400, Jason.encode!(%{}))
     end
   end
 
   def unfavorite(%{assigns: %{user: user}} = conn, %{"id" => id}) do
-    with {_, {:ok, id}} <- {:param_cast, Ecto.Type.cast(:integer, id)},
-         {:ok, activity} <- TwitterAPI.unfav(user, id) do
+    with {:ok, activity} <- TwitterAPI.unfav(user, id) do
       conn
       |> put_view(ActivityView)
       |> render("activity.json", %{activity: activity, for: user})
+    else
+      _ -> json_reply(conn, 400, Jason.encode!(%{}))
     end
   end
 
   def retweet(%{assigns: %{user: user}} = conn, %{"id" => id}) do
-    with {_, {:ok, id}} <- {:param_cast, Ecto.Type.cast(:integer, id)},
-         {:ok, activity} <- TwitterAPI.repeat(user, id) do
+    with {:ok, activity} <- TwitterAPI.repeat(user, id) do
       conn
       |> put_view(ActivityView)
       |> render("activity.json", %{activity: activity, for: user})
+    else
+      _ -> json_reply(conn, 400, Jason.encode!(%{}))
     end
   end
 
   def unretweet(%{assigns: %{user: user}} = conn, %{"id" => id}) do
-    with {_, {:ok, id}} <- {:param_cast, Ecto.Type.cast(:integer, id)},
-         {:ok, activity} <- TwitterAPI.unrepeat(user, id) do
+    with {:ok, activity} <- TwitterAPI.unrepeat(user, id) do
       conn
       |> put_view(ActivityView)
       |> render("activity.json", %{activity: activity, for: user})
+    else
+      _ -> json_reply(conn, 400, Jason.encode!(%{}))
     end
   end
 
   def pin(%{assigns: %{user: user}} = conn, %{"id" => id}) do
-    with {_, {:ok, id}} <- {:param_cast, Ecto.Type.cast(:integer, id)},
-         {:ok, activity} <- TwitterAPI.pin(user, id) do
+    with {:ok, activity} <- TwitterAPI.pin(user, id) do
       conn
       |> put_view(ActivityView)
       |> render("activity.json", %{activity: activity, for: user})
@@ -388,8 +389,7 @@ defmodule Pleroma.Web.TwitterAPI.Controller do
   end
 
   def unpin(%{assigns: %{user: user}} = conn, %{"id" => id}) do
-    with {_, {:ok, id}} <- {:param_cast, Ecto.Type.cast(:integer, id)},
-         {:ok, activity} <- TwitterAPI.unpin(user, id) do
+    with {:ok, activity} <- TwitterAPI.unpin(user, id) do
       conn
       |> put_view(ActivityView)
       |> render("activity.json", %{activity: activity, for: user})
@@ -503,7 +503,7 @@ defmodule Pleroma.Web.TwitterAPI.Controller do
       followers =
         cond do
           for_user && user.id == for_user.id -> followers
-          user.info.hide_network -> []
+          user.info.hide_followers -> []
           true -> followers
         end
 
@@ -523,7 +523,7 @@ defmodule Pleroma.Web.TwitterAPI.Controller do
       friends =
         cond do
           for_user && user.id == for_user.id -> friends
-          user.info.hide_network -> []
+          user.info.hide_follows -> []
           true -> friends
         end
 
@@ -556,7 +556,6 @@ defmodule Pleroma.Web.TwitterAPI.Controller do
 
   def approve_friend_request(conn, %{"user_id" => uid} = _params) do
     with followed <- conn.assigns[:user],
-         uid when is_number(uid) <- String.to_integer(uid),
          %User{} = follower <- Repo.get(User, uid),
          {:ok, follower} <- User.maybe_follow(follower, followed),
          %Activity{} = follow_activity <- Utils.fetch_latest_follow(follower, followed),
@@ -578,7 +577,6 @@ defmodule Pleroma.Web.TwitterAPI.Controller do
 
   def deny_friend_request(conn, %{"user_id" => uid} = _params) do
     with followed <- conn.assigns[:user],
-         uid when is_number(uid) <- String.to_integer(uid),
          %User{} = follower <- Repo.get(User, uid),
          %Activity{} = follow_activity <- Utils.fetch_latest_follow(follower, followed),
          {:ok, follow_activity} <- Utils.update_follow_state(follow_activity, "reject"),
@@ -620,7 +618,7 @@ defmodule Pleroma.Web.TwitterAPI.Controller do
 
   defp build_info_cng(user, params) do
     info_params =
-      ["no_rich_text", "locked", "hide_network"]
+      ["no_rich_text", "locked", "hide_followers", "hide_follows", "show_role"]
       |> Enum.reduce(%{}, fn key, res ->
         if value = params[key] do
           Map.put(res, key, value == "true")

lib/pleroma/web/twitter_api/views/activity_view.ex

@@ -10,6 +10,7 @@ defmodule Pleroma.Web.TwitterAPI.ActivityView do
   alias Pleroma.Web.TwitterAPI.ActivityView
   alias Pleroma.Web.TwitterAPI.TwitterAPI
   alias Pleroma.Web.TwitterAPI.Representers.ObjectRepresenter
+  alias Pleroma.Web.MastodonAPI.StatusView
   alias Pleroma.Activity
   alias Pleroma.HTML
   alias Pleroma.Object
@@ -114,7 +115,7 @@ defmodule Pleroma.Web.TwitterAPI.ActivityView do
       |> Map.put(:context_ids, context_ids)
       |> Map.put(:users, users)
 
-    render_many(
+    safe_render_many(
       opts.activities,
       ActivityView,
       "activity.json",
@@ -168,7 +169,7 @@ defmodule Pleroma.Web.TwitterAPI.ActivityView do
   def render("activity.json", %{activity: %{data: %{"type" => "Announce"}} = activity} = opts) do
     user = get_user(activity.data["actor"], opts)
     created_at = activity.data["published"] |> Utils.date_to_asctime()
-    announced_activity = Activity.get_create_activity_by_object_ap_id(activity.data["object"])
+    announced_activity = Activity.get_create_by_object_ap_id(activity.data["object"])
 
     text = "#{user.nickname} retweeted a status."
 
@@ -192,7 +193,7 @@ defmodule Pleroma.Web.TwitterAPI.ActivityView do
 
   def render("activity.json", %{activity: %{data: %{"type" => "Like"}} = activity} = opts) do
     user = get_user(activity.data["actor"], opts)
-    liked_activity = Activity.get_create_activity_by_object_ap_id(activity.data["object"])
+    liked_activity = Activity.get_create_by_object_ap_id(activity.data["object"])
     liked_activity_id = if liked_activity, do: liked_activity.id, else: nil
 
     created_at =
@@ -236,7 +237,9 @@ defmodule Pleroma.Web.TwitterAPI.ActivityView do
     pinned = activity.id in user.info.pinned_activities
 
     attentions =
-      activity.recipients
+      []
+      |> Utils.maybe_notify_to_recipients(activity)
+      |> Utils.maybe_notify_mentioned_recipients(activity)
       |> Enum.map(fn ap_id -> get_user(ap_id, opts) end)
       |> Enum.filter(& &1)
       |> Enum.map(fn user -> UserView.render("show.json", %{user: user, for: opts[:for]}) end)
@@ -272,6 +275,12 @@ defmodule Pleroma.Web.TwitterAPI.ActivityView do
 
     summary = HTML.strip_tags(summary)
 
+    card =
+      StatusView.render(
+        "card.json",
+        Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity)
+      )
+
     %{
       "id" => activity.id,
       "uri" => activity.data["object"]["id"],
@@ -298,9 +307,10 @@ defmodule Pleroma.Web.TwitterAPI.ActivityView do
       "tags" => tags,
       "activity_type" => "post",
       "possibly_sensitive" => possibly_sensitive,
-      "visibility" => Pleroma.Web.MastodonAPI.StatusView.get_visibility(object),
+      "visibility" => StatusView.get_visibility(object),
       "summary" => summary,
-      "summary_html" => summary |> Formatter.emojify(object["emoji"])
+      "summary_html" => summary |> Formatter.emojify(object["emoji"]),
+      "card" => card
     }
   end
 

lib/pleroma/web/twitter_api/views/user_view.ex

@@ -108,6 +108,8 @@ defmodule Pleroma.Web.TwitterAPI.UserView do
       "locked" => user.info.locked,
       "default_scope" => user.info.default_scope,
       "no_rich_text" => user.info.no_rich_text,
+      "hide_followers" => user.info.hide_followers,
+      "hide_follows" => user.info.hide_follows,
       "fields" => fields,
 
       # Pleroma extension
@@ -117,6 +119,12 @@ defmodule Pleroma.Web.TwitterAPI.UserView do
       }
     }
 
+    data =
+      if(user.info.is_admin || user.info.is_moderator,
+        do: maybe_with_role(data, user, for_user),
+        else: data
+      )
+
     if assigns[:token] do
       Map.put(data, "token", token_string(assigns[:token]))
     else
@@ -124,6 +132,20 @@ defmodule Pleroma.Web.TwitterAPI.UserView do
     end
   end
 
+  defp maybe_with_role(data, %User{id: id} = user, %User{id: id}) do
+    Map.merge(data, %{"role" => role(user), "show_role" => user.info.show_role})
+  end
+
+  defp maybe_with_role(data, %User{info: %{show_role: true}} = user, _user) do
+    Map.merge(data, %{"role" => role(user)})
+  end
+
+  defp maybe_with_role(data, _, _), do: data
+
+  defp role(%User{info: %{:is_admin => true}}), do: "admin"
+  defp role(%User{info: %{:is_moderator => true}}), do: "moderator"
+  defp role(_), do: "member"
+
   defp image_url(%{"url" => [%{"href" => href} | _]}), do: href
   defp image_url(_), do: nil
 

lib/pleroma/web/uploader_controller.ex

@@ -0,0 +1,25 @@
+defmodule Pleroma.Web.UploaderController do
+  use Pleroma.Web, :controller
+
+  alias Pleroma.Uploaders.Uploader
+
+  def callback(conn, params = %{"upload_path" => upload_path}) do
+    process_callback(conn, :global.whereis_name({Uploader, upload_path}), params)
+  end
+
+  def callbacks(conn, _) do
+    send_resp(conn, 400, "bad request")
+  end
+
+  defp process_callback(conn, pid, params) when is_pid(pid) do
+    send(pid, {Uploader, self(), conn, params})
+
+    receive do
+      {Uploader, conn} -> conn
+    end
+  end
+
+  defp process_callback(conn, _, _) do
+    send_resp(conn, 400, "bad request")
+  end
+end

lib/pleroma/web/web.ex

@@ -38,6 +38,33 @@ defmodule Pleroma.Web do
       import Phoenix.Controller, only: [get_csrf_token: 0, get_flash: 2, view_module: 1]
 
       import Pleroma.Web.{ErrorHelpers, Gettext, Router.Helpers}
+
+      require Logger
+
+      @doc "Same as `render/3` but wrapped in a rescue block"
+      def safe_render(view, template, assigns \\ %{}) do
+        Phoenix.View.render(view, template, assigns)
+      rescue
+        error ->
+          Logger.error(
+            "#{__MODULE__} failed to render #{inspect({view, template})}: #{inspect(error)}"
+          )
+
+          Logger.error(inspect(__STACKTRACE__))
+          nil
+      end
+
+      @doc """
+      Same as `render_many/4` but wrapped in rescue block.
+      """
+      def safe_render_many(collection, view, template, assigns \\ %{}) do
+        Enum.map(collection, fn resource ->
+          as = Map.get(assigns, :as) || view.__resource__
+          assigns = Map.put(assigns, as, resource)
+          safe_render(view, template, assigns)
+        end)
+        |> Enum.filter(& &1)
+      end
     end
   end
 

lib/pleroma/web/websub/websub.ex

@@ -5,6 +5,7 @@
 defmodule Pleroma.Web.Websub do
   alias Ecto.Changeset
   alias Pleroma.Repo
+  alias Pleroma.Instances
   alias Pleroma.Web.Websub.{WebsubServerSubscription, WebsubClientSubscription}
   alias Pleroma.Web.OStatus.FeedRepresenter
   alias Pleroma.Web.{XML, Endpoint, OStatus}
@@ -53,28 +54,34 @@ defmodule Pleroma.Web.Websub do
   ]
   def publish(topic, user, %{data: %{"type" => type}} = activity)
       when type in @supported_activities do
-    # TODO: Only send to still valid subscriptions.
+    response =
+      user
+      |> FeedRepresenter.to_simple_form([activity], [user])
+      |> :xmerl.export_simple(:xmerl_xml)
+      |> to_string
+
     query =
       from(
         sub in WebsubServerSubscription,
         where: sub.topic == ^topic and sub.state == "active",
-        where: fragment("? > NOW()", sub.valid_until)
+        where: fragment("? > (NOW() at time zone 'UTC')", sub.valid_until)
       )
 
     subscriptions = Repo.all(query)
 
-    Enum.each(subscriptions, fn sub ->
-      response =
-        user
-        |> FeedRepresenter.to_simple_form([activity], [user])
-        |> :xmerl.export_simple(:xmerl_xml)
-        |> to_string
+    callbacks = Enum.map(subscriptions, & &1.callback)
+    reachable_callbacks_metadata = Instances.filter_reachable(callbacks)
+    reachable_callbacks = Map.keys(reachable_callbacks_metadata)
 
+    subscriptions
+    |> Enum.filter(&(&1.callback in reachable_callbacks))
+    |> Enum.each(fn sub ->
       data = %{
         xml: response,
         topic: topic,
         callback: sub.callback,
-        secret: sub.secret
+        secret: sub.secret,
+        unreachable_since: reachable_callbacks_metadata[sub.callback]
       }
 
       Pleroma.Web.Federator.enqueue(:publish_single_websub, data)
@@ -263,11 +270,11 @@ defmodule Pleroma.Web.Websub do
     end)
   end
 
-  def publish_one(%{xml: xml, topic: topic, callback: callback, secret: secret}) do
+  def publish_one(%{xml: xml, topic: topic, callback: callback, secret: secret} = params) do
     signature = sign(secret || "", xml)
     Logger.info(fn -> "Pushing #{topic} to #{callback}" end)
 
-    with {:ok, %{status: code}} <-
+    with {:ok, %{status: code}} when code in 200..299 <-
            @httpoison.post(
              callback,
              xml,
@@ -276,12 +283,16 @@ defmodule Pleroma.Web.Websub do
                {"X-Hub-Signature", "sha1=#{signature}"}
              ]
            ) do
+      if !Map.has_key?(params, :unreachable_since) || params[:unreachable_since],
+        do: Instances.set_reachable(callback)
+
       Logger.info(fn -> "Pushed to #{callback}, code #{code}" end)
       {:ok, code}
     else
-      e ->
-        Logger.debug(fn -> "Couldn't push to #{callback}, #{inspect(e)}" end)
-        {:error, e}
+      {_post_result, response} ->
+        unless params[:unreachable_since], do: Instances.set_reachable(callback)
+        Logger.debug(fn -> "Couldn't push to #{callback}, #{inspect(response)}" end)
+        {:error, response}
     end
   end
 end

lib/pleroma/web/websub/websub_client_subscription.ex

@@ -13,7 +13,7 @@ defmodule Pleroma.Web.Websub.WebsubClientSubscription do
     field(:state, :string)
     field(:subscribers, {:array, :string}, default: [])
     field(:hub, :string)
-    belongs_to(:user, User)
+    belongs_to(:user, User, type: Pleroma.FlakeId)
 
     timestamps()
   end

lib/pleroma/web/websub/websub_controller.ex

@@ -4,9 +4,11 @@
 
 defmodule Pleroma.Web.Websub.WebsubController do
   use Pleroma.Web, :controller
+
   alias Pleroma.{Repo, User}
   alias Pleroma.Web.{Websub, Federator}
   alias Pleroma.Web.Websub.WebsubClientSubscription
+
   require Logger
 
   plug(