Merge remote-tracking branch 'origin/develop' into shigusegubu

* origin/develop: (92 commits)
  Web.MastodonApi.MastodonSocketTest: Add test for unauthenticated websocket
  Web.Streamer: Get unauthenticated statuses representation
  Web.MastodonAPI.MastodonSocket: Put access_token at function-level
  Web.MastodonAPI.MastodonSocket: Add unauthentified websocket endpoints
  Improved version string
  mediaproxy: fix empty url & add some tests
  RetryQueue: tiny refractor, add tests
  Various runtime configuration fixes
  update pleroma frontend
  Federator: add retry queue.
  activitypub: object view: avoid leaking private details
  ostatus controller: respond with AS2 objects instead of activities to notice URIs
  tests: federator: fix formatting
  activitypub: transmogrifier: make deletes secure
  Web.AdminAPI.AdminAPIControllerTest: New Test
  Web.AdminAPI.AdminAPIController: Fixes bugs found with ExUnit
  test/plugs/user_is_admin_plug_test: New test
  lib/mix/tasks/relay*: Use a with block
  Change Relay from `status` to `{status, message}`
  Web.Router: Change right to permission group (except for function names)
  ...

lib/mix/tasks/relay_follow.ex

@@ -14,9 +14,11 @@ defmodule Mix.Tasks.RelayFollow do
   def run([target]) do
     Mix.Task.run("app.start")
 
-    :ok = Relay.follow(target)
-
-    # put this task to sleep to allow the genserver to push out the messages
-    :timer.sleep(500)
+    with {:ok, activity} <- Relay.follow(target) do
+      # put this task to sleep to allow the genserver to push out the messages
+      :timer.sleep(500)
+    else
+      {:error, e} -> Mix.shell().error("Error while following #{target}: #{inspect(e)}")
+    end
   end
 end

lib/mix/tasks/relay_unfollow.ex

@@ -13,9 +13,11 @@ defmodule Mix.Tasks.RelayUnfollow do
   def run([target]) do
     Mix.Task.run("app.start")
 
-    :ok = Relay.unfollow(target)
-
-    # put this task to sleep to allow the genserver to push out the messages
-    :timer.sleep(500)
+    with {:ok, activity} <- Relay.follow(target) do
+      # put this task to sleep to allow the genserver to push out the messages
+      :timer.sleep(500)
+    else
+      {:error, e} -> Mix.shell().error("Error while following #{target}: #{inspect(e)}")
+    end
   end
 end

lib/mix/tasks/sample_config.eex

@@ -25,6 +25,10 @@ config :pleroma, Pleroma.Repo,
   hostname: "localhost",
   pool_size: 10
 
+# Enable Strict-Transport-Security once SSL is working:
+# config :pleroma, :http_security,
+#   sts: true
+
 # Configure S3 support if desired.
 # The public S3 endpoint is different depending on region and provider,
 # consult your S3 provider's documentation for details on what to use.

lib/mix/tasks/set_admin.ex

@@ -0,0 +1,32 @@
+defmodule Mix.Tasks.SetAdmin do
+  use Mix.Task
+  alias Pleroma.User
+
+  @doc """
+  Sets admin status
+  Usage: set_admin nickname [true|false]
+  """
+  def run([nickname | rest]) do
+    Application.ensure_all_started(:pleroma)
+
+    status =
+      case rest do
+        [status] -> status == "true"
+        _ -> true
+      end
+
+    with %User{local: true} = user <- User.get_by_nickname(nickname) do
+      info =
+        user.info
+        |> Map.put("is_admin", !!status)
+
+      cng = User.info_changeset(user, %{info: info})
+      {:ok, user} = User.update_and_set_cache(cng)
+
+      IO.puts("Admin status of #{nickname}: #{user.info["is_admin"]}")
+    else
+      _ ->
+        IO.puts("No local user #{nickname}")
+    end
+  end
+end

lib/pleroma/application.ex

@@ -1,8 +1,15 @@
 defmodule Pleroma.Application do
   use Application
 
+  @name "Pleroma"
+  @version Mix.Project.config()[:version]
+  def name, do: @name
+  def version, do: @version
+  def named_version(), do: @name <> " " <> @version
+
   # See http://elixir-lang.org/docs/stable/elixir/Application.html
   # for more information on OTP Applications
+  @env Mix.env()
   def start(_type, _args) do
     import Supervisor.Spec
     import Cachex.Spec
@@ -57,10 +64,11 @@ defmodule Pleroma.Application do
           id: :cachex_idem
         ),
         worker(Pleroma.Web.Federator, []),
-        worker(Pleroma.Stats, []),
-        worker(Pleroma.Gopher.Server, [])
+        worker(Pleroma.Web.Federator.RetryQueue, []),
+        worker(Pleroma.Gopher.Server, []),
+        worker(Pleroma.Stats, [])
       ] ++
-        if Mix.env() == :test,
+        if @env == :test,
           do: [],
           else:
             [worker(Pleroma.Web.Streamer, [])] ++

lib/pleroma/object.ex

@@ -31,10 +31,12 @@ defmodule Pleroma.Object do
   def normalize(ap_id) when is_binary(ap_id), do: Object.get_by_ap_id(ap_id)
   def normalize(_), do: nil
 
-  def get_cached_by_ap_id(ap_id) do
-    if Mix.env() == :test do
+  if Mix.env() == :test do
+    def get_cached_by_ap_id(ap_id) do
       get_by_ap_id(ap_id)
-    else
+    end
+  else
+    def get_cached_by_ap_id(ap_id) do
       key = "object:#{ap_id}"
 
       Cachex.fetch!(:object_cache, key, fn _ ->

lib/pleroma/plugs/http_security_plug.ex

@@ -0,0 +1,58 @@
+defmodule Pleroma.Plugs.HTTPSecurityPlug do
+  alias Pleroma.Config
+  import Plug.Conn
+
+  def init(opts), do: opts
+
+  def call(conn, options) do
+    if Config.get([:http_security, :enabled]) do
+      conn =
+        merge_resp_headers(conn, headers())
+        |> maybe_send_sts_header(Config.get([:http_security, :sts]))
+    else
+      conn
+    end
+  end
+
+  defp headers do
+    referrer_policy = Config.get([:http_security, :referrer_policy])
+
+    [
+      {"x-xss-protection", "1; mode=block"},
+      {"x-permitted-cross-domain-policies", "none"},
+      {"x-frame-options", "DENY"},
+      {"x-content-type-options", "nosniff"},
+      {"referrer-policy", referrer_policy},
+      {"x-download-options", "noopen"},
+      {"content-security-policy", csp_string() <> ";"}
+    ]
+  end
+
+  defp csp_string do
+    [
+      "default-src 'none'",
+      "base-uri 'self'",
+      "frame-ancestors 'none'",
+      "img-src 'self' data: https:",
+      "media-src 'self' https:",
+      "style-src 'self' 'unsafe-inline'",
+      "font-src 'self'",
+      "script-src 'self'",
+      "connect-src 'self' " <> String.replace(Pleroma.Web.Endpoint.static_url(), "http", "ws"),
+      "upgrade-insecure-requests"
+    ]
+    |> Enum.join("; ")
+  end
+
+  defp maybe_send_sts_header(conn, true) do
+    max_age_sts = Config.get([:http_security, :sts_max_age])
+    max_age_ct = Config.get([:http_security, :ct_max_age])
+
+    merge_resp_headers(conn, [
+      {"strict-transport-security", "max-age=#{max_age_sts}; includeSubDomains"},
+      {"expect-ct", "enforce, max-age=#{max_age_ct}"}
+    ])
+  end
+
+  defp maybe_send_sts_header(conn, _), do: conn
+end

lib/pleroma/plugs/user_is_admin_plug.ex

@@ -0,0 +1,19 @@
+defmodule Pleroma.Plugs.UserIsAdminPlug do
+  import Plug.Conn
+  alias Pleroma.User
+
+  def init(options) do
+    options
+  end
+
+  def call(%{assigns: %{user: %User{info: %{"is_admin" => true}}}} = conn, _) do
+    conn
+  end
+
+  def call(conn, _) do
+    conn
+    |> put_resp_content_type("application/json")
+    |> send_resp(403, Jason.encode!(%{error: "User is not admin."}))
+    |> halt
+  end
+end

lib/pleroma/uploaders/mdii.ex

@@ -0,0 +1,26 @@
+defmodule Pleroma.Uploaders.MDII do
+  alias Pleroma.Config
+
+  @behaviour Pleroma.Uploaders.Uploader
+
+  @httpoison Application.get_env(:pleroma, :httpoison)
+
+  def put_file(name, uuid, path, content_type, should_dedupe) do
+    cgi = Pleroma.Config.get([Pleroma.Uploaders.MDII, :cgi])
+    files = Pleroma.Config.get([Pleroma.Uploaders.MDII, :files])
+
+    {:ok, file_data} = File.read(path)
+
+    extension = String.split(name, ".") |> List.last()
+    query = "#{cgi}?#{extension}"
+
+    with {:ok, %{status_code: 200, body: body}} <- @httpoison.post(query, file_data) do
+      File.rm!(path)
+      remote_file_name = String.split(body) |> List.first()
+      public_url = "#{files}/#{remote_file_name}.#{extension}"
+      {:ok, public_url}
+    else
+      _ -> Pleroma.Uploaders.Local.put_file(name, uuid, path, content_type, should_dedupe)
+    end
+  end
+end

lib/pleroma/user.ex

@@ -498,7 +498,7 @@ defmodule Pleroma.User do
     Repo.all(query)
   end
 
-  def search(query, resolve) do
+  def search(query, resolve \\ false) do
     # strip the beginning @ off if there is a query
     query = String.trim_leading(query, "@")
 

lib/pleroma/web/activity_pub/activity_pub.ex

@@ -628,9 +628,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
   end
 
   def fetch_and_prepare_user_from_ap_id(ap_id) do
-    with {:ok, %{status_code: 200, body: body}} <-
-           @httpoison.get(ap_id, [Accept: "application/activity+json"], follow_redirect: true),
-         {:ok, data} <- Jason.decode(body) do
+    with {:ok, data} <- fetch_and_contain_remote_object_from_id(ap_id) do
       user_data_from_user_object(data)
     else
       e -> Logger.error("Could not decode user at fetch #{ap_id}, #{inspect(e)}")
@@ -732,22 +730,13 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     else
       Logger.info("Fetching #{id} via AP")
 
-      with true <- String.starts_with?(id, "http"),
-           {:ok, %{body: body, status_code: code}} when code in 200..299 <-
-             @httpoison.get(
-               id,
-               [Accept: "application/activity+json"],
-               follow_redirect: true,
-               timeout: 10000,
-               recv_timeout: 20000
-             ),
-           {:ok, data} <- Jason.decode(body),
+      with {:ok, data} <- fetch_and_contain_remote_object_from_id(id),
            nil <- Object.normalize(data),
            params <- %{
              "type" => "Create",
              "to" => data["to"],
              "cc" => data["cc"],
-             "actor" => data["attributedTo"],
+             "actor" => data["actor"] || data["attributedTo"],
              "object" => data
            },
            :ok <- Transmogrifier.contain_origin(id, params),
@@ -771,6 +760,27 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     end
   end
 
+  def fetch_and_contain_remote_object_from_id(id) do
+    Logger.info("Fetching #{id} via AP")
+
+    with true <- String.starts_with?(id, "http"),
+         {:ok, %{body: body, status_code: code}} when code in 200..299 <-
+           @httpoison.get(
+             id,
+             [Accept: "application/activity+json"],
+             follow_redirect: true,
+             timeout: 10000,
+             recv_timeout: 20000
+           ),
+         {:ok, data} <- Jason.decode(body),
+         :ok <- Transmogrifier.contain_origin_from_id(id, data) do
+      {:ok, data}
+    else
+      e ->
+        {:error, e}
+    end
+  end
+
   def is_public?(activity) do
     "https://www.w3.org/ns/activitystreams#Public" in (activity.data["to"] ++
                                                          (activity.data["cc"] || []))

lib/pleroma/web/activity_pub/relay.ex

@@ -12,11 +12,12 @@ defmodule Pleroma.Web.ActivityPub.Relay do
          %User{} = target_user <- User.get_or_fetch_by_ap_id(target_instance),
          {:ok, activity} <- ActivityPub.follow(local_user, target_user) do
       Logger.info("relay: followed instance: #{target_instance}; id=#{activity.data["id"]}")
+      {:ok, activity}
     else
-      e -> Logger.error("error: #{inspect(e)}")
+      e ->
+        Logger.error("error: #{inspect(e)}")
+        {:error, e}
     end
-
-    :ok
   end
 
   def unfollow(target_instance) do
@@ -24,11 +25,12 @@ defmodule Pleroma.Web.ActivityPub.Relay do
          %User{} = target_user <- User.get_or_fetch_by_ap_id(target_instance),
          {:ok, activity} <- ActivityPub.unfollow(local_user, target_user) do
       Logger.info("relay: unfollowed instance: #{target_instance}: id=#{activity.data["id"]}")
+      {:ok, activity}
     else
-      e -> Logger.error("error: #{inspect(e)}")
+      e ->
+        Logger.error("error: #{inspect(e)}")
+        {:error, e}
     end
-
-    :ok
   end
 
   def publish(%Activity{data: %{"type" => "Create"}} = activity) do

lib/pleroma/web/activity_pub/transmogrifier.ex

@@ -50,6 +50,19 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
     end
   end
 
+  def contain_origin_from_id(id, %{"id" => nil}), do: :error
+
+  def contain_origin_from_id(id, %{"id" => other_id} = params) do
+    id_uri = URI.parse(id)
+    other_uri = URI.parse(other_id)
+
+    if id_uri.host == other_uri.host do
+      :ok
+    else
+      :error
+    end
+  end
+
   @doc """
   Modifies an incoming AP object (mastodon format) to our internal format.
   """
@@ -454,15 +467,20 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
     end
   end
 
-  # TODO: Make secure.
+  # TODO: We presently assume that any actor on the same origin domain as the object being
+  # deleted has the rights to delete that object.  A better way to validate whether or not
+  # the object should be deleted is to refetch the object URI, which should return either
+  # an error or a tombstone.  This would allow us to verify that a deletion actually took
+  # place.
   def handle_incoming(
-        %{"type" => "Delete", "object" => object_id, "actor" => actor, "id" => _id} = data
+        %{"type" => "Delete", "object" => object_id, "actor" => _actor, "id" => _id} = data
       ) do
     object_id = Utils.get_ap_id(object_id)
 
     with actor <- get_actor(data),
-         %User{} = _actor <- User.get_or_fetch_by_ap_id(actor),
+         %User{} = actor <- User.get_or_fetch_by_ap_id(actor),
          {:ok, object} <- get_obj_helper(object_id) || fetch_obj_helper(object_id),
+         :ok <- contain_origin(actor.ap_id, object.data),
          {:ok, activity} <- ActivityPub.delete(object, false) do
       {:ok, activity}
     else

lib/pleroma/web/activity_pub/views/object_view.ex

@@ -10,7 +10,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectView do
     Map.merge(base, additional)
   end
 
-  def render("object.json", %{object: %Activity{} = activity}) do
+  def render("object.json", %{object: %Activity{data: %{"type" => "Create"}} = activity}) do
     base = Pleroma.Web.ActivityPub.Utils.make_json_ld_header()
     object = Object.normalize(activity.data["object"])
 
@@ -20,4 +20,15 @@ defmodule Pleroma.Web.ActivityPub.ObjectView do
 
     Map.merge(base, additional)
   end
+
+  def render("object.json", %{object: %Activity{} = activity}) do
+    base = Pleroma.Web.ActivityPub.Utils.make_json_ld_header()
+    object = Object.normalize(activity.data["object"])
+
+    additional =
+      Transmogrifier.prepare_object(activity.data)
+      |> Map.put("object", object.data["id"])
+
+    Map.merge(base, additional)
+  end
 end

lib/pleroma/web/admin_api/admin_api_controller.ex

@@ -0,0 +1,158 @@
+defmodule Pleroma.Web.AdminAPI.AdminAPIController do
+  use Pleroma.Web, :controller
+  alias Pleroma.{User, Repo}
+  alias Pleroma.Web.ActivityPub.Relay
+
+  require Logger
+
+  action_fallback(:errors)
+
+  def user_delete(conn, %{"nickname" => nickname}) do
+    user = User.get_by_nickname(nickname)
+
+    if user.local == true do
+      User.delete(user)
+    else
+      User.delete(user)
+    end
+
+    conn
+    |> json(nickname)
+  end
+
+  def user_create(
+        conn,
+        %{"nickname" => nickname, "email" => email, "password" => password}
+      ) do
+    new_user = %{
+      nickname: nickname,
+      name: nickname,
+      email: email,
+      password: password,
+      password_confirmation: password,
+      bio: "."
+    }
+
+    User.register_changeset(%User{}, new_user)
+    |> Repo.insert!()
+
+    conn
+    |> json(new_user.nickname)
+  end
+
+  def right_add(conn, %{"permission_group" => permission_group, "nickname" => nickname})
+      when permission_group in ["moderator", "admin"] do
+    user = User.get_by_nickname(nickname)
+
+    info =
+      user.info
+      |> Map.put("is_" <> permission_group, true)
+
+    cng = User.info_changeset(user, %{info: info})
+    {:ok, user} = User.update_and_set_cache(cng)
+
+    conn
+    |> json(user.info)
+  end
+
+  def right_get(conn, %{"nickname" => nickname}) do
+    user = User.get_by_nickname(nickname)
+
+    conn
+    |> json(user.info)
+  end
+
+  def right_add(conn, _) do
+    conn
+    |> put_status(404)
+    |> json(%{error: "No such permission_group"})
+  end
+
+  def right_delete(
+        %{assigns: %{user: %User{:nickname => admin_nickname}}} = conn,
+        %{
+          "permission_group" => permission_group,
+          "nickname" => nickname
+        }
+      )
+      when permission_group in ["moderator", "admin"] do
+    if admin_nickname == nickname do
+      conn
+      |> put_status(403)
+      |> json(%{error: "You can't revoke your own admin status."})
+    else
+      user = User.get_by_nickname(nickname)
+
+      info =
+        user.info
+        |> Map.put("is_" <> permission_group, false)
+
+      cng = User.info_changeset(user, %{info: info})
+      {:ok, user} = User.update_and_set_cache(cng)
+
+      conn
+      |> json(user.info)
+    end
+  end
+
+  def right_delete(conn, _) do
+    conn
+    |> put_status(404)
+    |> json(%{error: "No such permission_group"})
+  end
+
+  def relay_follow(conn, %{"relay_url" => target}) do
+    {status, message} = Relay.follow(target)
+
+    if status == :ok do
+      conn
+      |> json(target)
+    else
+      conn
+      |> put_status(500)
+      |> json(target)
+    end
+  end
+
+  def relay_unfollow(conn, %{"relay_url" => target}) do
+    {status, message} = Relay.unfollow(target)
+
+    if status == :ok do
+      conn
+      |> json(target)
+    else
+      conn
+      |> put_status(500)
+      |> json(target)
+    end
+  end
+
+  @shortdoc "Get a account registeration invite token (base64 string)"
+  def get_invite_token(conn, _params) do
+    {:ok, token} = Pleroma.UserInviteToken.create_token()
+
+    conn
+    |> json(token.token)
+  end
+
+  @shortdoc "Get a password reset token (base64 string) for given nickname"
+  def get_password_reset(conn, %{"nickname" => nickname}) do
+    (%User{local: true} = user) = User.get_by_nickname(nickname)
+    {:ok, token} = Pleroma.PasswordResetToken.create_token(user)
+
+    conn
+    |> json(token.token)
+  end
+
+  def errors(conn, {:param_cast, _}) do
+    conn
+    |> put_status(400)
+    |> json("Invalid parameters")
+  end
+
+  def errors(conn, _) do
+    conn
+    |> put_status(500)
+    |> json("Something went wrong")
+  end
+end

lib/pleroma/web/channels/user_socket.ex

@@ -4,9 +4,7 @@ defmodule Pleroma.Web.UserSocket do
 
   ## Channels
   # channel "room:*", Pleroma.Web.RoomChannel
-  if Application.get_env(:pleroma, :chat) |> Keyword.get(:enabled) do
-    channel("chat:*", Pleroma.Web.ChatChannel)
-  end
+  channel("chat:*", Pleroma.Web.ChatChannel)
 
   ## Transports
   transport(:websocket, Phoenix.Transports.WebSocket)
@@ -24,7 +22,8 @@ defmodule Pleroma.Web.UserSocket do
   # See `Phoenix.Token` documentation for examples in
   # performing token verification on connect.
   def connect(%{"token" => token}, socket) do
-    with {:ok, user_id} <- Phoenix.Token.verify(socket, "user socket", token, max_age: 84600),
+    with true <- Pleroma.Config.get([:chat, :enabled]),
+         {:ok, user_id} <- Phoenix.Token.verify(socket, "user socket", token, max_age: 84600),
          %User{} = user <- Pleroma.Repo.get(User, user_id) do
       {:ok, assign(socket, :user_name, user.nickname)}
     else

lib/pleroma/web/endpoint.ex

@@ -1,9 +1,7 @@
 defmodule Pleroma.Web.Endpoint do
   use Phoenix.Endpoint, otp_app: :pleroma
 
-  if Application.get_env(:pleroma, :chat) |> Keyword.get(:enabled) do
-    socket("/socket", Pleroma.Web.UserSocket)
-  end
+  socket("/socket", Pleroma.Web.UserSocket)
 
   socket("/api/v1", Pleroma.Web.MastodonAPI.MastodonSocket)
 
@@ -12,6 +10,7 @@ defmodule Pleroma.Web.Endpoint do
   # You should set gzip to true if you are running phoenix.digest
   # when deploying your static files in production.
   plug(CORSPlug)
+  plug(Pleroma.Plugs.HTTPSecurityPlug)
 
   plug(Plug.Static, at: "/media", from: Pleroma.Uploaders.Local.upload_path(), gzip: false)
 
@@ -45,14 +44,19 @@ defmodule Pleroma.Web.Endpoint do
   plug(Plug.MethodOverride)
   plug(Plug.Head)
 
+  cookie_name =
+    if Application.get_env(:pleroma, Pleroma.Web.Endpoint) |> Keyword.get(:secure_cookie_flag),
+      do: "__Host-pleroma_key",
+      else: "pleroma_key"
+
   # The session will be stored in the cookie and signed,
   # this means its contents can be read but not tampered with.
   # Set :encryption_salt if you would also like to encrypt it.
   plug(
     Plug.Session,
     store: :cookie,
-    key: "_pleroma_key",
-    signing_salt: "CqaoopA2",
+    key: cookie_name,
+    signing_salt: {Pleroma.Config, :get, [[__MODULE__, :signing_salt], "CqaoopA2"]},
     http_only: true,
     secure:
       Application.get_env(:pleroma, Pleroma.Web.Endpoint) |> Keyword.get(:secure_cookie_flag),

lib/pleroma/web/federator/federator.ex

@@ -3,6 +3,7 @@ defmodule Pleroma.Web.Federator do
   alias Pleroma.User
   alias Pleroma.Activity
   alias Pleroma.Web.{WebFinger, Websub}
+  alias Pleroma.Web.Federator.RetryQueue
   alias Pleroma.Web.ActivityPub.ActivityPub
   alias Pleroma.Web.ActivityPub.Relay
   alias Pleroma.Web.ActivityPub.Transmogrifier
@@ -101,44 +102,46 @@ defmodule Pleroma.Web.Federator do
 
     params = Utils.normalize_params(params)
 
+    # NOTE: we use the actor ID to do the containment, this is fine because an
+    # actor shouldn't be acting on objects outside their own AP server.
     with {:ok, _user} <- ap_enabled_actor(params["actor"]),
          nil <- Activity.normalize(params["id"]),
-         {:ok, _activity} <- Transmogrifier.handle_incoming(params) do
+         :ok <- Transmogrifier.contain_origin_from_id(params["actor"], params),
+         {:ok, activity} <- Transmogrifier.handle_incoming(params) do
+      {:ok, activity}
     else
       %Activity{} ->
         Logger.info("Already had #{params["id"]}")
+        :error
 
       _e ->
         # Just drop those for now
         Logger.info("Unhandled activity")
         Logger.info(Poison.encode!(params, pretty: 2))
+        :error
     end
   end
 
   def handle(:publish_single_ap, params) do
-    ActivityPub.publish_one(params)
+    case ActivityPub.publish_one(params) do
+      {:ok, _} ->
+        :ok
+
+      {:error, _} ->
+        RetryQueue.enqueue(params, ActivityPub)
+    end
   end
 
-  def handle(:publish_single_websub, %{xml: xml, topic: topic, callback: callback, secret: secret}) do
-    signature = @websub.sign(secret || "", xml)
-    Logger.debug(fn -> "Pushing #{topic} to #{callback}" end)
+  def handle(
+        :publish_single_websub,
+        %{xml: xml, topic: topic, callback: callback, secret: secret} = params
+      ) do
+    case Websub.publish_one(params) do
+      {:ok, _} ->
+        :ok
 
-    with {:ok, %{status_code: code}} <-
-           @httpoison.post(
-             callback,
-             xml,
-             [
-               {"Content-Type", "application/atom+xml"},
-               {"X-Hub-Signature", "sha1=#{signature}"}
-             ],
-             timeout: 10000,
-             recv_timeout: 20000,
-             hackney: [pool: :default]
-           ) do
-      Logger.debug(fn -> "Pushed to #{callback}, code #{code}" end)
-    else
-      e ->
-        Logger.debug(fn -> "Couldn't push to #{callback}, #{inspect(e)}" end)
+      {:error, _} ->
+        RetryQueue.enqueue(params, Websub)
     end
   end
 
@@ -147,11 +150,15 @@ defmodule Pleroma.Web.Federator do
     {:error, "Don't know what to do with this"}
   end
 
-  def enqueue(type, payload, priority \\ 1) do
-    if Pleroma.Config.get([:instance, :federating]) do
-      if Mix.env() == :test do
+  if Mix.env() == :test do
+    def enqueue(type, payload, priority \\ 1) do
+      if Pleroma.Config.get([:instance, :federating]) do
         handle(type, payload)
-      else
+      end
+    end
+  else
+    def enqueue(type, payload, priority \\ 1) do
+      if Pleroma.Config.get([:instance, :federating]) do
         GenServer.cast(__MODULE__, {:enqueue, type, payload, priority})
       end
     end

lib/pleroma/web/federator/retry_queue.ex

@@ -0,0 +1,71 @@
+defmodule Pleroma.Web.Federator.RetryQueue do
+  use GenServer
+  alias Pleroma.Web.{WebFinger, Websub}
+  alias Pleroma.Web.ActivityPub.ActivityPub
+  require Logger
+
+  @websub Application.get_env(:pleroma, :websub)
+  @ostatus Application.get_env(:pleroma, :websub)
+  @httpoison Application.get_env(:pleroma, :websub)
+  @instance Application.get_env(:pleroma, :websub)
+  # initial timeout, 5 min
+  @initial_timeout 30_000
+  @max_retries 5
+
+  def init(args) do
+    {:ok, args}
+  end
+
+  def start_link() do
+    GenServer.start_link(__MODULE__, %{delivered: 0, dropped: 0}, name: __MODULE__)
+  end
+
+  def enqueue(data, transport, retries \\ 0) do
+    GenServer.cast(__MODULE__, {:maybe_enqueue, data, transport, retries + 1})
+  end
+
+  def get_retry_params(retries) do
+    if retries > @max_retries do
+      {:drop, "Max retries reached"}
+    else
+      {:retry, growth_function(retries)}
+    end
+  end
+
+  def handle_cast({:maybe_enqueue, data, transport, retries}, %{dropped: drop_count} = state) do
+    case get_retry_params(retries) do
+      {:retry, timeout} ->
+        Process.send_after(
+          __MODULE__,
+          {:send, data, transport, retries},
+          growth_function(retries)
+        )
+
+        {:noreply, state}
+
+      {:drop, message} ->
+        Logger.debug(message)
+        {:noreply, %{state | dropped: drop_count + 1}}
+    end
+  end
+
+  def handle_info({:send, data, transport, retries}, %{delivered: delivery_count} = state) do
+    case transport.publish_one(data) do
+      {:ok, _} ->
+        {:noreply, %{state | delivered: delivery_count + 1}}
+
+      {:error, reason} ->
+        enqueue(data, transport, retries)
+        {:noreply, state}
+    end
+  end
+
+  def handle_info(unknown, state) do
+    Logger.debug("RetryQueue: don't know what to do with #{inspect(unknown)}, ignoring")
+    {:noreply, state}
+  end
+
+  defp growth_function(retries) do
+    round(@initial_timeout * :math.pow(retries, 3))
+  end
+end

lib/pleroma/web/mastodon_api/mastodon_api_controller.ex

@@ -141,7 +141,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
       uri: Web.base_url(),
       title: Keyword.get(instance, :name),
       description: Keyword.get(instance, :description),
-      version: "#{@mastodon_api_level} (compatible; #{Keyword.get(instance, :version)})",
+      version: "#{@mastodon_api_level} (compatible; #{Pleroma.Application.named_version()})",
       email: Keyword.get(instance, :email),
       urls: %{
         streaming_api: String.replace(Pleroma.Web.Endpoint.static_url(), "http", "ws")
@@ -278,9 +278,12 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
     end
   end
 
-  def dm_timeline(%{assigns: %{user: user}} = conn, _params) do
+  def dm_timeline(%{assigns: %{user: user}} = conn, params) do
     query =
-      ActivityPub.fetch_activities_query([user.ap_id], %{"type" => "Create", visibility: "direct"})
+      ActivityPub.fetch_activities_query(
+        [user.ap_id],
+        Map.merge(params, %{"type" => "Create", visibility: "direct"})
+      )
 
     activities = Repo.all(query)
 

lib/pleroma/web/mastodon_api/mastodon_socket.ex

@@ -11,9 +11,8 @@ defmodule Pleroma.Web.MastodonAPI.MastodonSocket do
     timeout: :infinity
   )
 
-  def connect(params, socket) do
-    with token when not is_nil(token) <- params["access_token"],
-         %Token{user_id: user_id} <- Repo.get_by(Token, token: token),
+  def connect(%{"access_token" => token} = params, socket) do
+    with %Token{user_id: user_id} <- Repo.get_by(Token, token: token),
          %User{} = user <- Repo.get(User, user_id),
          stream
          when stream in [
@@ -45,6 +44,24 @@ defmodule Pleroma.Web.MastodonAPI.MastodonSocket do
     end
   end
 
+  def connect(%{"stream" => stream} = params, socket)
+      when stream in ["public", "public:local", "hashtag"] do
+    topic =
+      case stream do
+        "hashtag" -> "hashtag:#{params["tag"]}"
+        _ -> stream
+      end
+
+    with socket =
+           socket
+           |> assign(:topic, topic) do
+      Pleroma.Web.Streamer.add_socket(topic, socket)
+      {:ok, socket}
+    else
+      _e -> :error
+    end
+  end
+
   def id(_), do: nil
 
   def handle(:text, message, _state) do

lib/pleroma/web/media_proxy/controller.ex

@@ -11,15 +11,47 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do
     error: "public, must-revalidate, max-age=160"
   }
 
-  def remote(conn, %{"sig" => sig, "url" => url}) do
+  # Content-types that will not be returned as content-disposition attachments
+  # Override with :media_proxy, :safe_content_types in the configuration
+  @safe_content_types [
+    "image/gif",
+    "image/jpeg",
+    "image/jpg",
+    "image/png",
+    "image/svg+xml",
+    "audio/mpeg",
+    "audio/mp3",
+    "video/webm",
+    "video/mp4"
+  ]
+
+  def remote(conn, params = %{"sig" => sig, "url" => url}) do
     config = Application.get_env(:pleroma, :media_proxy, [])
 
     with true <- Keyword.get(config, :enabled, false),
          {:ok, url} <- Pleroma.Web.MediaProxy.decode_url(sig, url),
-         {:ok, content_type, body} <- proxy_request(url) do
+         filename <- Path.basename(URI.parse(url).path),
+         true <-
+           if(Map.get(params, "filename"),
+             do: filename == Path.basename(conn.request_path),
+             else: true
+           ),
+         {:ok, content_type, body} <- proxy_request(url),
+         safe_content_type <-
+           Enum.member?(
+             Keyword.get(config, :safe_content_types, @safe_content_types),
+             content_type
+           ) do
       conn
       |> put_resp_content_type(content_type)
       |> set_cache_header(:default)
+      |> put_resp_header(
+        "content-security-policy",
+        "default-src 'none'; style-src 'unsafe-inline'; media-src data:; img-src 'self' data:"
+      )
+      |> put_resp_header("x-xss-protection", "1; mode=block")
+      |> put_resp_header("x-content-type-options", "nosniff")
+      |> put_attachement_header(safe_content_type, filename)
       |> send_resp(200, body)
     else
       false ->
@@ -92,6 +124,12 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do
   # TODO: the body is passed here as well because some hosts do not provide a content-type.
   # At some point we may want to use magic numbers to discover the content-type and reply a proper one.
   defp proxy_request_content_type(headers, _body) do
-    headers["Content-Type"] || headers["content-type"] || "image/jpeg"
+    headers["Content-Type"] || headers["content-type"] || "application/octet-stream"
+  end
+
+  defp put_attachement_header(conn, true, _), do: conn
+
+  defp put_attachement_header(conn, false, filename) do
+    put_resp_header(conn, "content-disposition", "attachment; filename='#{filename}'")
   end
 end

lib/pleroma/web/media_proxy/media_proxy.ex

@@ -3,6 +3,8 @@ defmodule Pleroma.Web.MediaProxy do
 
   def url(nil), do: nil
 
+  def url(""), do: nil
+
   def url(url = "/" <> _), do: url
 
   def url(url) do
@@ -15,7 +17,10 @@ defmodule Pleroma.Web.MediaProxy do
       base64 = Base.url_encode64(url, @base64_opts)
       sig = :crypto.hmac(:sha, secret, base64)
       sig64 = sig |> Base.url_encode64(@base64_opts)
-      Keyword.get(config, :base_url, Pleroma.Web.base_url()) <> "/proxy/#{sig64}/#{base64}"
+      filename = if path = URI.parse(url).path, do: "/" <> Path.basename(path), else: ""
+
+      Keyword.get(config, :base_url, Pleroma.Web.base_url()) <>
+        "/proxy/#{sig64}/#{base64}#{filename}"
     end
   end
 

lib/pleroma/web/nodeinfo/nodeinfo_controller.ex

@@ -86,8 +86,8 @@ defmodule Pleroma.Web.Nodeinfo.NodeinfoController do
     response = %{
       version: "2.0",
       software: %{
-        name: "pleroma",
-        version: Keyword.get(instance, :version)
+        name: Pleroma.Application.name(),
+        version: Pleroma.Application.version()
       },
       protocols: ["ostatus", "activitypub"],
       services: %{

lib/pleroma/web/ostatus/ostatus_controller.ex

@@ -1,7 +1,7 @@
 defmodule Pleroma.Web.OStatus.OStatusController do
   use Pleroma.Web, :controller
 
-  alias Pleroma.{User, Activity}
+  alias Pleroma.{User, Activity, Object}
   alias Pleroma.Web.OStatus.{FeedRepresenter, ActivityRepresenter}
   alias Pleroma.Repo
   alias Pleroma.Web.{OStatus, Federator}
@@ -136,7 +136,7 @@ defmodule Pleroma.Web.OStatus.OStatusController do
         "html" ->
           conn
           |> put_resp_content_type("text/html")
-          |> send_file(200, "priv_sid/static/index.html")
+          |> send_file(200, Application.app_dir(:pleroma, "priv_sid/static/index.html"))
 
         _ ->
           represent_activity(conn, format, activity, user)
@@ -153,10 +153,21 @@ defmodule Pleroma.Web.OStatus.OStatusController do
     end
   end
 
-  defp represent_activity(conn, "activity+json", activity, user) do
+  defp represent_activity(
+         conn,
+         "activity+json",
+         %Activity{data: %{"type" => "Create"}} = activity,
+         user
+       ) do
+    object = Object.normalize(activity.data["object"])
+
     conn
     |> put_resp_header("content-type", "application/activity+json")
-    |> json(ObjectView.render("object.json", %{object: activity}))
+    |> json(ObjectView.render("object.json", %{object: object}))
+  end
+
+  defp represent_activity(conn, "activity+json", _, _) do
+    {:error, :not_found}
   end
 
   defp represent_activity(conn, _, activity, user) do

lib/pleroma/web/router.ex

@@ -31,6 +31,21 @@ defmodule Pleroma.Web.Router do
     plug(Pleroma.Plugs.EnsureAuthenticatedPlug)
   end
 
+  pipeline :admin_api do
+    plug(:accepts, ["json"])
+    plug(:fetch_session)
+    plug(Pleroma.Plugs.OAuthPlug)
+    plug(Pleroma.Plugs.BasicAuthDecoderPlug)
+    plug(Pleroma.Plugs.UserFetcherPlug)
+    plug(Pleroma.Plugs.SessionAuthenticationPlug)
+    plug(Pleroma.Plugs.LegacyAuthenticationPlug)
+    plug(Pleroma.Plugs.AuthenticationPlug)
+    plug(Pleroma.Plugs.UserEnabledPlug)
+    plug(Pleroma.Plugs.SetUserSessionIdPlug)
+    plug(Pleroma.Plugs.EnsureAuthenticatedPlug)
+    plug(Pleroma.Plugs.UserIsAdminPlug)
+  end
+
   pipeline :mastodon_html do
     plug(:accepts, ["html"])
     plug(:fetch_session)
@@ -79,6 +94,23 @@ defmodule Pleroma.Web.Router do
     get("/emoji", UtilController, :emoji)
   end
 
+  scope "/api/pleroma/admin", Pleroma.Web.AdminAPI do
+    pipe_through(:admin_api)
+    delete("/user", AdminAPIController, :user_delete)
+    post("/user", AdminAPIController, :user_create)
+
+    get("/permission_group/:nickname", AdminAPIController, :right_get)
+    get("/permission_group/:nickname/:permission_group", AdminAPIController, :right_get)
+    post("/permission_group/:nickname/:permission_group", AdminAPIController, :right_add)
+    delete("/permission_group/:nickname/:permission_group", AdminAPIController, :right_delete)
+
+    post("/relay", AdminAPIController, :relay_follow)
+    delete("/relay", AdminAPIController, :relay_unfollow)
+
+    get("/invite_token", AdminAPIController, :get_invite_token)
+    get("/password_reset", AdminAPIController, :get_password_reset)
+  end
+
   scope "/", Pleroma.Web.TwitterAPI do
     pipe_through(:pleroma_html)
     get("/ostatus_subscribe", UtilController, :remote_follow)
@@ -250,7 +282,12 @@ defmodule Pleroma.Web.Router do
     get("/statuses/networkpublic_timeline", TwitterAPI.Controller, :public_and_external_timeline)
   end
 
-  scope "/api", Pleroma.Web do
+  scope "/api", Pleroma.Web, as: :twitter_api_search do
+    pipe_through(:api)
+    get("/pleroma/search_user", TwitterAPI.Controller, :search_user)
+  end
+
+  scope "/api", Pleroma.Web, as: :authenticated_twitter_api do
     pipe_through(:authenticated_api)
 
     get("/account/verify_credentials", TwitterAPI.Controller, :verify_credentials)
@@ -270,6 +307,7 @@ defmodule Pleroma.Web.Router do
     get("/statuses/friends_timeline", TwitterAPI.Controller, :friends_timeline)
     get("/statuses/mentions", TwitterAPI.Controller, :mentions_timeline)
     get("/statuses/mentions_timeline", TwitterAPI.Controller, :mentions_timeline)
+    get("/statuses/dm_timeline", TwitterAPI.Controller, :dm_timeline)
     get("/qvitter/statuses/notifications", TwitterAPI.Controller, :notifications)
 
     # XXX: this is really a pleroma API, but we want to keep the pleroma namespace clean
@@ -378,12 +416,12 @@ defmodule Pleroma.Web.Router do
   end
 
   pipeline :remote_media do
-    plug(:accepts, ["html"])
   end
 
   scope "/proxy/", Pleroma.Web.MediaProxy do
     pipe_through(:remote_media)
     get("/:sig/:url", MediaProxyController, :remote)
+    get("/:sig/:url/:filename", MediaProxyController, :remote)
   end
 
   scope "/", Fallback do
@@ -398,11 +436,9 @@ defmodule Fallback.RedirectController do
   use Pleroma.Web, :controller
 
   def redirector(conn, _params) do
-    if Mix.env() != :test do
-      conn
-      |> put_resp_content_type("text/html")
-      |> send_file(200, "priv_sid/static/index.html")
-    end
+    conn
+    |> put_resp_content_type("text/html")
+    |> send_file(200, Application.app_dir(:pleroma, "priv_sid/static/index.html"))
   end
 
   def registration_page(conn, params) do

lib/pleroma/web/streamer.ex

@@ -169,16 +169,33 @@ defmodule Pleroma.Web.Streamer do
     |> Jason.encode!()
   end
 
+  defp represent_update(%Activity{} = activity) do
+    %{
+      event: "update",
+      payload:
+        Pleroma.Web.MastodonAPI.StatusView.render(
+          "status.json",
+          activity: activity
+        )
+        |> Jason.encode!()
+    }
+    |> Jason.encode!()
+  end
+
   def push_to_socket(topics, topic, %Activity{data: %{"type" => "Announce"}} = item) do
     Enum.each(topics[topic] || [], fn socket ->
       # Get the current user so we have up-to-date blocks etc.
-      user = User.get_cached_by_ap_id(socket.assigns[:user].ap_id)
-      blocks = user.info["blocks"] || []
+      if socket.assigns[:user] do
+        user = User.get_cached_by_ap_id(socket.assigns[:user].ap_id)
+        blocks = user.info["blocks"] || []
 
-      parent = Object.normalize(item.data["object"])
+        parent = Object.normalize(item.data["object"])
 
-      unless is_nil(parent) or item.actor in blocks or parent.data["actor"] in blocks do
-        send(socket.transport_pid, {:text, represent_update(item, user)})
+        unless is_nil(parent) or item.actor in blocks or parent.data["actor"] in blocks do
+          send(socket.transport_pid, {:text, represent_update(item, user)})
+        end
+      else
+        send(socket.transport_pid, {:text, represent_update(item)})
       end
     end)
   end
@@ -186,11 +203,15 @@ defmodule Pleroma.Web.Streamer do
   def push_to_socket(topics, topic, item) do
     Enum.each(topics[topic] || [], fn socket ->
       # Get the current user so we have up-to-date blocks etc.
-      user = User.get_cached_by_ap_id(socket.assigns[:user].ap_id)
-      blocks = user.info["blocks"] || []
+      if socket.assigns[:user] do
+        user = User.get_cached_by_ap_id(socket.assigns[:user].ap_id)
+        blocks = user.info["blocks"] || []
 
-      unless item.actor in blocks do
-        send(socket.transport_pid, {:text, represent_update(item, user)})
+        unless item.actor in blocks do
+          send(socket.transport_pid, {:text, represent_update(item, user)})
+        end
+      else
+        send(socket.transport_pid, {:text, represent_update(item)})
       end
     end)
   end

lib/pleroma/web/twitter_api/controllers/util_controller.ex

@@ -197,7 +197,7 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do
   end
 
   def version(conn, _params) do
-    version = Pleroma.Config.get([:instance, :version])
+    version = Pleroma.Application.named_version()
 
     case get_format(conn) do
       "xml" ->

lib/pleroma/web/twitter_api/twitter_api_controller.ex

@@ -126,6 +126,19 @@ defmodule Pleroma.Web.TwitterAPI.Controller do
     |> render(ActivityView, "index.json", %{activities: activities, for: user})
   end
 
+  def dm_timeline(%{assigns: %{user: user}} = conn, params) do
+    query =
+      ActivityPub.fetch_activities_query(
+        [user.ap_id],
+        Map.merge(params, %{"type" => "Create", "user" => user, visibility: "direct"})
+      )
+
+    activities = Repo.all(query)
+
+    conn
+    |> render(ActivityView, "index.json", %{activities: activities, for: user})
+  end
+
   def notifications(%{assigns: %{user: user}} = conn, params) do
     notifications = Notification.for_user(user, params)
 
@@ -516,6 +529,13 @@ defmodule Pleroma.Web.TwitterAPI.Controller do
     |> render(ActivityView, "index.json", %{activities: activities, for: user})
   end
 
+  def search_user(%{assigns: %{user: user}} = conn, %{"query" => query}) do
+    users = User.search(query, true)
+
+    conn
+    |> render(UserView, "index.json", %{users: users, for: user})
+  end
+
   defp bad_request_reply(conn, error_message) do
     json = error_json(conn, error_message)
     json_reply(conn, 400, json)

lib/pleroma/web/twitter_api/views/user_view.ex

@@ -55,8 +55,12 @@ defmodule Pleroma.Web.TwitterAPI.UserView do
       "statusnet_blocking" => statusnet_blocking,
       "friends_count" => user_info[:following_count],
       "id" => user.id,
-      "name" => user.name,
-      "name_html" => HTML.strip_tags(user.name) |> Formatter.emojify(emoji),
+      "name" => user.name || user.nickname,
+      "name_html" =>
+        if(user.name,
+          do: HTML.strip_tags(user.name) |> Formatter.emojify(emoji),
+          else: user.nickname
+        ),
       "profile_image_url" => image,
       "profile_image_url_https" => image,
       "profile_image_url_profile_size" => image,

lib/pleroma/web/websub/websub.ex

@@ -252,4 +252,29 @@ defmodule Pleroma.Web.Websub do
       Pleroma.Web.Federator.enqueue(:request_subscription, sub)
     end)
   end
+
+  def publish_one(%{xml: xml, topic: topic, callback: callback, secret: secret}) do
+    signature = sign(secret || "", xml)
+    Logger.info(fn -> "Pushing #{topic} to #{callback}" end)
+
+    with {:ok, %{status_code: code}} <-
+           @httpoison.post(
+             callback,
+             xml,
+             [
+               {"Content-Type", "application/atom+xml"},
+               {"X-Hub-Signature", "sha1=#{signature}"}
+             ],
+             timeout: 10000,
+             recv_timeout: 20000,
+             hackney: [pool: :default]
+           ) do
+      Logger.info(fn -> "Pushed to #{callback}, code #{code}" end)
+      {:ok, code}
+    else
+      e ->
+        Logger.debug(fn -> "Couldn't push to #{callback}, #{inspect(e)}" end)
+        {:error, e}
+    end
+  end
 end