Merge remote-tracking branch 'origin/develop' into shigusegubu

* origin/develop: (194 commits)
  Mix.Tasks.Pleroma.Instance: Generate signing_salt
  Send delete event over Mastodon streaming api
  Add a test to ensure #39 is fixed.
  update frontend
  Set custom similarity limit.
  Make use of the indices.
  test: add regression test for to/cc clobbering
  [#477] User trigram index adjustment.
  [#477] User: FTS and trigram search results mixing (to handle misspelled requests).
  [#491] Made full nicknames be preserved in user links text only in Bio.
  activitypub: add a match clause for objects, not just activities
  activitypub: transmogrifier: do not clobber the addressing on relayed announcements
  activitypub: allow is_public?() to work on any type of map representing an AS2 object
  activitypub: relay: chase selective public announce changes
  activitypub: announce: add new public parameter
  Add comments and change default path of the Mix binary.
  Fix bad link in likes collection
  [#502] Fixed `user_count` in `/api/v1/instance` to include only active local users.
  formatting
  Default to disabled in the code in case the setting is absent from config.exs
  ...

.gitlab-ci.yml

@@ -8,6 +8,7 @@ variables:
   POSTGRES_USER: postgres
   POSTGRES_PASSWORD: postgres
   DB_HOST: postgres
+  MIX_ENV: test
 
 cache:
   key: ${CI_COMMIT_REF_SLUG}
@@ -23,15 +24,15 @@ before_script:
   - mix local.rebar --force
   - mix deps.get
   - mix compile --force
-  - MIX_ENV=test mix ecto.create
+  - mix ecto.create
-  - MIX_ENV=test mix ecto.migrate
+  - mix ecto.migrate
 
 lint:
   stage: lint
   script:
-    - MIX_ENV=test mix format --check-formatted
+    - mix format --check-formatted
 
 unit-testing:
   stage: test
   script:
-    - MIX_ENV=test mix test --trace
+    - mix test --trace

README.md

@@ -30,7 +30,7 @@ While we don't provide docker files, other people have written very good ones. T
 ### Dependencies
 
 * Postgresql version 9.6 or newer
-* Elixir version 1.5 or newer. If your distribution only has an old version available, check [Elixir's install page](https://elixir-lang.org/install.html) or use a tool like [asdf](https://github.com/asdf-vm/asdf).
+* Elixir version 1.7 or newer. If your distribution only has an old version available, check [Elixir's install page](https://elixir-lang.org/install.html) or use a tool like [asdf](https://github.com/asdf-vm/asdf).
 * Build-essential tools
 
 ### Configuration

config/config.exs

@@ -12,7 +12,7 @@ config :pleroma, Pleroma.Repo, types: Pleroma.PostgresTypes
 
 config :pleroma, Pleroma.Captcha,
   enabled: false,
-  seconds_retained: 180,
+  seconds_valid: 60,
   method: Pleroma.Captcha.Kocaptcha
 
 config :pleroma, Pleroma.Captcha.Kocaptcha, endpoint: "https://captcha.kotobank.ch"
@@ -54,6 +54,17 @@ config :pleroma, :uri_schemes,
     "xmpp"
   ]
 
+websocket_config = [
+  path: "/websocket",
+  serializer: [
+    {Phoenix.Socket.V1.JSONSerializer, "~> 1.0.0"},
+    {Phoenix.Socket.V2.JSONSerializer, "~> 2.0.0"}
+  ],
+  timeout: 60_000,
+  transport_log: false,
+  compress: false
+]
+
 # Configures the endpoint
 config :pleroma, Pleroma.Web.Endpoint,
   url: [host: "localhost"],
@@ -62,6 +73,8 @@ config :pleroma, Pleroma.Web.Endpoint,
       {:_,
        [
          {"/api/v1/streaming", Elixir.Pleroma.Web.MastodonAPI.WebsocketHandler, []},
+         {"/socket/websocket", Phoenix.Endpoint.CowboyWebSocket,
+          {nil, {Pleroma.Web.Endpoint, Pleroma.Web.UserSocket, websocket_config}}},
          {:_, Plug.Adapters.Cowboy.Handler, {Pleroma.Web.Endpoint, []}}
        ]}
     ]
@@ -78,6 +91,12 @@ config :logger, :console,
   format: "$time $metadata[$level] $message\n",
   metadata: [:request_id]
 
+config :logger, :ex_syslogger,
+  level: :debug,
+  ident: "Pleroma",
+  format: "$date $time $metadata[$level] $message",
+  metadata: [:request_id]
+
 config :mime, :types, %{
   "application/xml" => ["xml"],
   "application/xrd+xml" => ["xrd+xml"],
@@ -98,7 +117,8 @@ config :pleroma, :instance,
   name: "Shigusegubu",
   email: "pleroma@hjkos.com",
   description: "SigSegV, a pleroma instance",
-  limit: 5000,
+  limit: 5_000,
+  remote_limit: 100_000,
   upload_limit: 20_000_000,
   avatar_upload_limit: 2_000_000,
   background_upload_limit: 4_000_000,
@@ -117,7 +137,10 @@ config :pleroma, :instance,
     "text/markdown"
   ],
   finmoji_enabled: true,
-  mrf_transparency: true
+  mrf_transparency: true,
+  autofollowed_nicknames: [],
+  max_pinned_statuses: 1,
+  no_attachment_links: false
 
 config :pleroma, :markup,
   # XXX - unfortunately, inline images must be enabled by default right now, because
@@ -135,10 +158,10 @@ config :pleroma, :fe,
   theme: "sigsegv2",
   logo: "/static/logo.svg",
   background: "/static/sigsegv_s.png",
+  redirect_root_no_login: "/main/all",
+  redirect_root_login: "/main/friends",
   logo_mask: true,
   logo_margin: "0.1em",
-  redirect_root_no_login: "/~/main/all",
-  redirect_root_login: "/~/main/friends",
   show_instance_panel: true,
   scope_options_enabled: false,
   formatting_options_enabled: false,
@@ -172,13 +195,7 @@ config :pleroma, :mrf_simple,
   reject: [],
   accept: []
 
-config :pleroma, :media_proxy,
+config :pleroma, :media_proxy, enabled: false
-  enabled: false,
-  # base_url: "https://cache.pleroma.social",
-  proxy_opts: [
-    # inline_content_types: [] | false | true,
-    # http: [:insecure]
-  ]
 
 config :pleroma, :chat, enabled: false
 
@@ -220,6 +237,46 @@ config :cors_plug,
   credentials: true,
   headers: ["Authorization", "Content-Type", "Idempotency-Key"]
 
+config :pleroma, Pleroma.User,
+  restricted_nicknames: [
+    ".well-known",
+    "~",
+    "about",
+    "activities",
+    "api",
+    "auth",
+    "dev",
+    "friend-requests",
+    "inbox",
+    "internal",
+    "main",
+    "media",
+    "nodeinfo",
+    "notice",
+    "oauth",
+    "objects",
+    "ostatus_subscribe",
+    "pleroma",
+    "proxy",
+    "push",
+    "registration",
+    "relay",
+    "settings",
+    "status",
+    "tag",
+    "user-search",
+    "users",
+    "web"
+  ]
+
+config :pleroma, Pleroma.Web.Federator, max_jobs: 50
+
+config :pleroma, Pleroma.Web.Federator.RetryQueue,
+  enabled: false,
+  max_jobs: 20,
+  initial_timeout: 30,
+  max_retries: 5
+
 # Import environment specific config. This must remain at the bottom
 # of this file so it overrides the configuration defined above.
 import_config "#{Mix.env()}.exs"

config/test.exs

@@ -9,7 +9,8 @@ config :pleroma, Pleroma.Web.Endpoint,
 
 # Disable captha for tests
 config :pleroma, Pleroma.Captcha,
-  enabled: true,
+  # It should not be enabled for automatic tests
+  enabled: false,
   # A fake captcha service for tests
   method: Pleroma.Captcha.Mock
 

docs/Admin-API.md

@@ -0,0 +1,100 @@
+# Admin API
+Authentication is required and the user must be an admin.
+
+## `/api/pleroma/admin/user`
+### Remove a user
+* Method `DELETE`
+* Params:
+    * `nickname`
+* Response: User’s nickname
+### Create a user
+* Method: `POST`
+* Params:
+    * `nickname`
+    * `email`
+    * `password`
+* Response: User’s nickname
+
+## `/api/pleroma/admin/users/tag`
+### Tag a list of users
+* Method: `PUT`
+* Params:
+    * `nickname`
+    * `tags`
+### Untag a list of users
+* Method: `DELETE`
+* Params:
+    * `nickname`
+    * `tags`
+
+## `/api/pleroma/admin/permission_group/:nickname`
+### Get user user permission groups membership
+* Method: `GET`
+* Params: none
+* Response:
+```JSON
+{
+	"is_moderator": bool,
+	"is_admin": bool
+}
+```
+
+## `/api/pleroma/admin/permission_group/:nickname/:permission_group`
+Note: Available `:permission_group` is currently moderator and admin. 404 is returned when the permission group doesn’t exist.
+
+### Get user user permission groups membership
+* Method: `GET`
+* Params: none
+* Response:
+```JSON
+{
+	"is_moderator": bool,
+	"is_admin": bool
+}
+```
+### Add user in permission group
+* Method: `POST`
+* Params: none
+* Response:
+    * On failure: ``{"error": "…"}``
+    * On success: JSON of the ``user.info``
+### Remove user from permission group
+* Method: `DELETE`
+* Params: none
+* Response:
+    * On failure: ``{"error": "…"}``
+    * On success: JSON of the ``user.info``
+* Note: An admin cannot revoke their own admin status.
+
+## `/api/pleroma/admin/relay`
+### Follow a Relay
+* Methods: `POST`
+* Params:
+    * `relay_url`
+* Response:
+    * On success: URL of the followed relay
+### Unfollow a Relay
+* Methods: `DELETE`
+* Params:
+    * `relay_url`
+* Response:
+    * On success: URL of the unfollowed relay
+
+## `/api/pleroma/admin/invite_token`
+### Get a account registeration invite token
+* Methods: `GET`
+* Params: none
+* Response: invite token (base64 string)
+
+## `/api/pleroma/admin/email_invite`
+### Sends registration invite via email
+* Methods: `POST`
+* Params:
+    * `email`
+    * `name`, optionnal
+
+## `/api/pleroma/admin/password_reset`
+### Get a password reset token for a given nickname
+* Methods: `GET`
+* Params: none
+* Response: password reset token (base64 string)

docs/Pleroma-API.md

@@ -92,4 +92,7 @@ Request parameters can be passed via [query strings](https://en.wikipedia.org/wi
 	"statusnet_blocking": false,
 	"statusnet_profile_url": "https://pleroma.soykaf.com/users/lain"
 }
-```
+```
+
+## `/api/pleroma/admin/`…
+See [Admin-API](Admin-API.md)

docs/config.md

@@ -32,7 +32,7 @@ This filter replaces the filename (not the path) of an upload. For complete obfu
 
 ## Pleroma.Mailer
 * `adapter`: one of the mail adapters listed in [Swoosh readme](https://github.com/swoosh/swoosh#adapters), or `Swoosh.Adapters.Local` for in-memory mailbox.
-* `api_key` / `password` and / or other adapter-specific settings, per the above documentation. 
+* `api_key` / `password` and / or other adapter-specific settings, per the above documentation.
 
 An example for Sendgrid adapter:
 
@@ -63,6 +63,7 @@ config :pleroma, Pleroma.Mailer,
 * `email`: Email used to reach an Administrator/Moderator of the instance
 * `description`: The instance’s description, can be seen in nodeinfo and ``/api/v1/instance``
 * `limit`: Posts character limit (CW/Subject included in the counter)
+* `remote_limit`: Hard character limit beyond which remote posts will be dropped.
 * `upload_limit`: File size limit of uploads (except for avatar, background, banner)
 * `avatar_upload_limit`: File size limit of user’s profile avatars
 * `background_upload_limit`: File size limit of user’s profile backgrounds
@@ -92,6 +93,13 @@ config :pleroma, Pleroma.Mailer,
 * `always_show_subject_input`: When set to false, auto-hide the subject field when it's empty.
 * `extended_nickname_format`: Set to `true` to use extended local nicknames format (allows underscores/dashes). This will break federation with
     older software for theses nicknames.
+* `max_pinned_statuses`: The maximum number of pinned statuses. `0` will disable the feature.
+* `autofollowed_nicknames`: Set to nicknames of (local) users that every new user should automatically follow.
+* `no_attachment_links`: Set to true to disable automatically adding attachment link text to statuses
+
+## :logger
+* `backends`: `:console` is used to send logs to stdout, `{ExSyslogger, :ex_syslogger}` to log to syslog
+See: [logger’s documentation](https://hexdocs.pm/logger/Logger.html) and [ex_syslogger’s documentation](https://hexdocs.pm/ex_syslogger/)
 
 ## :fe
 This section is used to configure Pleroma-FE, unless ``:managed_config`` in ``:instance`` is set to false.
@@ -171,7 +179,7 @@ Web Push Notifications configuration. You can use the mix task `mix web_push.gen
 ## Pleroma.Captcha
 * `enabled`: Whether the captcha should be shown on registration
 * `method`: The method/service to use for captcha
-* `seconds_retained`: The time in seconds for which the captcha is valid (stored in the cache)
+* `seconds_valid`: The time in seconds for which the captcha is valid
 
 ### Pleroma.Captcha.Kocaptcha
 Kocaptcha is a very simple captcha service with a single API endpoint,
@@ -192,3 +200,14 @@ You can then do
 ```
 curl "http://localhost:4000/api/pleroma/admin/invite_token?admin_token=somerandomtoken"
 ```
+
+## Pleroma.Web.Federator
+
+* `max_jobs`: The maximum amount of parallel federation jobs running at the same time.
+
+## Pleroma.Web.Federator.RetryQueue
+
+* `enabled`: If set to `true`, failed federation jobs will be retried
+* `max_jobs`: The maximum amount of parallel federation jobs running at the same time.
+* `initial_timeout`: The initial timeout in seconds
+* `max_retries`: The maximum number of times a federation job is retried

installation/pleroma.nginx

@@ -79,8 +79,10 @@ server {
         proxy_cache_valid  200 206 301 304 1h;
         proxy_cache_lock on;
         proxy_ignore_client_abort on;
-        proxy_buffering off;
+        proxy_buffering on;
         chunked_transfer_encoding on;
+        proxy_ignore_headers Cache-Control;
+        proxy_hide_header Cache-Control;
         proxy_pass http://localhost:4000;
     }
 }

installation/pleroma.service

@@ -3,16 +3,24 @@ Description=Pleroma social network
 After=network.target postgresql.service
 
 [Service]
-User=pleroma
-WorkingDirectory=/home/pleroma/pleroma
-Environment="HOME=/home/pleroma"
-Environment="MIX_ENV=prod"
-ExecStart=/usr/local/bin/mix phx.server
 ExecReload=/bin/kill $MAINPID
 KillMode=process
 Restart=on-failure
 StandardOutput=journal
 
+; Name of the user that runs the Pleroma service.
+User=pleroma
+; Declares that Pleroma runs in production mode.
+Environment="MIX_ENV=prod"
+
+; Make sure that all paths fit your installation.
+; Path to the home directory of the user running the Pleroma service.
+Environment="HOME=/home/pleroma"
+; Path to the folder containing the Pleroma installation.
+WorkingDirectory=/home/pleroma/pleroma
+; Path to the Mix binary.
+ExecStart=/usr/bin/mix phx.server
+
 ; Some security directives.
 ; Use private /tmp and /var/tmp folders inside a new file system namespace, which are discarded after the process stops.
 PrivateTmp=true
@@ -22,6 +30,8 @@ ProtectSystem=full
 PrivateDevices=false
 ; Ensures that the service process and all its children can never gain new privileges through execve().
 NoNewPrivileges=true
+; Drops the sysadmin capability from the daemon.
+CapabilityBoundingSet=~CAP_SYS_ADMIN
 
 [Install]
 WantedBy=multi-user.target

installation/pleroma.vcl

@@ -14,43 +14,45 @@ acl purge {
 sub vcl_recv {
     # Redirect HTTP to HTTPS
     if (std.port(server.ip) != 443) {
-        set req.http.x-redir = "https://" + req.http.host + req.url;
+      set req.http.x-redir = "https://" + req.http.host + req.url;
-        return (synth(750, ""));
+      return (synth(750, ""));
+    }
+
+    # CHUNKED SUPPORT
+    if (req.http.Range ~ "bytes=") {
+      set req.http.x-range = req.http.Range;
     }
 
     # Pipe if WebSockets request is coming through
     if (req.http.upgrade ~ "(?i)websocket") {
-        return (pipe);
+      return (pipe);
     }
 
     # Allow purging of the cache
     if (req.method == "PURGE") {
-        if (!client.ip ~ purge) {
+      if (!client.ip ~ purge) {
-          return(synth(405,"Not allowed."));
+        return(synth(405,"Not allowed."));
-        }
+      }
-        return(purge);
+      return(purge);
     }
 
     # Pleroma MediaProxy - strip headers that will affect caching
     if (req.url ~ "^/proxy/") {
-        unset req.http.Cookie;
+      unset req.http.Cookie;
-        unset req.http.Authorization;
+      unset req.http.Authorization;
-        unset req.http.Accept;
+      unset req.http.Accept;
-        return (hash);
+      return (hash);
     }
 
     # Strip headers that will affect caching from all other static content
     # This also permits caching of individual toots and AP Activities
     if ((req.url ~ "^/(media|static)/") ||
-    (req.url ~ "(?i)\.(html|js|css|jpg|jpeg|png|gif|gz|tgz|bz2|tbz|mp3|ogg|svg|swf|ttf|pdf|woff|woff2)$"))
+    (req.url ~ "(?i)\.(html|js|css|jpg|jpeg|png|gif|gz|tgz|bz2|tbz|mp3|mp4|ogg|webm|svg|swf|ttf|pdf|woff|woff2)$"))
     {
       unset req.http.Cookie;
       unset req.http.Authorization;
       return (hash);
     }
-
-    # Everything else should just be piped to Pleroma
-    return (pipe);
 }
 
 sub vcl_backend_response {
@@ -59,8 +61,11 @@ sub vcl_backend_response {
       set beresp.do_gzip = true;
     }
 
-    # etags are bad
+    # CHUNKED SUPPORT
-    unset beresp.http.etag;
+    if (bereq.http.x-range ~ "bytes=" && beresp.status == 206) {
+      set beresp.ttl = 10m;
+      set beresp.http.CR = beresp.http.content-range;
+    }
 
     # Don't cache objects that require authentication
     if (beresp.http.Authorization && !beresp.http.Cache-Control ~ "public") {
@@ -81,9 +86,9 @@ sub vcl_backend_response {
 
     # Do not cache redirects and errors
     if ((beresp.status >= 300) && (beresp.status < 500)) {
-        set beresp.uncacheable = true;
+      set beresp.uncacheable = true;
-        set beresp.ttl = 30s;
+      set beresp.ttl = 30s;
-        return (deliver);
+      return (deliver);
     }
 
     # Pleroma MediaProxy internally sets headers properly
@@ -92,14 +97,12 @@ sub vcl_backend_response {
     }
 
     # Strip cache-restricting headers from Pleroma on static content that we want to cache
-    # Also enable streaming of cached content to clients (no waiting for Varnish to complete backend fetch)
+    if (bereq.url ~ "(?i)\.(js|css|jpg|jpeg|png|gif|gz|tgz|bz2|tbz|mp3|mp4|ogg|webm|svg|swf|ttf|pdf|woff|woff2)$")
-    if (bereq.url ~ "(?i)\.(js|css|jpg|jpeg|png|gif|gz|tgz|bz2|tbz|mp3|ogg|svg|swf|ttf|pdf|woff|woff2)$")
     {
       unset beresp.http.set-cookie;
       unset beresp.http.Cache-Control;
       unset beresp.http.x-request-id;
       set beresp.http.Cache-Control = "public, max-age=86400";
-      set beresp.do_stream = true;
     }
 }
 
@@ -115,7 +118,30 @@ sub vcl_synth {
 # Ensure WebSockets through the pipe do not close prematurely
 sub vcl_pipe {
     if (req.http.upgrade) {
-        set bereq.http.upgrade = req.http.upgrade;
+      set bereq.http.upgrade = req.http.upgrade;
-        set bereq.http.connection = req.http.connection;
+      set bereq.http.connection = req.http.connection;
+    }
+}
+
+sub vcl_hash {
+    # CHUNKED SUPPORT
+    if (req.http.x-range ~ "bytes=") {
+      hash_data(req.http.x-range);
+      unset req.http.Range;
+    }
+}
+
+sub vcl_backend_fetch {
+    # CHUNKED SUPPORT
+    if (bereq.http.x-range) {
+      set bereq.http.Range = bereq.http.x-range;
+    }
+}
+
+sub vcl_deliver {
+    # CHUNKED SUPPORT
+    if (resp.http.CR) {
+      set resp.http.Content-Range = resp.http.CR;
+      unset resp.http.CR;
     }
 }

lib/mix/tasks/pleroma/instance.ex

@@ -75,7 +75,7 @@ defmodule Mix.Tasks.Pleroma.Instance do
       name =
         Common.get_option(
           options,
-          :name,
+          :instance_name,
           "What is the name of your instance? (e.g. Pleroma/Soykaf)"
         )
 
@@ -105,6 +105,7 @@ defmodule Mix.Tasks.Pleroma.Instance do
         )
 
       secret = :crypto.strong_rand_bytes(64) |> Base.encode64() |> binary_part(0, 64)
+      signing_salt = :crypto.strong_rand_bytes(8) |> Base.encode64() |> binary_part(0, 8)
       {web_push_public_key, web_push_private_key} = :crypto.generate_key(:ecdh, :prime256v1)
 
       result_config =
@@ -120,6 +121,7 @@ defmodule Mix.Tasks.Pleroma.Instance do
           dbpass: dbpass,
           version: Pleroma.Mixfile.project() |> Keyword.get(:version),
           secret: secret,
+          signing_salt: signing_salt,
           web_push_public_key: Base.url_encode64(web_push_public_key, padding: false),
           web_push_private_key: Base.url_encode64(web_push_private_key, padding: false)
         )

lib/mix/tasks/pleroma/sample_config.eex

@@ -7,7 +7,8 @@ use Mix.Config
 
 config :pleroma, Pleroma.Web.Endpoint,
    url: [host: "<%= domain %>", scheme: "https", port: <%= port %>],
-   secret_key_base: "<%= secret %>"
+   secret_key_base: "<%= secret %>",
+   signing_salt: "<%= signing_salt %>"
 
 config :pleroma, :instance,
   name: "<%= name %>",

lib/mix/tasks/pleroma/user.ex

@@ -22,6 +22,7 @@ defmodule Mix.Tasks.Pleroma.User do
   - `--password PASSWORD` - the user's password
   - `--moderator`/`--no-moderator` - whether the user is a moderator
   - `--admin`/`--no-admin` - whether the user is an admin
+  - `-y`, `--assume-yes`/`--no-assume-yes` - whether to assume yes to all questions 
 
   ## Generate an invite link.
 
@@ -61,7 +62,11 @@ defmodule Mix.Tasks.Pleroma.User do
           bio: :string,
           password: :string,
           moderator: :boolean,
-          admin: :boolean
+          admin: :boolean,
+          assume_yes: :boolean
+        ],
+        aliases: [
+          y: :assume_yes
         ]
       )
 
@@ -79,6 +84,7 @@ defmodule Mix.Tasks.Pleroma.User do
 
     moderator? = Keyword.get(options, :moderator, false)
     admin? = Keyword.get(options, :admin, false)
+    assume_yes? = Keyword.get(options, :assume_yes, false)
 
     Mix.shell().info("""
     A user will be created with the following information:
@@ -93,7 +99,7 @@ defmodule Mix.Tasks.Pleroma.User do
       - admin: #{if(admin?, do: "true", else: "false")}
     """)
 
-    proceed? = Mix.shell().yes?("Continue?")
+    proceed? = assume_yes? or Mix.shell().yes?("Continue?")
 
     unless not proceed? do
       Common.start_pleroma()

lib/pleroma/PasswordResetToken.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.PasswordResetToken do

lib/pleroma/activity.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Activity do
@@ -7,6 +7,8 @@ defmodule Pleroma.Activity do
   alias Pleroma.{Repo, Activity, Notification}
   import Ecto.Query
 
+  @type t :: %__MODULE__{}
+
   # https://github.com/tootsuite/mastodon/blob/master/app/models/notification.rb#L19
   @mastodon_notification_types %{
     "Create" => "mention",

lib/pleroma/application.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Application do
@@ -29,6 +29,16 @@ defmodule Pleroma.Application do
         supervisor(Pleroma.Repo, []),
         worker(Pleroma.Emoji, []),
         worker(Pleroma.Captcha, []),
+        worker(
+          Cachex,
+          [
+            :used_captcha_cache,
+            [
+              ttl_interval: :timer.seconds(Pleroma.Config.get!([Pleroma.Captcha, :seconds_valid]))
+            ]
+          ],
+          id: :cachex_used_captcha_cache
+        ),
         worker(
           Cachex,
           [
@@ -53,6 +63,27 @@ defmodule Pleroma.Application do
           ],
           id: :cachex_object
         ),
+        worker(
+          Cachex,
+          [
+            :rich_media_cache,
+            [
+              default_ttl: :timer.minutes(120),
+              limit: 5000
+            ]
+          ],
+          id: :cachex_rich_media
+        ),
+        worker(
+          Cachex,
+          [
+            :scrubber_cache,
+            [
+              limit: 2500
+            ]
+          ],
+          id: :cachex_scrubber
+        ),
         worker(
           Cachex,
           [

lib/pleroma/captcha/captcha.ex

@@ -1,11 +1,13 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Captcha do
-  use GenServer
+  alias Plug.Crypto.KeyGenerator
+  alias Plug.Crypto.MessageEncryptor
+  alias Calendar.DateTime
 
-  @ets_options [:ordered_set, :private, :named_table, {:read_concurrency, true}]
+  use GenServer
 
   @doc false
   def start_link() do
@@ -14,14 +16,6 @@ defmodule Pleroma.Captcha do
 
   @doc false
   def init(_) do
-    # Create a ETS table to store captchas
-    ets_name = Module.concat(method(), Ets)
-    ^ets_name = :ets.new(Module.concat(method(), Ets), @ets_options)
-
-    # Clean up old captchas every few minutes
-    seconds_retained = Pleroma.Config.get!([__MODULE__, :seconds_retained])
-    Process.send_after(self(), :cleanup, 1000 * seconds_retained)
-
     {:ok, nil}
   end
 
@@ -35,8 +29,8 @@ defmodule Pleroma.Captcha do
   @doc """
   Ask the configured captcha service to validate the captcha
   """
-  def validate(token, captcha) do
+  def validate(token, captcha, answer_data) do
-    GenServer.call(__MODULE__, {:validate, token, captcha})
+    GenServer.call(__MODULE__, {:validate, token, captcha, answer_data})
   end
 
   @doc false
@@ -46,24 +40,71 @@ defmodule Pleroma.Captcha do
     if !enabled do
       {:reply, %{type: :none}, state}
     else
-      {:reply, method().new(), state}
+      new_captcha = method().new()
+
+      secret_key_base = Pleroma.Config.get!([Pleroma.Web.Endpoint, :secret_key_base])
+
+      # This make salt a little different for two keys
+      token = new_captcha[:token]
+      secret = KeyGenerator.generate(secret_key_base, token <> "_encrypt")
+      sign_secret = KeyGenerator.generate(secret_key_base, token <> "_sign")
+      # Basicallty copy what Phoenix.Token does here, add the time to
+      # the actual data and make it a binary to then encrypt it
+      encrypted_captcha_answer =
+        %{
+          at: DateTime.now_utc(),
+          answer_data: new_captcha[:answer_data]
+        }
+        |> :erlang.term_to_binary()
+        |> MessageEncryptor.encrypt(secret, sign_secret)
+
+      {
+        :reply,
+        # Repalce the answer with the encrypted answer
+        %{new_captcha | answer_data: encrypted_captcha_answer},
+        state
+      }
     end
   end
 
   @doc false
-  def handle_call({:validate, token, captcha}, _from, state) do
+  def handle_call({:validate, token, captcha, answer_data}, _from, state) do
-    {:reply, method().validate(token, captcha), state}
+    secret_key_base = Pleroma.Config.get!([Pleroma.Web.Endpoint, :secret_key_base])
-  end
+    secret = KeyGenerator.generate(secret_key_base, token <> "_encrypt")
+    sign_secret = KeyGenerator.generate(secret_key_base, token <> "_sign")
 
-  @doc false
+    # If the time found is less than (current_time - seconds_valid), then the time has already passed.
-  def handle_info(:cleanup, state) do
+    # Later we check that the time found is more than the presumed invalidatation time, that means
-    :ok = method().cleanup()
+    # that the data is still valid and the captcha can be checked
+    seconds_valid = Pleroma.Config.get!([Pleroma.Captcha, :seconds_valid])
+    valid_if_after = DateTime.subtract!(DateTime.now_utc(), seconds_valid)
 
-    seconds_retained = Pleroma.Config.get!([__MODULE__, :seconds_retained])
+    result =
-    # Schedule the next clenup
+      with {:ok, data} <- MessageEncryptor.decrypt(answer_data, secret, sign_secret),
-    Process.send_after(self(), :cleanup, 1000 * seconds_retained)
+           %{at: at, answer_data: answer_md5} <- :erlang.binary_to_term(data) do
+        try do
+          if DateTime.before?(at, valid_if_after), do: throw({:error, "CAPTCHA expired"})
 
-    {:noreply, state}
+          if not is_nil(Cachex.get!(:used_captcha_cache, token)),
+            do: throw({:error, "CAPTCHA already used"})
+
+          res = method().validate(token, captcha, answer_md5)
+          # Throw if an error occurs
+          if res != :ok, do: throw(res)
+
+          # Mark this captcha as used
+          {:ok, _} =
+            Cachex.put(:used_captcha_cache, token, true, ttl: :timer.seconds(seconds_valid))
+
+          :ok
+        catch
+          :throw, e -> e
+        end
+      else
+        _ -> {:error, "Invalid answer data"}
+      end
+
+    {:reply, result, state}
   end
 
   defp method, do: Pleroma.Config.get!([__MODULE__, :method])

lib/pleroma/captcha/captcha_service.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Captcha.Service do
@@ -8,9 +8,14 @@ defmodule Pleroma.Captcha.Service do
 
   Returns:
 
-  Service-specific data for using the newly created captcha
+  Type/Name of the service, the token to identify the captcha,
+  the data of the answer and service-specific data to use the newly created captcha
   """
-  @callback new() :: map
+  @callback new() :: %{
+              type: atom(),
+              token: String.t(),
+              answer_data: any()
+            }
 
   @doc """
   Validated the provided captcha solution.
@@ -18,15 +23,15 @@ defmodule Pleroma.Captcha.Service do
   Arguments:
   * `token` the captcha is associated with
   * `captcha` solution of the captcha to validate
+  * `answer_data` is the data needed to validate the answer (presumably encrypted)
 
   Returns:
 
   `true` if captcha is valid, `false` if not
   """
-  @callback validate(token :: String.t(), captcha :: String.t()) :: boolean
+  @callback validate(
-
+              token :: String.t(),
-  @doc """
+              captcha :: String.t(),
-  This function is called periodically to clean up old captchas
+              answer_data :: any()
-  """
+            ) :: :ok | {:error, String.t()}
-  @callback cleanup() :: :ok
 end

lib/pleroma/captcha/kocaptcha.ex

@@ -1,15 +1,11 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Captcha.Kocaptcha do
-  alias Calendar.DateTime
-
   alias Pleroma.Captcha.Service
   @behaviour Service
 
-  @ets __MODULE__.Ets
-
   @impl Service
   def new() do
     endpoint = Pleroma.Config.get!([__MODULE__, :endpoint])
@@ -21,51 +17,21 @@ defmodule Pleroma.Captcha.Kocaptcha do
       {:ok, res} ->
         json_resp = Poison.decode!(res.body)
 
-        token = json_resp["token"]
+        %{
-
+          type: :kocaptcha,
-        true =
+          token: json_resp["token"],
-          :ets.insert(
+          url: endpoint <> json_resp["url"],
-            @ets,
+          answer_data: json_resp["md5"]
-            {token, json_resp["md5"], DateTime.now_utc() |> DateTime.Format.unix()}
-          )
-
-        %{type: :kocaptcha, token: token, url: endpoint <> json_resp["url"]}
-    end
-  end
-
-  @impl Service
-  def validate(token, captcha) do
-    with false <- is_nil(captcha),
-         [{^token, saved_md5, _}] <- :ets.lookup(@ets, token),
-         true <- :crypto.hash(:md5, captcha) |> Base.encode16() == String.upcase(saved_md5) do
-      # Clear the saved value
-      :ets.delete(@ets, token)
-
-      true
-    else
-      _ -> false
-    end
-  end
-
-  @impl Service
-  def cleanup() do
-    seconds_retained = Pleroma.Config.get!([Pleroma.Captcha, :seconds_retained])
-    # If the time in ETS is less than current_time - seconds_retained, then the time has
-    # already passed
-    delete_after =
-      DateTime.subtract!(DateTime.now_utc(), seconds_retained) |> DateTime.Format.unix()
-
-    :ets.select_delete(
-      @ets,
-      [
-        {
-          {:_, :_, :"$1"},
-          [{:<, :"$1", {:const, delete_after}}],
-          [true]
         }
-      ]
+    end
-    )
+  end
 
-    :ok
+  @impl Service
+  def validate(_token, captcha, answer_data) do
+    # Here the token is unsed, because the unencrypted captcha answer is just passed to method
+    if not is_nil(captcha) and
+         :crypto.hash(:md5, captcha) |> Base.encode16() == String.upcase(answer_data),
+       do: :ok,
+       else: {:error, "Invalid CAPTCHA"}
   end
 end

lib/pleroma/config.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Config do

lib/pleroma/emails/mailer.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Mailer do

lib/pleroma/emails/user_email.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.UserEmail do

lib/pleroma/emoji.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Emoji do

lib/pleroma/filter.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Filter do

lib/pleroma/formatter.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Formatter do
@@ -120,7 +120,7 @@ defmodule Pleroma.Formatter do
   end
 
   @doc "Adds the links to mentioned users"
-  def add_user_links({subs, text}, mentions) do
+  def add_user_links({subs, text}, mentions, options \\ []) do
     mentions =
       mentions
       |> Enum.sort_by(fn {name, _} -> -String.length(name) end)
@@ -142,10 +142,16 @@ defmodule Pleroma.Formatter do
               ap_id
             end
 
-          short_match = String.split(match, "@") |> tl() |> hd()
+          nickname =
+            if options[:format] == :full do
+              User.full_nickname(match)
+            else
+              User.local_nickname(match)
+            end
 
           {uuid,
-           "<span><a data-user='#{id}' class='mention' href='#{ap_id}'>@<span>#{short_match}</span></a></span>"}
+           "<span class='h-card'><a data-user='#{id}' class='u-url mention' href='#{ap_id}'>" <>
+             "@<span>#{nickname}</span></a></span>"}
         end)
 
     {subs, uuid_text}
@@ -168,7 +174,7 @@ defmodule Pleroma.Formatter do
       subs ++
         Enum.map(tags, fn {tag_text, tag, uuid} ->
           url =
-            "<a data-tag='#{tag}' href='#{Pleroma.Web.base_url()}/tag/#{tag}' rel='tag'>#{
+            "<a class='hashtag' data-tag='#{tag}' href='#{Pleroma.Web.base_url()}/tag/#{tag}' rel='tag'>#{
               tag_text
             }</a>"
 

lib/pleroma/gopher/server.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Gopher.Server do

lib/pleroma/html.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.HTML do
@@ -15,8 +15,11 @@ defmodule Pleroma.HTML do
   end
 
   def filter_tags(html, nil) do
-    get_scrubbers()
+    filter_tags(html, get_scrubbers())
-    |> Enum.reduce(html, fn scrubber, html ->
+  end
+
+  def filter_tags(html, scrubbers) when is_list(scrubbers) do
+    Enum.reduce(scrubbers, html, fn scrubber, html ->
       filter_tags(html, scrubber)
     end)
   end
@@ -24,6 +27,37 @@ defmodule Pleroma.HTML do
   def filter_tags(html, scrubber), do: Scrubber.scrub(html, scrubber)
   def filter_tags(html), do: filter_tags(html, nil)
   def strip_tags(html), do: Scrubber.scrub(html, Scrubber.StripTags)
+
+  def get_cached_scrubbed_html_for_object(content, scrubbers, object, module) do
+    key = "#{module}#{generate_scrubber_signature(scrubbers)}|#{object.id}"
+    Cachex.fetch!(:scrubber_cache, key, fn _key -> ensure_scrubbed_html(content, scrubbers) end)
+  end
+
+  def get_cached_stripped_html_for_object(content, object, module) do
+    get_cached_scrubbed_html_for_object(
+      content,
+      HtmlSanitizeEx.Scrubber.StripTags,
+      object,
+      module
+    )
+  end
+
+  def ensure_scrubbed_html(
+        content,
+        scrubbers
+      ) do
+    {:commit, filter_tags(content, scrubbers)}
+  end
+
+  defp generate_scrubber_signature(scrubber) when is_atom(scrubber) do
+    generate_scrubber_signature([scrubber])
+  end
+
+  defp generate_scrubber_signature(scrubbers) do
+    Enum.reduce(scrubbers, "", fn scrubber, signature ->
+      "#{signature}#{to_string(scrubber)}"
+    end)
+  end
 end
 
 defmodule Pleroma.HTML.Scrubber.TwitterText do
@@ -44,14 +78,14 @@ defmodule Pleroma.HTML.Scrubber.TwitterText do
 
   # links
   Meta.allow_tag_with_uri_attributes("a", ["href", "data-user", "data-tag"], @valid_schemes)
-  Meta.allow_tag_with_these_attributes("a", ["name", "title"])
+  Meta.allow_tag_with_these_attributes("a", ["name", "title", "class"])
 
   # paragraphs and linebreaks
   Meta.allow_tag_with_these_attributes("br", [])
   Meta.allow_tag_with_these_attributes("p", [])
 
   # microformats
-  Meta.allow_tag_with_these_attributes("span", [])
+  Meta.allow_tag_with_these_attributes("span", ["class"])
 
   # allow inline images for custom emoji
   @allow_inline_images Keyword.get(@markup, :allow_inline_images)
@@ -85,7 +119,7 @@ defmodule Pleroma.HTML.Scrubber.Default do
   Meta.strip_comments()
 
   Meta.allow_tag_with_uri_attributes("a", ["href", "data-user", "data-tag"], @valid_schemes)
-  Meta.allow_tag_with_these_attributes("a", ["name", "title"])
+  Meta.allow_tag_with_these_attributes("a", ["name", "title", "class"])
 
   Meta.allow_tag_with_these_attributes("abbr", ["title"])
 
@@ -100,7 +134,7 @@ defmodule Pleroma.HTML.Scrubber.Default do
   Meta.allow_tag_with_these_attributes("ol", [])
   Meta.allow_tag_with_these_attributes("p", [])
   Meta.allow_tag_with_these_attributes("pre", [])
-  Meta.allow_tag_with_these_attributes("span", [])
+  Meta.allow_tag_with_these_attributes("span", ["class"])
   Meta.allow_tag_with_these_attributes("strong", [])
   Meta.allow_tag_with_these_attributes("u", [])
   Meta.allow_tag_with_these_attributes("ul", [])

lib/pleroma/http/connection.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.HTTP.Connection do
@@ -8,7 +8,6 @@ defmodule Pleroma.HTTP.Connection do
   """
 
   @hackney_options [
-    pool: :default,
     timeout: 10000,
     recv_timeout: 20000,
     follow_redirect: true

lib/pleroma/http/http.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.HTTP do
@@ -10,6 +10,8 @@ defmodule Pleroma.HTTP do
   alias Pleroma.HTTP.Connection
   alias Pleroma.HTTP.RequestBuilder, as: Builder
 
+  @type t :: __MODULE__
+
   @doc """
   Builds and perform http request.
 
@@ -29,12 +31,15 @@ defmodule Pleroma.HTTP do
       process_request_options(options)
       |> process_sni_options(url)
 
+    params = Keyword.get(options, :params, [])
+
     %{}
     |> Builder.method(method)
     |> Builder.headers(headers)
     |> Builder.opts(options)
     |> Builder.url(url)
     |> Builder.add_param(:body, :body, body)
+    |> Builder.add_param(:query, :query, params)
     |> Enum.into([])
     |> (&Tesla.request(Connection.new(), &1)).()
   end
@@ -54,7 +59,6 @@ defmodule Pleroma.HTTP do
   def process_request_options(options) do
     config = Application.get_env(:pleroma, :http, [])
     proxy = Keyword.get(config, :proxy_url, nil)
-    options = options ++ [adapter: [pool: :default]]
 
     case proxy do
       nil -> options

lib/pleroma/http/request_builder.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.HTTP.RequestBuilder do
@@ -100,6 +100,8 @@ defmodule Pleroma.HTTP.RequestBuilder do
   Map
   """
   @spec add_param(map(), atom, atom, any()) :: map()
+  def add_param(request, :query, :query, values), do: Map.put(request, :query, values)
+
   def add_param(request, :body, :body, value), do: Map.put(request, :body, value)
 
   def add_param(request, :body, key, value) do
@@ -107,7 +109,10 @@ defmodule Pleroma.HTTP.RequestBuilder do
     |> Map.put_new_lazy(:body, &Tesla.Multipart.new/0)
     |> Map.update!(
       :body,
-      &Tesla.Multipart.add_field(&1, key, Poison.encode!(value),
+      &Tesla.Multipart.add_field(
+        &1,
+        key,
+        Jason.encode!(value),
         headers: [{:"Content-Type", "application/json"}]
       )
     )

lib/pleroma/list.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.List do

lib/pleroma/mime.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.MIME do

lib/pleroma/notification.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Notification do
@@ -80,9 +80,8 @@ defmodule Pleroma.Notification do
   end
 
   def clear(user) do
-    query = from(n in Notification, where: n.user_id == ^user.id)
+    from(n in Notification, where: n.user_id == ^user.id)
-
+    |> Repo.delete_all()
-    Repo.delete_all(query)
   end
 
   def dismiss(%{id: user_id} = _user, id) do
@@ -110,7 +109,12 @@ defmodule Pleroma.Notification do
   # TODO move to sql, too.
   def create_notification(%Activity{} = activity, %User{} = user) do
     unless User.blocks?(user, %{ap_id: activity.data["actor"]}) or
-             user.ap_id == activity.data["actor"] do
+             user.ap_id == activity.data["actor"] or
+             (activity.data["type"] == "Follow" and
+                Enum.any?(Notification.for_user(user), fn notif ->
+                  notif.activity.data["type"] == "Follow" and
+                    notif.activity.data["actor"] == activity.data["actor"]
+                end)) do
       notification = %Notification{user_id: user.id, activity: activity}
       {:ok, notification} = Repo.insert(notification)
       Pleroma.Web.Streamer.stream("user", notification)

lib/pleroma/object.ex

@@ -1,10 +1,10 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Object do
   use Ecto.Schema
-  alias Pleroma.{Repo, Object, User, Activity}
+  alias Pleroma.{Repo, Object, User, Activity, ObjectTombstone}
   import Ecto.{Query, Changeset}
 
   schema "objects" do
@@ -66,8 +66,25 @@ defmodule Pleroma.Object do
     Object.change(%Object{}, %{data: %{"id" => context}})
   end
 
+  def make_tombstone(%Object{data: %{"id" => id, "type" => type}}, deleted \\ DateTime.utc_now()) do
+    %ObjectTombstone{
+      id: id,
+      formerType: type,
+      deleted: deleted
+    }
+    |> Map.from_struct()
+  end
+
+  def swap_object_with_tombstone(object) do
+    tombstone = make_tombstone(object)
+
+    object
+    |> Object.change(%{data: tombstone})
+    |> Repo.update()
+  end
+
   def delete(%Object{data: %{"id" => id}} = object) do
-    with Repo.delete(object),
+    with {:ok, _obj} = swap_object_with_tombstone(object),
          Repo.delete_all(Activity.all_non_create_by_object_ap_id_q(id)),
          {:ok, true} <- Cachex.del(:object_cache, "object:#{id}") do
       {:ok, object}

lib/pleroma/object_tombstone.ex

@@ -0,0 +1,4 @@
+defmodule Pleroma.ObjectTombstone do
+  @enforce_keys [:id, :formerType, :deleted]
+  defstruct [:id, :formerType, :deleted, type: "Tombstone"]
+end

lib/pleroma/plugs/admin_secret_authentication_plug.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Plugs.AdminSecretAuthenticationPlug do

lib/pleroma/plugs/authentication_plug.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Plugs.AuthenticationPlug do

lib/pleroma/plugs/basic_auth_decoder_plug.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Plugs.BasicAuthDecoderPlug do

lib/pleroma/plugs/digest.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.Plugs.DigestPlug do

lib/pleroma/plugs/ensure_authenticated_plug.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Plugs.EnsureAuthenticatedPlug do

lib/pleroma/plugs/ensure_user_key_plug.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Plugs.EnsureUserKeyPlug do

lib/pleroma/plugs/federating_plug.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.FederatingPlug do

lib/pleroma/plugs/http_security_plug.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Plugs.HTTPSecurityPlug do

lib/pleroma/plugs/http_signature.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.Plugs.HTTPSignaturePlug do

lib/pleroma/plugs/instance_static.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Plugs.InstanceStatic do

lib/pleroma/plugs/legacy_authentication_plug.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Plugs.LegacyAuthenticationPlug do

lib/pleroma/plugs/oauth_plug.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Plugs.OAuthPlug do

lib/pleroma/plugs/session_authentication_plug.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Plugs.SessionAuthenticationPlug do

lib/pleroma/plugs/set_user_session_id_plug.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Plugs.SetUserSessionIdPlug do

lib/pleroma/plugs/uploaded_media.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Plugs.UploadedMedia do

lib/pleroma/plugs/user_enabled_plug.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Plugs.UserEnabledPlug do

lib/pleroma/plugs/user_fetcher_plug.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Plugs.UserFetcherPlug do

lib/pleroma/plugs/user_is_admin_plug.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Plugs.UserIsAdminPlug do

lib/pleroma/repo.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Repo do

lib/pleroma/reverse_proxy.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.ReverseProxy do

lib/pleroma/stats.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Stats do
@@ -34,10 +34,11 @@ defmodule Pleroma.Stats do
     peers =
       from(
         u in Pleroma.User,
-        select: fragment("distinct ?->'host'", u.info),
+        select: fragment("distinct split_part(?, '@', 2)", u.nickname),
         where: u.local != ^true
       )
       |> Repo.all()
+      |> Enum.filter(& &1)
 
     domain_count = Enum.count(peers)
 
@@ -45,7 +46,7 @@ defmodule Pleroma.Stats do
       from(u in User.local_user_query(), select: fragment("sum((?->>'note_count')::int)", u.info))
 
     status_count = Repo.one(status_query)
-    user_count = Repo.aggregate(User.local_user_query(), :count, :id)
+    user_count = Repo.aggregate(User.active_local_user_query(), :count, :id)
 
     Agent.update(__MODULE__, fn _ ->
       {peers, %{domain_count: domain_count, status_count: status_count, user_count: user_count}}

lib/pleroma/upload.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Upload do
@@ -34,8 +34,9 @@ defmodule Pleroma.Upload do
   require Logger
 
   @type source ::
-          Plug.Upload.t() | data_uri_string ::
+          Plug.Upload.t()
-          String.t() | {:from_local, name :: String.t(), id :: String.t(), path :: String.t()}
+          | (data_uri_string :: String.t())
+          | {:from_local, name :: String.t(), id :: String.t(), path :: String.t()}
 
   @type option ::
           {:type, :avatar | :banner | :background}
@@ -215,6 +216,12 @@ defmodule Pleroma.Upload do
   end
 
   defp url_from_spec(base_url, {:file, path}) do
+    path =
+      path
+      |> URI.encode()
+      |> String.replace("?", "%3F")
+      |> String.replace(":", "%3A")
+
     [base_url, "media", path]
     |> Path.join()
   end

lib/pleroma/upload/filter.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Upload.Filter do

lib/pleroma/upload/filter/anonymize_filename.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Upload.Filter.AnonymizeFilename do

lib/pleroma/upload/filter/dedupe.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Upload.Filter.Dedupe do

lib/pleroma/upload/filter/mogrifun.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Upload.Filter.Mogrifun do

lib/pleroma/upload/filter/mogrify.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Upload.Filter.Mogrify do

lib/pleroma/uploaders/local.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Uploaders.Local do

lib/pleroma/uploaders/mdii.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Uploaders.MDII do

lib/pleroma/uploaders/s3.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Uploaders.S3 do

lib/pleroma/uploaders/swift/keystone.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Uploaders.Swift.Keystone do

lib/pleroma/uploaders/swift/swift.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Uploaders.Swift.Client do

lib/pleroma/uploaders/swift/uploader.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Uploaders.Swift do

lib/pleroma/uploaders/uploader.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Uploaders.Uploader do

lib/pleroma/user.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.User do
@@ -13,6 +13,8 @@ defmodule Pleroma.User do
   alias Pleroma.Web.{OStatus, Websub, OAuth}
   alias Pleroma.Web.ActivityPub.{Utils, ActivityPub}
 
+  require Logger
+
   @type t :: %__MODULE__{}
 
   @email_regex ~r/^[a-zA-Z0-9.!#$%&'*+\/=?^_`{|}~-]+@[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*$/
@@ -33,7 +35,7 @@ defmodule Pleroma.User do
     field(:avatar, :map)
     field(:local, :boolean, default: true)
     field(:follower_address, :string)
-    field(:search_distance, :float, virtual: true)
+    field(:search_rank, :float, virtual: true)
     field(:tags, {:array, :string}, default: [])
     field(:last_refreshed_at, :naive_datetime)
     has_many(:notifications, Notification)
@@ -42,12 +44,28 @@ defmodule Pleroma.User do
     timestamps()
   end
 
-  def auth_active?(%User{} = user) do
+  def auth_active?(%User{local: false}), do: true
-    (user.info && !user.info.confirmation_pending) ||
+
-      !Pleroma.Config.get([:instance, :account_activation_required])
+  def auth_active?(%User{info: %User.Info{confirmation_pending: false}}), do: true
+
+  def auth_active?(%User{info: %User.Info{confirmation_pending: true}}),
+    do: !Pleroma.Config.get([:instance, :account_activation_required])
+
+  def auth_active?(_), do: false
+
+  def visible_for?(user, for_user \\ nil)
+
+  def visible_for?(%User{id: user_id}, %User{id: for_id}) when user_id == for_id, do: true
+
+  def visible_for?(%User{} = user, for_user) do
+    auth_active?(user) || superuser?(for_user)
   end
 
-  def superuser?(%User{} = user), do: user.info && User.Info.superuser?(user.info)
+  def visible_for?(_, _), do: false
+
+  def superuser?(%User{local: true, info: %User.Info{is_admin: true}}), do: true
+  def superuser?(%User{local: true, info: %User.Info{is_moderator: true}}), do: true
+  def superuser?(_), do: false
 
   def avatar_url(user) do
     case user.avatar do
@@ -197,6 +215,7 @@ defmodule Pleroma.User do
       |> validate_confirmation(:password)
       |> unique_constraint(:email)
       |> unique_constraint(:nickname)
+      |> validate_exclusion(:nickname, Pleroma.Config.get([Pleroma.User, :restricted_nicknames]))
       |> validate_format(:nickname, local_nickname_regex())
       |> validate_format(:email, @email_regex)
       |> validate_length(:bio, max: 1000)
@@ -218,10 +237,24 @@ defmodule Pleroma.User do
     end
   end
 
+  defp autofollow_users(user) do
+    candidates = Pleroma.Config.get([:instance, :autofollowed_nicknames])
+
+    autofollowed_users =
+      from(u in User,
+        where: u.local == true,
+        where: u.nickname in ^candidates
+      )
+      |> Repo.all()
+
+    follow_all(user, autofollowed_users)
+  end
+
   @doc "Inserts provided changeset, performs post-registration actions (confirmation email sending etc.)"
   def register(%Ecto.Changeset{} = changeset) do
     with {:ok, user} <- Repo.insert(changeset),
-         {:ok, _} = try_send_confirmation_email(user) do
+         {:ok, _} <- try_send_confirmation_email(user),
+         {:ok, user} <- autofollow_users(user) do
       {:ok, user}
     end
   end
@@ -271,6 +304,25 @@ defmodule Pleroma.User do
     end
   end
 
+  @doc "A mass follow for local users. Ignores blocks and has no side effects"
+  @spec follow_all(User.t(), list(User.t())) :: {atom(), User.t()}
+  def follow_all(follower, followeds) do
+    following =
+      (follower.following ++ Enum.map(followeds, fn %{follower_address: fa} -> fa end))
+      |> Enum.uniq()
+
+    {:ok, follower} =
+      follower
+      |> follow_changeset(%{following: following})
+      |> update_and_set_cache
+
+    Enum.each(followeds, fn followed ->
+      update_follower_count(followed)
+    end)
+
+    {:ok, follower}
+  end
+
   def follow(%User{} = follower, %User{info: info} = followed) do
     user_config = Application.get_env(:pleroma, :user)
     deny_follow_blocked = Keyword.get(user_config, :deny_follow_blocked)
@@ -330,6 +382,24 @@ defmodule Pleroma.User do
     Enum.member?(follower.following, followed.follower_address)
   end
 
+  def follow_import(%User{} = follower, followed_identifiers)
+      when is_list(followed_identifiers) do
+    Enum.map(
+      followed_identifiers,
+      fn followed_identifier ->
+        with %User{} = followed <- get_or_fetch(followed_identifier),
+             {:ok, follower} <- maybe_direct_follow(follower, followed),
+             {:ok, _} <- ActivityPub.follow(follower, followed) do
+          followed
+        else
+          err ->
+            Logger.debug("follow_import failed for #{followed_identifier} with: #{inspect(err)}")
+            err
+        end
+      end
+    )
+  end
+
   def locked?(%User{} = user) do
     user.info.locked || false
   end
@@ -338,6 +408,15 @@ defmodule Pleroma.User do
     Repo.get_by(User, ap_id: ap_id)
   end
 
+  # This is mostly an SPC migration fix. This guesses the user nickname (by taking the last part of the ap_id and the domain) and tries to get that user
+  def get_by_guessed_nickname(ap_id) do
+    domain = URI.parse(ap_id).host
+    name = List.last(String.split(ap_id, "/"))
+    nickname = "#{name}@#{domain}"
+
+    get_by_nickname(nickname)
+  end
+
   def update_and_set_cache(changeset) do
     with {:ok, user} <- Repo.update(changeset) do
       Cachex.put(:user_cache, "ap_id:#{user.ap_id}", user)
@@ -366,7 +445,10 @@ defmodule Pleroma.User do
   end
 
   def get_by_nickname(nickname) do
-    Repo.get_by(User, nickname: nickname)
+    Repo.get_by(User, nickname: nickname) ||
+      if Regex.match?(~r(@#{Pleroma.Web.Endpoint.host()})i, nickname) do
+        Repo.get_by(User, nickname: local_nickname(nickname))
+      end
   end
 
   def get_by_nickname_or_email(nickname_or_email) do
@@ -404,7 +486,7 @@ defmodule Pleroma.User do
     end
   end
 
-  def get_followers_query(%User{id: id, follower_address: follower_address}) do
+  def get_followers_query(%User{id: id, follower_address: follower_address}, nil) do
     from(
       u in User,
       where: fragment("? <@ ?", ^[follower_address], u.following),
@@ -412,13 +494,29 @@ defmodule Pleroma.User do
     )
   end
 
-  def get_followers(user) do
+  def get_followers_query(user, page) do
-    q = get_followers_query(user)
+    from(
+      u in get_followers_query(user, nil),
+      limit: 20,
+      offset: ^((page - 1) * 20)
+    )
+  end
+
+  def get_followers_query(user), do: get_followers_query(user, nil)
+
+  def get_followers(user, page \\ nil) do
+    q = get_followers_query(user, page)
 
     {:ok, Repo.all(q)}
   end
 
-  def get_friends_query(%User{id: id, following: following}) do
+  def get_followers_ids(user, page \\ nil) do
+    q = get_followers_query(user, page)
+
+    Repo.all(from(u in q, select: u.id))
+  end
+
+  def get_friends_query(%User{id: id, following: following}, nil) do
     from(
       u in User,
       where: u.follower_address in ^following,
@@ -426,12 +524,28 @@ defmodule Pleroma.User do
     )
   end
 
-  def get_friends(user) do
+  def get_friends_query(user, page) do
-    q = get_friends_query(user)
+    from(
+      u in get_friends_query(user, nil),
+      limit: 20,
+      offset: ^((page - 1) * 20)
+    )
+  end
+
+  def get_friends_query(user), do: get_friends_query(user, nil)
+
+  def get_friends(user, page \\ nil) do
+    q = get_friends_query(user, page)
 
     {:ok, Repo.all(q)}
   end
 
+  def get_friends_ids(user, page \\ nil) do
+    q = get_friends_query(user, page)
+
+    Repo.all(from(u in q, select: u.id))
+  end
+
   def get_follow_requests_query(%User{} = user) do
     from(
       a in Activity,
@@ -462,6 +576,7 @@ defmodule Pleroma.User do
       Enum.map(reqs, fn req -> req.actor end)
       |> Enum.uniq()
       |> Enum.map(fn ap_id -> get_by_ap_id(ap_id) end)
+      |> Enum.filter(fn u -> !is_nil(u) end)
       |> Enum.filter(fn u -> !following?(u, user) end)
 
     {:ok, users}
@@ -562,37 +677,137 @@ defmodule Pleroma.User do
     Repo.all(query)
   end
 
-  def search(query, resolve \\ false) do
+  def search(query, resolve \\ false, for_user \\ nil) do
-    # strip the beginning @ off if there is a query
+    # Strip the beginning @ off if there is a query
     query = String.trim_leading(query, "@")
 
-    if resolve do
+    if resolve, do: User.get_or_fetch_by_nickname(query)
-      User.get_or_fetch_by_nickname(query)
-    end
 
-    inner =
+    fts_results = do_search(fts_search_subquery(query), for_user)
-      from(
-        u in User,
-        select_merge: %{
-          search_distance:
-            fragment(
-              "? <-> (? || ?)",
-              ^query,
-              u.nickname,
-              u.name
-            )
-        },
-        where: not is_nil(u.nickname)
-      )
 
+    {:ok, trigram_results} =
+      Repo.transaction(fn ->
+        Ecto.Adapters.SQL.query(Repo, "select set_limit(0.25)", [])
+        do_search(trigram_search_subquery(query), for_user)
+      end)
+
+    Enum.uniq_by(fts_results ++ trigram_results, & &1.id)
+  end
+
+  defp do_search(subquery, for_user, options \\ []) do
     q =
       from(
-        s in subquery(inner),
+        s in subquery(subquery),
-        order_by: s.search_distance,
+        order_by: [desc: s.search_rank],
-        limit: 20
+        limit: ^(options[:limit] || 20)
       )
 
-    Repo.all(q)
+    results =
+      q
+      |> Repo.all()
+      |> Enum.filter(&(&1.search_rank > 0))
+
+    boost_search_results(results, for_user)
+  end
+
+  defp fts_search_subquery(query) do
+    processed_query =
+      query
+      |> String.replace(~r/\W+/, " ")
+      |> String.trim()
+      |> String.split()
+      |> Enum.map(&(&1 <> ":*"))
+      |> Enum.join(" | ")
+
+    from(
+      u in User,
+      select_merge: %{
+        search_rank:
+          fragment(
+            """
+            ts_rank_cd(
+              setweight(to_tsvector('simple', regexp_replace(?, '\\W', ' ', 'g')), 'A') ||
+              setweight(to_tsvector('simple', regexp_replace(coalesce(?, ''), '\\W', ' ', 'g')), 'B'),
+              to_tsquery('simple', ?),
+              32
+            )
+            """,
+            u.nickname,
+            u.name,
+            ^processed_query
+          )
+      },
+      where:
+        fragment(
+          """
+            (setweight(to_tsvector('simple', regexp_replace(?, '\\W', ' ', 'g')), 'A') ||
+            setweight(to_tsvector('simple', regexp_replace(coalesce(?, ''), '\\W', ' ', 'g')), 'B')) @@ to_tsquery('simple', ?)
+          """,
+          u.nickname,
+          u.name,
+          ^processed_query
+        )
+    )
+  end
+
+  defp trigram_search_subquery(query) do
+    from(
+      u in User,
+      select_merge: %{
+        search_rank:
+          fragment(
+            "similarity(?, trim(? || ' ' || coalesce(?, '')))",
+            ^query,
+            u.nickname,
+            u.name
+          )
+      },
+      where: fragment("trim(? || ' ' || coalesce(?, '')) % ?", u.nickname, u.name, ^query)
+    )
+  end
+
+  defp boost_search_results(results, nil), do: results
+
+  defp boost_search_results(results, for_user) do
+    friends_ids = get_friends_ids(for_user)
+    followers_ids = get_followers_ids(for_user)
+
+    Enum.map(
+      results,
+      fn u ->
+        search_rank_coef =
+          cond do
+            u.id in friends_ids ->
+              1.2
+
+            u.id in followers_ids ->
+              1.1
+
+            true ->
+              1
+          end
+
+        Map.put(u, :search_rank, u.search_rank * search_rank_coef)
+      end
+    )
+    |> Enum.sort_by(&(-&1.search_rank))
+  end
+
+  def blocks_import(%User{} = blocker, blocked_identifiers) when is_list(blocked_identifiers) do
+    Enum.map(
+      blocked_identifiers,
+      fn blocked_identifier ->
+        with %User{} = blocked <- get_or_fetch(blocked_identifier),
+             {:ok, blocker} <- block(blocker, blocked),
+             {:ok, _} <- ActivityPub.block(blocker, blocked) do
+          blocked
+        else
+          err ->
+            Logger.debug("blocks_import failed for #{blocked_identifier} with: #{inspect(err)}")
+            err
+        end
+      end
+    )
   end
 
   def block(blocker, %User{ap_id: ap_id} = blocked) do
@@ -648,6 +863,9 @@ defmodule Pleroma.User do
       end)
   end
 
+  def blocked_users(user),
+    do: Repo.all(from(u in User, where: u.ap_id in ^user.info.blocks))
+
   def block_domain(user, domain) do
     info_cng =
       user.info
@@ -672,7 +890,7 @@ defmodule Pleroma.User do
     update_and_set_cache(cng)
   end
 
-  def local_user_query() do
+  def local_user_query do
     from(
       u in User,
       where: u.local == true,
@@ -680,7 +898,14 @@ defmodule Pleroma.User do
     )
   end
 
-  def moderator_user_query() do
+  def active_local_user_query do
+    from(
+      u in local_user_query(),
+      where: fragment("?->'deactivated' @> 'false'", u.info)
+    )
+  end
+
+  def moderator_user_query do
     from(
       u in User,
       where: u.local == true,
@@ -733,7 +958,9 @@ defmodule Pleroma.User do
     Pleroma.HTML.Scrubber.TwitterText
   end
 
-  def html_filter_policy(_), do: nil
+  @default_scrubbers Pleroma.Config.get([:markup, :scrub_policy])
+
+  def html_filter_policy(_), do: @default_scrubbers
 
   def get_or_fetch_by_ap_id(ap_id) do
     user = get_by_ap_id(ap_id)
@@ -864,7 +1091,7 @@ defmodule Pleroma.User do
       end)
 
     bio
-    |> CommonUtils.format_input(mentions, tags, "text/plain")
+    |> CommonUtils.format_input(mentions, tags, "text/plain", user_links: [format: :full])
     |> Formatter.emojify(emoji)
   end
 
@@ -914,4 +1141,24 @@ defmodule Pleroma.User do
       @strict_local_nickname_regex
     end
   end
+
+  def local_nickname(nickname_or_mention) do
+    nickname_or_mention
+    |> full_nickname()
+    |> String.split("@")
+    |> hd()
+  end
+
+  def full_nickname(nickname_or_mention),
+    do: String.trim_leading(nickname_or_mention, "@")
+
+  def error_user(ap_id) do
+    %User{
+      name: ap_id,
+      ap_id: ap_id,
+      info: %User.Info{},
+      nickname: "erroruser@example.com",
+      inserted_at: NaiveDateTime.utc_now()
+    }
+  end
 end

lib/pleroma/user/info.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.User.Info do
@@ -31,6 +31,7 @@ defmodule Pleroma.User.Info do
     field(:hub, :string, default: nil)
     field(:salmon, :string, default: nil)
     field(:hide_network, :boolean, default: false)
+    field(:pinned_activities, {:array, :integer}, default: [])
 
     # Found in the wild
     # ap_id -> Where is this used?
@@ -41,8 +42,6 @@ defmodule Pleroma.User.Info do
     # subject _> Where is this used?
   end
 
-  def superuser?(info), do: info.is_admin || info.is_moderator
-
   def set_activation_status(info, deactivated) do
     params = %{deactivated: deactivated}
 
@@ -198,4 +197,26 @@ defmodule Pleroma.User.Info do
       :is_admin
     ])
   end
+
+  def add_pinnned_activity(info, %Pleroma.Activity{id: id}) do
+    if id not in info.pinned_activities do
+      max_pinned_statuses = Pleroma.Config.get([:instance, :max_pinned_statuses], 0)
+      params = %{pinned_activities: info.pinned_activities ++ [id]}
+
+      info
+      |> cast(params, [:pinned_activities])
+      |> validate_length(:pinned_activities,
+        max: max_pinned_statuses,
+        message: "You have already pinned the maximum number of statuses"
+      )
+    else
+      change(info)
+    end
+  end
+
+  def remove_pinnned_activity(info, %Pleroma.Activity{id: id}) do
+    params = %{pinned_activities: List.delete(info.pinned_activities, id)}
+
+    cast(info, params, [:pinned_activities])
+  end
 end

lib/pleroma/user_invite_token.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.UserInviteToken do

lib/pleroma/web/activity_pub/activity_pub.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.ActivityPub.ActivityPub do
@@ -56,10 +56,18 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     end
   end
 
+  defp check_remote_limit(%{"object" => %{"content" => content}}) do
+    limit = Pleroma.Config.get([:instance, :remote_limit])
+    String.length(content) <= limit
+  end
+
+  defp check_remote_limit(_), do: true
+
   def insert(map, local \\ true) when is_map(map) do
     with nil <- Activity.normalize(map),
          map <- lazy_put_activity_defaults(map),
          :ok <- check_actor_is_active(map["actor"]),
+         {_, true} <- {:remote_limit_error, check_remote_limit(map)},
          {:ok, map} <- MRF.filter(map),
          :ok <- insert_full_object(map) do
       {recipients, _, _} = get_recipients(map)
@@ -84,7 +92,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
   def stream_out(activity) do
     public = "https://www.w3.org/ns/activitystreams#Public"
 
-    if activity.data["type"] in ["Create", "Announce"] do
+    if activity.data["type"] in ["Create", "Announce", "Delete"] do
       Pleroma.Web.Streamer.stream("user", activity)
       Pleroma.Web.Streamer.stream("list", activity)
 
@@ -95,16 +103,18 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
           Pleroma.Web.Streamer.stream("public:local", activity)
         end
 
-        activity.data["object"]
+        if activity.data["type"] in ["Create"] do
-        |> Map.get("tag", [])
+          activity.data["object"]
-        |> Enum.filter(fn tag -> is_bitstring(tag) end)
+          |> Map.get("tag", [])
-        |> Enum.map(fn tag -> Pleroma.Web.Streamer.stream("hashtag:" <> tag, activity) end)
+          |> Enum.filter(fn tag -> is_bitstring(tag) end)
+          |> Enum.map(fn tag -> Pleroma.Web.Streamer.stream("hashtag:" <> tag, activity) end)
 
-        if activity.data["object"]["attachment"] != [] do
+          if activity.data["object"]["attachment"] != [] do
-          Pleroma.Web.Streamer.stream("public:media", activity)
+            Pleroma.Web.Streamer.stream("public:media", activity)
 
-          if activity.local do
+            if activity.local do
-            Pleroma.Web.Streamer.stream("public:local:media", activity)
+              Pleroma.Web.Streamer.stream("public:local:media", activity)
+            end
           end
         end
       else
@@ -216,10 +226,11 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
         %User{ap_id: _} = user,
         %Object{data: %{"id" => _}} = object,
         activity_id \\ nil,
-        local \\ true
+        local \\ true,
+        public \\ true
       ) do
     with true <- is_public?(object),
-         announce_data <- make_announce_data(user, object, activity_id),
+         announce_data <- make_announce_data(user, object, activity_id, public),
          {:ok, activity} <- insert(announce_data, local),
          {:ok, object} <- add_announce_to_object(activity, object),
          :ok <- maybe_federate(activity) do
@@ -356,21 +367,18 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
 
   @valid_visibilities ~w[direct unlisted public private]
 
-  defp restrict_visibility(query, %{visibility: "direct"}) do
+  defp restrict_visibility(query, %{visibility: visibility})
-    public = "https://www.w3.org/ns/activitystreams#Public"
+       when visibility in @valid_visibilities do
+    query =
+      from(
+        a in query,
+        where:
+          fragment("activity_visibility(?, ?, ?) = ?", a.actor, a.recipients, a.data, ^visibility)
+      )
 
-    from(
+    Ecto.Adapters.SQL.to_sql(:all, Repo, query)
-      activity in query,
+
-      join: sender in User,
+    query
-      on: sender.ap_id == activity.actor,
-      # Are non-direct statuses with no to/cc possible?
-      where:
-        fragment(
-          "not (? && ?)",
-          [^public, sender.follower_address],
-          activity.recipients
-        )
-    )
   end
 
   defp restrict_visibility(_query, %{visibility: visibility})
@@ -386,6 +394,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
       |> Map.put("type", ["Create", "Announce"])
       |> Map.put("actor_id", user.ap_id)
       |> Map.put("whole_db", true)
+      |> Map.put("pinned_activity_ids", user.info.pinned_activities)
 
     recipients =
       if reading_user do
@@ -503,15 +512,12 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
 
   defp restrict_replies(query, _), do: query
 
-  # Only search through last 100_000 activities by default
+  defp restrict_reblogs(query, %{"exclude_reblogs" => val}) when val == "true" or val == "1" do
-  defp restrict_recent(query, %{"whole_db" => true}), do: query
+    from(activity in query, where: fragment("?->>'type' != 'Announce'", activity.data))
-
-  defp restrict_recent(query, _) do
-    since = (Repo.aggregate(Activity, :max, :id) || 0) - 100_000
-
-    from(activity in query, where: activity.id > ^since)
   end
 
+  defp restrict_reblogs(query, _), do: query
+
   defp restrict_blocked(query, %{"blocking_user" => %User{info: info}}) do
     blocks = info.blocks || []
     domain_blocks = info.domain_blocks || []
@@ -538,6 +544,12 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     )
   end
 
+  defp restrict_pinned(query, %{"pinned" => "true", "pinned_activity_ids" => ids}) do
+    from(activity in query, where: activity.id in ^ids)
+  end
+
+  defp restrict_pinned(query, _), do: query
+
   def fetch_activities_query(recipients, opts \\ %{}) do
     base_query =
       from(
@@ -556,11 +568,12 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     |> restrict_actor(opts)
     |> restrict_type(opts)
     |> restrict_favorited_by(opts)
-    |> restrict_recent(opts)
     |> restrict_blocked(opts)
     |> restrict_media(opts)
     |> restrict_visibility(opts)
     |> restrict_replies(opts)
+    |> restrict_reblogs(opts)
+    |> restrict_pinned(opts)
   end
 
   def fetch_activities(recipients, opts \\ %{}) do
@@ -726,8 +739,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
         {"Content-Type", "application/activity+json"},
         {"signature", signature},
         {"digest", digest}
-      ],
+      ]
-      hackney: [pool: :default]
     )
   end
 
@@ -787,9 +799,12 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     end
   end
 
-  def is_public?(activity) do
+  def is_public?(%Object{data: %{"type" => "Tombstone"}}), do: false
-    "https://www.w3.org/ns/activitystreams#Public" in (activity.data["to"] ++
+  def is_public?(%Object{data: data}), do: is_public?(data)
-                                                         (activity.data["cc"] || []))
+  def is_public?(%Activity{data: data}), do: is_public?(data)
+
+  def is_public?(data) do
+    "https://www.w3.org/ns/activitystreams#Public" in (data["to"] ++ (data["cc"] || []))
   end
 
   def visible_for_user?(activity, nil) do

lib/pleroma/web/activity_pub/activity_pub_controller.ex

@@ -1,14 +1,15 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.ActivityPub.ActivityPubController do
   use Pleroma.Web, :controller
-  alias Pleroma.{User, Object}
+  alias Pleroma.{Activity, User, Object}
   alias Pleroma.Web.ActivityPub.{ObjectView, UserView}
   alias Pleroma.Web.ActivityPub.ActivityPub
   alias Pleroma.Web.ActivityPub.Relay
   alias Pleroma.Web.ActivityPub.Utils
+  alias Pleroma.Web.ActivityPub.Transmogrifier
   alias Pleroma.Web.Federator
 
   require Logger
@@ -53,6 +54,49 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
     end
   end
 
+  def object_likes(conn, %{"uuid" => uuid, "page" => page}) do
+    with ap_id <- o_status_url(conn, :object, uuid),
+         %Object{} = object <- Object.get_cached_by_ap_id(ap_id),
+         {_, true} <- {:public?, ActivityPub.is_public?(object)},
+         likes <- Utils.get_object_likes(object) do
+      {page, _} = Integer.parse(page)
+
+      conn
+      |> put_resp_header("content-type", "application/activity+json")
+      |> json(ObjectView.render("likes.json", ap_id, likes, page))
+    else
+      {:public?, false} ->
+        {:error, :not_found}
+    end
+  end
+
+  def object_likes(conn, %{"uuid" => uuid}) do
+    with ap_id <- o_status_url(conn, :object, uuid),
+         %Object{} = object <- Object.get_cached_by_ap_id(ap_id),
+         {_, true} <- {:public?, ActivityPub.is_public?(object)},
+         likes <- Utils.get_object_likes(object) do
+      conn
+      |> put_resp_header("content-type", "application/activity+json")
+      |> json(ObjectView.render("likes.json", ap_id, likes))
+    else
+      {:public?, false} ->
+        {:error, :not_found}
+    end
+  end
+
+  def activity(conn, %{"uuid" => uuid}) do
+    with ap_id <- o_status_url(conn, :activity, uuid),
+         %Activity{} = activity <- Activity.normalize(ap_id),
+         {_, true} <- {:public?, ActivityPub.is_public?(activity)} do
+      conn
+      |> put_resp_header("content-type", "application/activity+json")
+      |> json(ObjectView.render("object.json", %{object: activity}))
+    else
+      {:public?, false} ->
+        {:error, :not_found}
+    end
+  end
+
   def following(conn, %{"nickname" => nickname, "page" => page}) do
     with %User{} = user <- User.get_cached_by_nickname(nickname),
          {:ok, user} <- Pleroma.Web.WebFinger.ensure_keys_present(user) do
@@ -93,19 +137,15 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
     end
   end
 
-  def outbox(conn, %{"nickname" => nickname, "max_id" => max_id}) do
+  def outbox(conn, %{"nickname" => nickname} = params) do
     with %User{} = user <- User.get_cached_by_nickname(nickname),
          {:ok, user} <- Pleroma.Web.WebFinger.ensure_keys_present(user) do
       conn
       |> put_resp_header("content-type", "application/activity+json")
-      |> json(UserView.render("outbox.json", %{user: user, max_id: max_id}))
+      |> json(UserView.render("outbox.json", %{user: user, max_id: params["max_id"]}))
     end
   end
 
-  def outbox(conn, %{"nickname" => nickname}) do
-    outbox(conn, %{"nickname" => nickname, "max_id" => nil})
-  end
-
   def inbox(%{assigns: %{valid_signature: true}} = conn, %{"nickname" => nickname} = params) do
     with %User{} = user <- User.get_cached_by_nickname(nickname),
          true <- Utils.recipient_in_message(user.ap_id, params),
@@ -156,6 +196,88 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
     end
   end
 
+  def read_inbox(%{assigns: %{user: user}} = conn, %{"nickname" => nickname} = params) do
+    if nickname == user.nickname do
+      conn
+      |> put_resp_header("content-type", "application/activity+json")
+      |> json(UserView.render("inbox.json", %{user: user, max_id: params["max_id"]}))
+    else
+      conn
+      |> put_status(:forbidden)
+      |> json("can't read inbox of #{nickname} as #{user.nickname}")
+    end
+  end
+
+  def handle_user_activity(user, %{"type" => "Create"} = params) do
+    object =
+      params["object"]
+      |> Map.merge(Map.take(params, ["to", "cc"]))
+      |> Map.put("attributedTo", user.ap_id())
+      |> Transmogrifier.fix_object()
+
+    ActivityPub.create(%{
+      to: params["to"],
+      actor: user,
+      context: object["context"],
+      object: object,
+      additional: Map.take(params, ["cc"])
+    })
+  end
+
+  def handle_user_activity(user, %{"type" => "Delete"} = params) do
+    with %Object{} = object <- Object.normalize(params["object"]),
+         true <- user.info.is_moderator || user.ap_id == object.data["actor"],
+         {:ok, delete} <- ActivityPub.delete(object) do
+      {:ok, delete}
+    else
+      _ -> {:error, "Can't delete object"}
+    end
+  end
+
+  def handle_user_activity(user, %{"type" => "Like"} = params) do
+    with %Object{} = object <- Object.normalize(params["object"]),
+         {:ok, activity, _object} <- ActivityPub.like(user, object) do
+      {:ok, activity}
+    else
+      _ -> {:error, "Can't like object"}
+    end
+  end
+
+  def handle_user_activity(_, _) do
+    {:error, "Unhandled activity type"}
+  end
+
+  def update_outbox(
+        %{assigns: %{user: user}} = conn,
+        %{"nickname" => nickname} = params
+      ) do
+    if nickname == user.nickname do
+      actor = user.ap_id()
+
+      params =
+        params
+        |> Map.drop(["id"])
+        |> Map.put("actor", actor)
+        |> Transmogrifier.fix_addressing()
+
+      with {:ok, %Activity{} = activity} <- handle_user_activity(user, params) do
+        conn
+        |> put_status(:created)
+        |> put_resp_header("location", activity.data["id"])
+        |> json(activity.data)
+      else
+        {:error, message} ->
+          conn
+          |> put_status(:bad_request)
+          |> json(message)
+      end
+    else
+      conn
+      |> put_status(:forbidden)
+      |> json("can't update outbox of #{nickname} as #{user.nickname}")
+    end
+  end
+
   def errors(conn, {:error, :not_found}) do
     conn
     |> put_status(404)

lib/pleroma/web/activity_pub/mrf.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.ActivityPub.MRF do

lib/pleroma/web/activity_pub/mrf/drop_policy.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.ActivityPub.MRF.DropPolicy do

lib/pleroma/web/activity_pub/mrf/ensure_re_prepended.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.ActivityPub.MRF.EnsureRePrepended do

lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.ActivityPub.MRF.HellthreadPolicy do

lib/pleroma/web/activity_pub/mrf/no_placeholder_text_policy.ex

@@ -0,0 +1,29 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ActivityPub.MRF.NoPlaceholderTextPolicy do
+  @behaviour Pleroma.Web.ActivityPub.MRF
+
+  @impl true
+  def filter(
+        %{
+          "type" => "Create",
+          "object" => %{"content" => content, "attachment" => _attachment} = child_object
+        } = object
+      )
+      when content in [".", "<p>.</p>"] do
+    child_object =
+      child_object
+      |> Map.put("content", "")
+
+    object =
+      object
+      |> Map.put("object", child_object)
+
+    {:ok, object}
+  end
+
+  @impl true
+  def filter(object), do: {:ok, object}
+end

lib/pleroma/web/activity_pub/mrf/noop_policy.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.ActivityPub.MRF.NoOpPolicy do

lib/pleroma/web/activity_pub/mrf/normalize_markup.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.ActivityPub.MRF.NormalizeMarkup do

lib/pleroma/web/activity_pub/mrf/reject_non_public.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.ActivityPub.MRF.RejectNonPublic do

lib/pleroma/web/activity_pub/mrf/simple_policy.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do

lib/pleroma/web/activity_pub/mrf/user_allowlist.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.ActivityPub.MRF.UserAllowListPolicy do

lib/pleroma/web/activity_pub/relay.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.ActivityPub.Relay do
@@ -40,7 +40,7 @@ defmodule Pleroma.Web.ActivityPub.Relay do
   def publish(%Activity{data: %{"type" => "Create"}} = activity) do
     with %User{} = user <- get_actor(),
          %Object{} = object <- Object.normalize(activity.data["object"]["id"]) do
-      ActivityPub.announce(user, object)
+      ActivityPub.announce(user, object, nil, true, false)
     else
       e -> Logger.error("error: #{inspect(e)}")
     end

lib/pleroma/web/activity_pub/transmogrifier.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.ActivityPub.Transmogrifier do
@@ -451,7 +451,8 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
     with actor <- get_actor(data),
          %User{} = actor <- User.get_or_fetch_by_ap_id(actor),
          {:ok, object} <- get_obj_helper(object_id) || fetch_obj_helper(object_id),
-         {:ok, activity, _object} <- ActivityPub.announce(actor, object, id, false) do
+         public <- ActivityPub.is_public?(data),
+         {:ok, activity, _object} <- ActivityPub.announce(actor, object, id, false, public) do
       {:ok, activity}
     else
       _e -> :error
@@ -629,6 +630,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
     |> add_mention_tags
     |> add_emoji_tags
     |> add_attributed_to
+    |> add_likes
     |> prepare_attachments
     |> set_conversation
     |> set_reply_to_uri
@@ -641,7 +643,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
   #  internal -> Mastodon
   #  """
 
-  def prepare_outgoing(%{"type" => "Create", "object" => %{"type" => "Note"} = object} = data) do
+  def prepare_outgoing(%{"type" => "Create", "object" => object} = data) do
     object =
       object
       |> prepare_object
@@ -788,6 +790,22 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
     |> Map.put("attributedTo", attributedTo)
   end
 
+  def add_likes(%{"id" => id, "like_count" => likes} = object) do
+    likes = %{
+      "id" => "#{id}/likes",
+      "first" => "#{id}/likes?page=1",
+      "type" => "OrderedCollection",
+      "totalItems" => likes
+    }
+
+    object
+    |> Map.put("likes", likes)
+  end
+
+  def add_likes(object) do
+    object
+  end
+
   def prepare_attachments(object) do
     attachments =
       (object["attachment"] || [])
@@ -803,7 +821,6 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
   defp strip_internal_fields(object) do
     object
     |> Map.drop([
-      "likes",
       "like_count",
       "announcements",
       "announcement_count",

lib/pleroma/web/activity_pub/utils.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.ActivityPub.Utils do
@@ -231,6 +231,27 @@ defmodule Pleroma.Web.ActivityPub.Utils do
     Repo.one(query)
   end
 
+  @doc """
+  Returns like activities targeting an object
+  """
+  def get_object_likes(%{data: %{"id" => id}}) do
+    query =
+      from(
+        activity in Activity,
+        # this is to use the index
+        where:
+          fragment(
+            "coalesce((?)->'object'->>'id', (?)->>'object') = ?",
+            activity.data,
+            activity.data,
+            ^id
+          ),
+        where: fragment("(?)->>'type' = 'Like'", activity.data)
+      )
+
+    Repo.all(query)
+  end
+
   def make_like_data(%User{ap_id: ap_id} = actor, %{data: %{"id" => id}} = object, activity_id) do
     data = %{
       "type" => "Like",
@@ -365,9 +386,10 @@ defmodule Pleroma.Web.ActivityPub.Utils do
   """
   # for relayed messages, we only want to send to subscribers
   def make_announce_data(
-        %User{ap_id: ap_id, nickname: nil} = user,
+        %User{ap_id: ap_id} = user,
         %Object{data: %{"id" => id}} = object,
-        activity_id
+        activity_id,
+        false
       ) do
     data = %{
       "type" => "Announce",
@@ -384,7 +406,8 @@ defmodule Pleroma.Web.ActivityPub.Utils do
   def make_announce_data(
         %User{ap_id: ap_id} = user,
         %Object{data: %{"id" => id}} = object,
-        activity_id
+        activity_id,
+        true
       ) do
     data = %{
       "type" => "Announce",

lib/pleroma/web/activity_pub/views/object_view.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.ActivityPub.ObjectView do
@@ -35,4 +35,38 @@ defmodule Pleroma.Web.ActivityPub.ObjectView do
 
     Map.merge(base, additional)
   end
+
+  def render("likes.json", ap_id, likes, page) do
+    collection(likes, "#{ap_id}/likes", page)
+    |> Map.merge(Pleroma.Web.ActivityPub.Utils.make_json_ld_header())
+  end
+
+  def render("likes.json", ap_id, likes) do
+    %{
+      "id" => "#{ap_id}/likes",
+      "type" => "OrderedCollection",
+      "totalItems" => length(likes),
+      "first" => collection(likes, "#{ap_id}/likes", 1)
+    }
+    |> Map.merge(Pleroma.Web.ActivityPub.Utils.make_json_ld_header())
+  end
+
+  def collection(collection, iri, page) do
+    offset = (page - 1) * 10
+    items = Enum.slice(collection, offset, 10)
+    items = Enum.map(items, fn object -> Transmogrifier.prepare_object(object.data) end)
+    total = length(collection)
+
+    map = %{
+      "id" => "#{iri}?page=#{page}",
+      "type" => "OrderedCollectionPage",
+      "partOf" => iri,
+      "totalItems" => total,
+      "orderedItems" => items
+    }
+
+    if offset < total do
+      Map.put(map, "next", "#{iri}?page=#{page + 1}")
+    end
+  end
 end

lib/pleroma/web/activity_pub/views/user_view.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.ActivityPub.UserView do
@@ -176,6 +176,53 @@ defmodule Pleroma.Web.ActivityPub.UserView do
     end
   end
 
+  def render("inbox.json", %{user: user, max_id: max_qid}) do
+    params = %{
+      "limit" => "10"
+    }
+
+    params =
+      if max_qid != nil do
+        Map.put(params, "max_id", max_qid)
+      else
+        params
+      end
+
+    activities = ActivityPub.fetch_activities([user.ap_id | user.following], params)
+
+    min_id = Enum.at(Enum.reverse(activities), 0).id
+    max_id = Enum.at(activities, 0).id
+
+    collection =
+      Enum.map(activities, fn act ->
+        {:ok, data} = Transmogrifier.prepare_outgoing(act.data)
+        data
+      end)
+
+    iri = "#{user.ap_id}/inbox"
+
+    page = %{
+      "id" => "#{iri}?max_id=#{max_id}",
+      "type" => "OrderedCollectionPage",
+      "partOf" => iri,
+      "totalItems" => -1,
+      "orderedItems" => collection,
+      "next" => "#{iri}?max_id=#{min_id - 1}"
+    }
+
+    if max_qid == nil do
+      %{
+        "id" => iri,
+        "type" => "OrderedCollection",
+        "totalItems" => -1,
+        "first" => page
+      }
+      |> Map.merge(Utils.make_json_ld_header())
+    else
+      page |> Map.merge(Utils.make_json_ld_header())
+    end
+  end
+
   def collection(collection, iri, page, show_items \\ true, total \\ nil) do
     offset = (page - 1) * 10
     items = Enum.slice(collection, offset, 10)

lib/pleroma/web/admin_api/admin_api_controller.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.AdminAPI.AdminAPIController do
@@ -14,13 +14,8 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
   action_fallback(:errors)
 
   def user_delete(conn, %{"nickname" => nickname}) do
-    user = User.get_by_nickname(nickname)
+    User.get_by_nickname(nickname)
-
+    |> User.delete()
-    if user.local == true do
-      User.delete(user)
-    else
-      User.delete(user)
-    end
 
     conn
     |> json(nickname)

lib/pleroma/web/channels/user_socket.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.UserSocket do

lib/pleroma/web/chat_channel.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.ChatChannel do

lib/pleroma/web/common_api/common_api.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.CommonAPI do
@@ -14,6 +14,7 @@ defmodule Pleroma.Web.CommonAPI do
     with %Activity{data: %{"object" => %{"id" => object_id}}} <- Repo.get(Activity, activity_id),
          %Object{} = object <- Object.normalize(object_id),
          true <- user.info.is_moderator || user.ap_id == object.data["actor"],
+         {:ok, _} <- unpin(activity_id, user),
          {:ok, delete} <- ActivityPub.delete(object) do
       {:ok, delete}
     end
@@ -102,7 +103,14 @@ defmodule Pleroma.Web.CommonAPI do
              attachments,
              tags,
              get_content_type(data["content_type"]),
-             data["no_attachment_links"]
+             Enum.member?(
+               [true, "true"],
+               Map.get(
+                 data,
+                 "no_attachment_links",
+                 Pleroma.Config.get([:instance, :no_attachment_links], false)
+               )
+             )
            ),
          context <- make_context(inReplyTo),
          cw <- data["spoiler_text"],
@@ -124,7 +132,7 @@ defmodule Pleroma.Web.CommonAPI do
            Map.put(
              object,
              "emoji",
-             Formatter.get_emoji(status)
+             (Formatter.get_emoji(status) ++ Formatter.get_emoji(data["spoiler_text"]))
              |> Enum.reduce(%{}, fn {name, file}, acc ->
                Map.put(acc, name, "#{Pleroma.Web.Endpoint.static_url()}#{file}")
              end)
@@ -164,4 +172,48 @@ defmodule Pleroma.Web.CommonAPI do
       object: Pleroma.Web.ActivityPub.UserView.render("user.json", %{user: user})
     })
   end
+
+  def pin(id_or_ap_id, %{ap_id: user_ap_id} = user) do
+    with %Activity{
+           actor: ^user_ap_id,
+           data: %{
+             "type" => "Create",
+             "object" => %{
+               "to" => object_to,
+               "type" => "Note"
+             }
+           }
+         } = activity <- get_by_id_or_ap_id(id_or_ap_id),
+         true <- Enum.member?(object_to, "https://www.w3.org/ns/activitystreams#Public"),
+         %{valid?: true} = info_changeset <-
+           Pleroma.User.Info.add_pinnned_activity(user.info, activity),
+         changeset <-
+           Ecto.Changeset.change(user) |> Ecto.Changeset.put_embed(:info, info_changeset),
+         {:ok, _user} <- User.update_and_set_cache(changeset) do
+      {:ok, activity}
+    else
+      %{errors: [pinned_activities: {err, _}]} ->
+        {:error, err}
+
+      _ ->
+        {:error, "Could not pin"}
+    end
+  end
+
+  def unpin(id_or_ap_id, user) do
+    with %Activity{} = activity <- get_by_id_or_ap_id(id_or_ap_id),
+         %{valid?: true} = info_changeset <-
+           Pleroma.User.Info.remove_pinnned_activity(user.info, activity),
+         changeset <-
+           Ecto.Changeset.change(user) |> Ecto.Changeset.put_embed(:info, info_changeset),
+         {:ok, _user} <- User.update_and_set_cache(changeset) do
+      {:ok, activity}
+    else
+      %{errors: [pinned_activities: {err, _}]} ->
+        {:error, err}
+
+      _ ->
+        {:error, "Could not unpin"}
+    end
+  end
 end

lib/pleroma/web/common_api/utils.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.CommonAPI.Utils do
@@ -116,16 +116,18 @@ defmodule Pleroma.Web.CommonAPI.Utils do
     Enum.join([text | attachment_text], "<br>")
   end
 
+  def format_input(text, mentions, tags, format, options \\ [])
+
   @doc """
   Formatting text to plain text.
   """
-  def format_input(text, mentions, tags, "text/plain") do
+  def format_input(text, mentions, tags, "text/plain", options) do
     text
     |> Formatter.html_escape("text/plain")
     |> String.replace(~r/\r?\n/, "<br>")
     |> (&{[], &1}).()
     |> Formatter.add_links()
-    |> Formatter.add_user_links(mentions)
+    |> Formatter.add_user_links(mentions, options[:user_links] || [])
     |> Formatter.add_hashtag_links(tags)
     |> Formatter.finalize()
   end
@@ -133,26 +135,24 @@ defmodule Pleroma.Web.CommonAPI.Utils do
   @doc """
   Formatting text to html.
   """
-  def format_input(text, mentions, _tags, "text/html") do
+  def format_input(text, mentions, _tags, "text/html", options) do
     text
     |> Formatter.html_escape("text/html")
-    |> String.replace(~r/\r?\n/, "<br>")
     |> (&{[], &1}).()
-    |> Formatter.add_user_links(mentions)
+    |> Formatter.add_user_links(mentions, options[:user_links] || [])
     |> Formatter.finalize()
   end
 
   @doc """
   Formatting text to markdown.
   """
-  def format_input(text, mentions, tags, "text/markdown") do
+  def format_input(text, mentions, tags, "text/markdown", options) do
     text
     |> Formatter.mentions_escape(mentions)
     |> Earmark.as_html!()
     |> Formatter.html_escape("text/html")
-    |> String.replace(~r/\r?\n/, "")
     |> (&{[], &1}).()
-    |> Formatter.add_user_links(mentions)
+    |> Formatter.add_user_links(mentions, options[:user_links] || [])
     |> Formatter.add_hashtag_links(tags)
     |> Formatter.finalize()
   end

lib/pleroma/web/controller_helper.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.ControllerHelper do

lib/pleroma/web/endpoint.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.Endpoint do

lib/pleroma/web/federator/federator.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.Federator do
@@ -17,7 +17,6 @@ defmodule Pleroma.Web.Federator do
 
   @websub Application.get_env(:pleroma, :websub)
   @ostatus Application.get_env(:pleroma, :ostatus)
-  @max_jobs 20
 
   def init(args) do
     {:ok, args}
@@ -168,7 +167,7 @@ defmodule Pleroma.Web.Federator do
   end
 
   def maybe_start_job(running_jobs, queue) do
-    if :sets.size(running_jobs) < @max_jobs && queue != [] do
+    if :sets.size(running_jobs) < Pleroma.Config.get([__MODULE__, :max_jobs]) && queue != [] do
       {{type, payload}, queue} = queue_pop(queue)
       {:ok, pid} = Task.start(fn -> handle(type, payload) end)
       mref = Process.monitor(pid)

lib/pleroma/web/federator/retry_queue.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.Federator.RetryQueue do
@@ -7,20 +7,28 @@ defmodule Pleroma.Web.Federator.RetryQueue do
 
   require Logger
 
-  # initial timeout, 5 min
-  @initial_timeout 30_000
-  @max_retries 5
-
   def init(args) do
-    {:ok, args}
+    queue_table = :ets.new(:pleroma_retry_queue, [:bag, :protected])
+
+    {:ok, %{args | queue_table: queue_table, running_jobs: :sets.new()}}
   end
 
   def start_link() do
-    enabled = Pleroma.Config.get([:retry_queue, :enabled], false)
+    enabled =
+      if Mix.env() == :test, do: true, else: Pleroma.Config.get([__MODULE__, :enabled], false)
 
     if enabled do
       Logger.info("Starting retry queue")
-      GenServer.start_link(__MODULE__, %{delivered: 0, dropped: 0}, name: __MODULE__)
+
+      linkres =
+        GenServer.start_link(
+          __MODULE__,
+          %{delivered: 0, dropped: 0, queue_table: nil, running_jobs: nil},
+          name: __MODULE__
+        )
+
+      maybe_kickoff_timer()
+      linkres
     else
       Logger.info("Retry queue disabled")
       :ignore
@@ -31,24 +39,133 @@ defmodule Pleroma.Web.Federator.RetryQueue do
     GenServer.cast(__MODULE__, {:maybe_enqueue, data, transport, retries + 1})
   end
 
+  def get_stats() do
+    GenServer.call(__MODULE__, :get_stats)
+  end
+
+  def reset_stats() do
+    GenServer.call(__MODULE__, :reset_stats)
+  end
+
   def get_retry_params(retries) do
-    if retries > @max_retries do
+    if retries > Pleroma.Config.get([__MODULE__, :max_retries]) do
       {:drop, "Max retries reached"}
     else
       {:retry, growth_function(retries)}
     end
   end
 
-  def handle_cast({:maybe_enqueue, data, transport, retries}, %{dropped: drop_count} = state) do
+  def get_retry_timer_interval() do
+    Pleroma.Config.get([:retry_queue, :interval], 1000)
+  end
+
+  defp ets_count_expires(table, current_time) do
+    :ets.select_count(
+      table,
+      [
+        {
+          {:"$1", :"$2"},
+          [{:"=<", :"$1", {:const, current_time}}],
+          [true]
+        }
+      ]
+    )
+  end
+
+  defp ets_pop_n_expired(table, current_time, desired) do
+    {popped, _continuation} =
+      :ets.select(
+        table,
+        [
+          {
+            {:"$1", :"$2"},
+            [{:"=<", :"$1", {:const, current_time}}],
+            [:"$_"]
+          }
+        ],
+        desired
+      )
+
+    popped
+    |> Enum.each(fn e ->
+      :ets.delete_object(table, e)
+    end)
+
+    popped
+  end
+
+  def maybe_start_job(running_jobs, queue_table) do
+    # we don't want to hit the ets or the DateTime more times than we have to
+    # could optimize slightly further by not using the count, and instead grabbing
+    # up to N objects early...
+    current_time = DateTime.to_unix(DateTime.utc_now())
+    n_running_jobs = :sets.size(running_jobs)
+
+    if n_running_jobs < Pleroma.Config.get([__MODULE__, :max_jobs]) do
+      n_ready_jobs = ets_count_expires(queue_table, current_time)
+
+      if n_ready_jobs > 0 do
+        # figure out how many we could start
+        available_job_slots = Pleroma.Config.get([__MODULE__, :max_jobs]) - n_running_jobs
+        start_n_jobs(running_jobs, queue_table, current_time, available_job_slots)
+      else
+        running_jobs
+      end
+    else
+      running_jobs
+    end
+  end
+
+  defp start_n_jobs(running_jobs, _queue_table, _current_time, 0) do
+    running_jobs
+  end
+
+  defp start_n_jobs(running_jobs, queue_table, current_time, available_job_slots)
+       when available_job_slots > 0 do
+    candidates = ets_pop_n_expired(queue_table, current_time, available_job_slots)
+
+    candidates
+    |> List.foldl(running_jobs, fn {_, e}, rj ->
+      {:ok, pid} = Task.start(fn -> worker(e) end)
+      mref = Process.monitor(pid)
+      :sets.add_element(mref, rj)
+    end)
+  end
+
+  def worker({:send, data, transport, retries}) do
+    case transport.publish_one(data) do
+      {:ok, _} ->
+        GenServer.cast(__MODULE__, :inc_delivered)
+        :delivered
+
+      {:error, _reason} ->
+        enqueue(data, transport, retries)
+        :retry
+    end
+  end
+
+  def handle_call(:get_stats, _from, %{delivered: delivery_count, dropped: drop_count} = state) do
+    {:reply, %{delivered: delivery_count, dropped: drop_count}, state}
+  end
+
+  def handle_call(:reset_stats, _from, %{delivered: delivery_count, dropped: drop_count} = state) do
+    {:reply, %{delivered: delivery_count, dropped: drop_count},
+     %{state | delivered: 0, dropped: 0}}
+  end
+
+  def handle_cast(:reset_stats, state) do
+    {:noreply, %{state | delivered: 0, dropped: 0}}
+  end
+
+  def handle_cast(
+        {:maybe_enqueue, data, transport, retries},
+        %{dropped: drop_count, queue_table: queue_table, running_jobs: running_jobs} = state
+      ) do
     case get_retry_params(retries) do
       {:retry, timeout} ->
-        Process.send_after(
+        :ets.insert(queue_table, {timeout, {:send, data, transport, retries}})
-          __MODULE__,
+        running_jobs = maybe_start_job(running_jobs, queue_table)
-          {:send, data, transport, retries},
+        {:noreply, %{state | running_jobs: running_jobs}}
-          timeout
-        )
-
-        {:noreply, state}
 
       {:drop, message} ->
         Logger.debug(message)
@@ -56,6 +173,20 @@ defmodule Pleroma.Web.Federator.RetryQueue do
     end
   end
 
+  def handle_cast(:kickoff_timer, state) do
+    retry_interval = get_retry_timer_interval()
+    Process.send_after(__MODULE__, :retry_timer_run, retry_interval)
+    {:noreply, state}
+  end
+
+  def handle_cast(:inc_delivered, %{delivered: delivery_count} = state) do
+    {:noreply, %{state | delivered: delivery_count + 1}}
+  end
+
+  def handle_cast(:inc_dropped, %{dropped: drop_count} = state) do
+    {:noreply, %{state | dropped: drop_count + 1}}
+  end
+
   def handle_info({:send, data, transport, retries}, %{delivered: delivery_count} = state) do
     case transport.publish_one(data) do
       {:ok, _} ->
@@ -67,12 +198,40 @@ defmodule Pleroma.Web.Federator.RetryQueue do
     end
   end
 
+  def handle_info(
+        :retry_timer_run,
+        %{queue_table: queue_table, running_jobs: running_jobs} = state
+      ) do
+    maybe_kickoff_timer()
+    running_jobs = maybe_start_job(running_jobs, queue_table)
+    {:noreply, %{state | running_jobs: running_jobs}}
+  end
+
+  def handle_info({:DOWN, ref, :process, _pid, _reason}, state) do
+    %{running_jobs: running_jobs, queue_table: queue_table} = state
+    running_jobs = :sets.del_element(ref, running_jobs)
+    running_jobs = maybe_start_job(running_jobs, queue_table)
+    {:noreply, %{state | running_jobs: running_jobs}}
+  end
+
   def handle_info(unknown, state) do
     Logger.debug("RetryQueue: don't know what to do with #{inspect(unknown)}, ignoring")
     {:noreply, state}
   end
 
-  defp growth_function(retries) do
+  if Mix.env() == :test do
-    round(@initial_timeout * :math.pow(retries, 3))
+    defp growth_function(_retries) do
+      _shutit = Pleroma.Config.get([__MODULE__, :initial_timeout])
+      DateTime.to_unix(DateTime.utc_now()) - 1
+    end
+  else
+    defp growth_function(retries) do
+      round(Pleroma.Config.get([__MODULE__, :initial_timeout]) * :math.pow(retries, 3)) +
+        DateTime.to_unix(DateTime.utc_now())
+    end
+  end
+
+  defp maybe_kickoff_timer() do
+    GenServer.cast(__MODULE__, :kickoff_timer)
   end
 end

lib/pleroma/web/gettext.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.Gettext do

lib/pleroma/web/http_signatures/http_signatures.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 # https://tools.ietf.org/html/draft-cavage-http-signatures-08