Merge branch 'feature/jobs' into 'develop'

Job Queue

See merge request pleroma/pleroma!732

test/jobs_test.exs

@@ -0,0 +1,83 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.JobsTest do
+  use ExUnit.Case, async: true
+
+  alias Pleroma.Jobs
+  alias Jobs.WorkerMock
+
+  setup do
+    state = %{
+      queues: Enum.into([Jobs.create_queue(:testing)], %{}),
+      refs: %{}
+    }
+
+    [state: state]
+  end
+
+  test "creates queue" do
+    queue = Jobs.create_queue(:foobar)
+
+    assert {:foobar, set} = queue
+    assert :set == elem(set, 0) |> elem(0)
+  end
+
+  test "enqueues an element according to priority" do
+    queue = [%{item: 1, priority: 2}]
+
+    new_queue = Jobs.enqueue_sorted(queue, 2, 1)
+    assert new_queue == [%{item: 2, priority: 1}, %{item: 1, priority: 2}]
+
+    new_queue = Jobs.enqueue_sorted(queue, 2, 3)
+    assert new_queue == [%{item: 1, priority: 2}, %{item: 2, priority: 3}]
+  end
+
+  test "pop first item" do
+    queue = [%{item: 2, priority: 1}, %{item: 1, priority: 2}]
+
+    assert {2, [%{item: 1, priority: 2}]} = Jobs.queue_pop(queue)
+  end
+
+  test "enqueue a job", %{state: state} do
+    assert {:noreply, new_state} =
+             Jobs.handle_cast({:enqueue, :testing, WorkerMock, [:test_job, :foo, :bar], 3}, state)
+
+    assert %{queues: %{testing: {running_jobs, []}}, refs: _} = new_state
+    assert :sets.size(running_jobs) == 1
+    assert [ref] = :sets.to_list(running_jobs)
+    assert %{refs: %{^ref => :testing}} = new_state
+  end
+
+  test "max jobs setting", %{state: state} do
+    max_jobs = Pleroma.Config.get([Jobs, :testing, :max_jobs])
+
+    {:noreply, state} =
+      Enum.reduce(1..(max_jobs + 1), {:noreply, state}, fn _, {:noreply, state} ->
+        Jobs.handle_cast({:enqueue, :testing, WorkerMock, [:test_job, :foo, :bar], 3}, state)
+      end)
+
+    assert %{
+             queues: %{
+               testing:
+                 {running_jobs, [%{item: {WorkerMock, [:test_job, :foo, :bar]}, priority: 3}]}
+             }
+           } = state
+
+    assert :sets.size(running_jobs) == max_jobs
+  end
+
+  test "remove job after it finished", %{state: state} do
+    {:noreply, new_state} =
+      Jobs.handle_cast({:enqueue, :testing, WorkerMock, [:test_job, :foo, :bar], 3}, state)
+
+    %{queues: %{testing: {running_jobs, []}}} = new_state
+    [ref] = :sets.to_list(running_jobs)
+
+    assert {:noreply, %{queues: %{testing: {running_jobs, []}}, refs: %{}}} =
+             Jobs.handle_info({:DOWN, ref, :process, nil, nil}, new_state)
+
+    assert :sets.size(running_jobs) == 0
+  end
+end

test/support/jobs_worker_mock.ex

@@ -0,0 +1,19 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Jobs.WorkerMock do
+  require Logger
+
+  def perform(:test_job, arg, arg2) do
+    Logger.debug({:perform, :test_job, arg, arg2})
+  end
+
+  def perform(:test_job, payload) do
+    Logger.debug({:perform, :test_job, payload})
+  end
+
+  def test_job(payload) do
+    Pleroma.Jobs.enqueue(:testing, __MODULE__, [:test_job, payload])
+  end
+end

test/web/federator_test.exs

@@ -15,22 +15,6 @@ defmodule Pleroma.Web.FederatorTest do
     :ok
   end
 
-  test "enqueues an element according to priority" do
-    queue = [%{item: 1, priority: 2}]
-
-    new_queue = Federator.enqueue_sorted(queue, 2, 1)
-    assert new_queue == [%{item: 2, priority: 1}, %{item: 1, priority: 2}]
-
-    new_queue = Federator.enqueue_sorted(queue, 2, 3)
-    assert new_queue == [%{item: 1, priority: 2}, %{item: 2, priority: 3}]
-  end
-
-  test "pop first item" do
-    queue = [%{item: 2, priority: 1}, %{item: 1, priority: 2}]
-
-    assert {2, [%{item: 1, priority: 2}]} = Federator.queue_pop(queue)
-  end
-
   describe "Publish an activity" do
     setup do
       user = insert(:user)
@@ -50,7 +34,7 @@ defmodule Pleroma.Web.FederatorTest do
       relay_mock: relay_mock
     } do
       with_mocks([relay_mock]) do
-        Federator.handle(:publish, activity)
+        Federator.publish(activity)
       end
 
       assert_received :relay_publish
@@ -63,7 +47,7 @@ defmodule Pleroma.Web.FederatorTest do
       Pleroma.Config.put([:instance, :allow_relay], false)
 
       with_mocks([relay_mock]) do
-        Federator.handle(:publish, activity)
+        Federator.publish(activity)
       end
 
       refute_received :relay_publish
@@ -104,11 +88,9 @@ defmodule Pleroma.Web.FederatorTest do
       {:ok, _activity} =
         CommonAPI.post(user, %{"status" => "HI @nick1@domain.com, @nick2@domain2.com!"})
 
-      assert called(
-               Federator.enqueue(:publish_single_ap, %{inbox: inbox1, unreachable_since: dt})
-             )
+      assert called(Federator.publish_single_ap(%{inbox: inbox1, unreachable_since: dt}))
 
-      refute called(Federator.enqueue(:publish_single_ap, %{inbox: inbox2}))
+      refute called(Federator.publish_single_ap(%{inbox: inbox2}))
     end
 
     test_with_mock "it federates only to reachable instances via Websub",
@@ -140,13 +122,13 @@ defmodule Pleroma.Web.FederatorTest do
       {:ok, _activity} = CommonAPI.post(user, %{"status" => "HI"})
 
       assert called(
-               Federator.enqueue(:publish_single_websub, %{
+               Federator.publish_single_websub(%{
                  callback: sub2.callback,
                  unreachable_since: dt
                })
              )
 
-      refute called(Federator.enqueue(:publish_single_websub, %{callback: sub1.callback}))
+      refute called(Federator.publish_single_websub(%{callback: sub1.callback}))
     end
 
     test_with_mock "it federates only to reachable instances via Salmon",
@@ -180,13 +162,13 @@ defmodule Pleroma.Web.FederatorTest do
         CommonAPI.post(user, %{"status" => "HI @nick1@domain.com, @nick2@domain2.com!"})
 
       assert called(
-               Federator.enqueue(:publish_single_salmon, %{
+               Federator.publish_single_salmon(%{
                  recipient: remote_user2,
                  unreachable_since: dt
                })
              )
 
-      refute called(Federator.enqueue(:publish_single_websub, %{recipient: remote_user1}))
+      refute called(Federator.publish_single_websub(%{recipient: remote_user1}))
     end
   end
 
@@ -206,7 +188,7 @@ defmodule Pleroma.Web.FederatorTest do
         "to" => ["https://www.w3.org/ns/activitystreams#Public"]
       }
 
-      {:ok, _activity} = Federator.handle(:incoming_ap_doc, params)
+      {:ok, _activity} = Federator.incoming_ap_doc(params)
     end
 
     test "rejects incoming AP docs with incorrect origin" do
@@ -224,7 +206,7 @@ defmodule Pleroma.Web.FederatorTest do
         "to" => ["https://www.w3.org/ns/activitystreams#Public"]
       }
 
-      :error = Federator.handle(:incoming_ap_doc, params)
+      :error = Federator.incoming_ap_doc(params)
     end
   end
 end