diff --git a/lib/mix/tasks/pleroma/test_runner.ex b/lib/mix/tasks/pleroma/test_runner.ex
index 69fefb001..d9cf0d445 100644
--- a/lib/mix/tasks/pleroma/test_runner.ex
+++ b/lib/mix/tasks/pleroma/test_runner.ex
@@ -4,7 +4,7 @@ defmodule Mix.Tasks.Pleroma.TestRunner do
   use Mix.Task
 
   def run(args \\ []) do
-    case System.cmd("mix", ["test"] ++ args, into: IO.stream(:stdio, :line)) do
+    case System.cmd("mix", ["test", "--warnings-as-errors"] ++ args, into: IO.stream(:stdio, :line)) do
       {_, 0} ->
         :ok
 
diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex
index 497623ee1..fd3c66c63 100644
--- a/lib/pleroma/application.ex
+++ b/lib/pleroma/application.ex
@@ -43,9 +43,6 @@ defmodule Pleroma.Application do
     # every time the application is restarted, so we disable module
     # conflicts at runtime
     Code.compiler_options(ignore_module_conflict: true)
-    # Disable warnings_as_errors at runtime, it breaks Phoenix live reload
-    # due to protocol consolidation warnings
-    Code.compiler_options(warnings_as_errors: false)
     Pleroma.Telemetry.Logger.attach()
     Config.Holder.save_default()
     Pleroma.HTML.compile_scrubbers()
@@ -93,6 +90,21 @@ defmodule Pleroma.Application do
       end
     end
 
+    # Disable warnings_as_errors at runtime, it breaks Phoenix live reload
+    # due to protocol consolidation warnings
+    # :warnings_as_errors is deprecated via Code.compiler_options/2 since 1.18
+    if elixir_version = System.version() do
+      [major, minor] =
+        elixir_version
+        |> String.split(".")
+        |> Enum.map(&String.to_integer/1)
+        |> Enum.take(2)
+
+        if major == 1 and minor < 18 do
+          Code.compiler_options(warnings_as_errors: false)
+        end
+    end
+
     # Define workers and child supervisors to be supervised
     children =
       [
diff --git a/mix.exs b/mix.exs
index 808a2b12c..dc6c2492f 100644
--- a/mix.exs
+++ b/mix.exs
@@ -236,7 +236,7 @@ defmodule Pleroma.Mixfile do
       "ecto.rollback": ["pleroma.ecto.rollback"],
       "ecto.setup": ["ecto.create", "ecto.migrate", "run priv/repo/seeds.exs"],
       "ecto.reset": ["ecto.drop", "ecto.setup"],
-      test: ["ecto.create --quiet", "ecto.migrate", "test"],
+      test: ["ecto.create --quiet", "ecto.migrate", "test --warnings-as-errors"],
       docs: ["pleroma.docs", "docs"],
       analyze: ["credo --strict --only=warnings,todo,fixme,consistency,readability"],
       copyright: &add_copyright/1,
diff --git a/test/test_helper.exs b/test/test_helper.exs
index 94661353b..dc6c05a74 100644
--- a/test/test_helper.exs
+++ b/test/test_helper.exs
@@ -2,8 +2,6 @@
 # Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
-Code.put_compiler_option(:warnings_as_errors, true)
-
 ExUnit.configure(capture_log: true, max_cases: System.schedulers_online())
 
 ExUnit.start(exclude: [:federated])