don't fail on url fetch
This commit is contained in:
parent
d48755791d
commit
03d06062ab
4 changed files with 49 additions and 38 deletions
|
|
@ -3,6 +3,8 @@
|
|||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
defmodule Pleroma.Web.RichMedia.Parser do
|
||||
require Logger
|
||||
|
||||
defp parsers do
|
||||
Pleroma.Config.get([:rich_media, :parsers])
|
||||
end
|
||||
|
|
@ -10,18 +12,19 @@ defmodule Pleroma.Web.RichMedia.Parser do
|
|||
def parse(nil), do: {:error, "No URL provided"}
|
||||
|
||||
if Pleroma.Config.get(:env) == :test do
|
||||
@spec parse(String.t()) :: {:ok, map()} | {:error, any()}
|
||||
def parse(url), do: parse_url(url)
|
||||
else
|
||||
@spec parse(String.t()) :: {:ok, map()} | {:error, any()}
|
||||
def parse(url) do
|
||||
try do
|
||||
Cachex.fetch!(:rich_media_cache, url, fn _ ->
|
||||
{:commit, parse_url(url)}
|
||||
end)
|
||||
|> set_ttl_based_on_image(url)
|
||||
rescue
|
||||
e ->
|
||||
{:error, "Cachex error: #{inspect(e)}"}
|
||||
end
|
||||
Cachex.fetch!(:rich_media_cache, url, fn _ ->
|
||||
with {:ok, data} <- parse_url(url) do
|
||||
{:commit, {:ok, data}}
|
||||
else
|
||||
error -> {:ignore, error}
|
||||
end
|
||||
end)
|
||||
|> set_ttl_based_on_image(url)
|
||||
end
|
||||
end
|
||||
|
||||
|
|
@ -47,9 +50,11 @@ defmodule Pleroma.Web.RichMedia.Parser do
|
|||
config :pleroma, :rich_media,
|
||||
ttl_setters: [MyModule]
|
||||
"""
|
||||
@spec set_ttl_based_on_image({:ok, map()} | {:error, any()}, String.t()) ::
|
||||
{:ok, map()} | {:error, any()}
|
||||
def set_ttl_based_on_image({:ok, data}, url) do
|
||||
with {:ok, nil} <- Cachex.ttl(:rich_media_cache, url),
|
||||
ttl when is_number(ttl) <- get_ttl_from_image(data, url) do
|
||||
{:ok, ttl} when is_number(ttl) <- get_ttl_from_image(data, url) do
|
||||
Cachex.expire_at(:rich_media_cache, url, ttl * 1000)
|
||||
{:ok, data}
|
||||
else
|
||||
|
|
@ -58,8 +63,14 @@ defmodule Pleroma.Web.RichMedia.Parser do
|
|||
end
|
||||
end
|
||||
|
||||
def set_ttl_based_on_image({:error, _} = error, _) do
|
||||
Logger.error("parsing error: #{inspect(error)}")
|
||||
error
|
||||
end
|
||||
|
||||
defp get_ttl_from_image(data, url) do
|
||||
Pleroma.Config.get([:rich_media, :ttl_setters])
|
||||
[:rich_media, :ttl_setters]
|
||||
|> Pleroma.Config.get()
|
||||
|> Enum.reduce({:ok, nil}, fn
|
||||
module, {:ok, _ttl} ->
|
||||
module.ttl(data, url)
|
||||
|
|
@ -70,23 +81,16 @@ defmodule Pleroma.Web.RichMedia.Parser do
|
|||
end
|
||||
|
||||
defp parse_url(url) do
|
||||
try do
|
||||
{:ok, %Tesla.Env{body: html}} = Pleroma.Web.RichMedia.Helpers.rich_media_get(url)
|
||||
|
||||
with {:ok, %Tesla.Env{body: html}} <- Pleroma.Web.RichMedia.Helpers.rich_media_get(url),
|
||||
{:ok, html} <- Floki.parse_document(html) do
|
||||
html
|
||||
|> parse_html()
|
||||
|> maybe_parse()
|
||||
|> Map.put("url", url)
|
||||
|> clean_parsed_data()
|
||||
|> check_parsed_data()
|
||||
rescue
|
||||
e ->
|
||||
{:error, "Parsing error: #{inspect(e)} #{inspect(__STACKTRACE__)}"}
|
||||
end
|
||||
end
|
||||
|
||||
defp parse_html(html), do: Floki.parse_document!(html)
|
||||
|
||||
defp maybe_parse(html) do
|
||||
Enum.reduce_while(parsers(), %{}, fn parser, acc ->
|
||||
case parser.parse(html, acc) do
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue