From 535fddd2864f3ed8eebc24cbaf0e5b04ec6f4dbe Mon Sep 17 00:00:00 2001 From: Rin Toshaka Date: Sun, 30 Dec 2018 19:33:36 +0100 Subject: [PATCH] Friendship ended with Postgresql now Cachex is my best friend --- lib/pleroma/application.ex | 10 +++ lib/pleroma/object.ex | 34 +++++++- lib/pleroma/web/common_api/common_api.ex | 3 +- lib/pleroma/web/common_api/utils.ex | 81 +------------------ .../web/mastodon_api/views/status_view.ex | 3 +- .../web/twitter_api/views/activity_view.ex | 4 +- 6 files changed, 49 insertions(+), 86 deletions(-) diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 36a3694f2..4b997c048 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -53,6 +53,16 @@ defmodule Pleroma.Application do ], id: :cachex_object ), + worker( + Cachex, + [ + :scrubber_cache, + [ + limit: 2500 + ] + ], + id: :cachex_scrubber + ), worker( Cachex, [ diff --git a/lib/pleroma/object.ex b/lib/pleroma/object.ex index cc4a2181a..e148c1d75 100644 --- a/lib/pleroma/object.ex +++ b/lib/pleroma/object.ex @@ -4,7 +4,7 @@ defmodule Pleroma.Object do use Ecto.Schema - alias Pleroma.{Repo, Object, User, Activity} + alias Pleroma.{Repo, Object, User, Activity, HTML} import Ecto.{Query, Changeset} schema "objects" do @@ -73,4 +73,36 @@ defmodule Pleroma.Object do {:ok, object} end end + + def get_cached_scrubbed_html(content, scrubbers, object) do + key = "#{generate_scrubber_signature(scrubbers)}|#{object.id}" + Cachex.fetch!(:scrubber_cache, key, fn(_key) -> ensure_scrubbed_html(content, scrubbers) end ) + end + + def get_cached_stripped_html(content, object) do + get_cached_scrubbed_html(content, HtmlSanitizeEx.Scrubber.StripTags, object) + end + + def ensure_scrubbed_html( + content, + scrubbers + ) do + {:commit, HTML.filter_tags(content, scrubbers)} + end + + defp generate_scrubber_signature(scrubber) when is_atom(scrubber) do + generate_scrubber_signature([scrubber]) + end + + defp generate_scrubber_signature(scrubbers) do + Enum.reduce(scrubbers, "", fn scrubber, signature -> + # If a scrubber does not have a version(e.g HtmlSanitizeEx.Scrubber) it is assumed it is always 0) + version = if Kernel.function_exported?(scrubber, :version, 0) do + scrubber.version + else + 0 + end + "#{signature}#{to_string(scrubber)}#{version}" + end) + end end diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex index 06d44451e..5e5821561 100644 --- a/lib/pleroma/web/common_api/common_api.ex +++ b/lib/pleroma/web/common_api/common_api.ex @@ -128,8 +128,7 @@ defmodule Pleroma.Web.CommonAPI do |> Enum.reduce(%{}, fn {name, file}, acc -> Map.put(acc, name, "#{Pleroma.Web.Endpoint.static_url()}#{file}") end) - ), - object <- Map.put(object, "scrubber_cache", %{}) do + ) do res = ActivityPub.create(%{ to: to, diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex index 759bd62af..813eb4093 100644 --- a/lib/pleroma/web/common_api/utils.ex +++ b/lib/pleroma/web/common_api/utils.ex @@ -5,7 +5,7 @@ defmodule Pleroma.Web.CommonAPI.Utils do alias Calendar.Strftime alias Comeonin.Pbkdf2 - alias Pleroma.{Activity, Formatter, Object, Repo, HTML} + alias Pleroma.{Activity, Formatter, Object, Repo} alias Pleroma.User alias Pleroma.Web alias Pleroma.Web.ActivityPub.Utils @@ -262,83 +262,4 @@ defmodule Pleroma.Web.CommonAPI.Utils do end) end - def get_scrubbed_html_for_object(content, scrubber, activity) when is_atom(scrubber) do - get_scrubbed_html_for_object(content, [scrubber], activity) - end - @doc """ - Get sanitized HTML from cache, or scrub it and save to cache. - """ - def get_scrubbed_html_for_object( - content, - scrubbers, - %{data: %{"object" => object}} = activity - ) do - scrubber_cache = - if is_list(object["scrubber_cache"]) do - object["scrubber_cache"] - else - [] - end - - signature = generate_scrubber_signature(scrubbers) - - {new_scrubber_cache, scrubbed_html} = - Enum.map_reduce(scrubber_cache, nil, fn - entry, content -> - if Map.keys(entry["scrubbers"]) == Map.keys(signature) do - if entry["scrubbers"] == signature do - {entry, entry["content"]} - else - # Remove the entry if scrubber version is outdated - {nil, nil} - end - else - {entry, content} - end - end) - - # Remove nil objects - new_scrubber_cache = Enum.reject(new_scrubber_cache, &is_nil/1) - - if scrubbed_html == nil or new_scrubber_cache != scrubber_cache do - scrubbed_html = HTML.filter_tags(content, scrubbers) - - new_scrubber_cache = [ - %{:scrubbers => signature, :content => scrubbed_html} | new_scrubber_cache - ] - - update_scrubber_cache(activity, new_scrubber_cache) - scrubbed_html - else - scrubbed_html - end - end - - defp generate_scrubber_signature(scrubbers) do - Enum.reduce(scrubbers, %{}, fn scrubber, signature -> - Map.put( - signature, - to_string(scrubber), - # If a scrubber does not have a version(e.g HtmlSanitizeEx.Scrubber) it is assumed it is always 0) - if Kernel.function_exported?(scrubber, :version, 0) do - scrubber.version - else - 0 - end - ) - end) - end - - defp update_scrubber_cache(activity, scrubber_cache) do - cng = - Object.change(activity, %{ - data: Kernel.put_in(activity.data, ["object", "scrubber_cache"], scrubber_cache) - }) - - {:ok, _struct} = Repo.update(cng) - end - - def get_stripped_html_for_object(content, activity) do - get_scrubbed_html_for_object(content, [HtmlSanitizeEx.Scrubber.StripTags], activity) - end end diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index 05ed602d5..8a57a233a 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -9,6 +9,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do alias Pleroma.HTML alias Pleroma.Repo alias Pleroma.User + alias Pleroma.Object alias Pleroma.Web.CommonAPI.Utils alias Pleroma.Web.MediaProxy alias Pleroma.Web.MastodonAPI.AccountView @@ -120,7 +121,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do content = object |> render_content() - |> Utils.get_scrubbed_html_for_object(User.html_filter_policy(opts[:for]), activity) + |> Object.get_cached_scrubbed_html(User.html_filter_policy(opts[:for]), activity) %{ id: to_string(activity.id), diff --git a/lib/pleroma/web/twitter_api/views/activity_view.ex b/lib/pleroma/web/twitter_api/views/activity_view.ex index 7d0dea8c2..4c29e03ce 100644 --- a/lib/pleroma/web/twitter_api/views/activity_view.ex +++ b/lib/pleroma/web/twitter_api/views/activity_view.ex @@ -244,14 +244,14 @@ defmodule Pleroma.Web.TwitterAPI.ActivityView do html = content - |> Utils.get_scrubbed_html_for_object(User.html_filter_policy(opts[:for]), activity) + |> Object.get_cached_scrubbed_html(User.html_filter_policy(opts[:for]), activity) |> Formatter.emojify(object["emoji"]) text = if content do content |> String.replace(~r//, "\n") - |> Utils.get_stripped_html_for_object(activity) + |> Object.get_cached_stripped_html(activity) end reply_parent = Activity.get_in_reply_to_activity(activity)