From 86f90c0a54d8f0ee6e6a787e8d28ebc53ff13b05 Mon Sep 17 00:00:00 2001 From: Alex S Date: Wed, 3 Apr 2019 20:51:09 +0700 Subject: [PATCH 01/26] adding indexes to oauth_tokens table --- .../20190403131720_add_oauth_token_indexes.exs | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 priv/repo/migrations/20190403131720_add_oauth_token_indexes.exs diff --git a/priv/repo/migrations/20190403131720_add_oauth_token_indexes.exs b/priv/repo/migrations/20190403131720_add_oauth_token_indexes.exs new file mode 100644 index 000000000..ebcd29389 --- /dev/null +++ b/priv/repo/migrations/20190403131720_add_oauth_token_indexes.exs @@ -0,0 +1,9 @@ +defmodule Pleroma.Repo.Migrations.AddOauthTokenIndexes do + use Ecto.Migration + + def change do + create(unique_index(:oauth_tokens, [:token])) + create(index(:oauth_tokens, [:app_id])) + create(index(:oauth_tokens, [:user_id])) + end +end From af0065a71feb470bd69bec36999bc40a662e3e83 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Thu, 4 Apr 2019 09:07:25 +0200 Subject: [PATCH 02/26] mastodon_api_controller.ex: Add pleroma-tan to initial_state --- lib/pleroma/web/mastodon_api/mastodon_api_controller.ex | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index 0de2cca4e..89fd7629a 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -1121,7 +1121,8 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do auto_play_gif: false, display_sensitive_media: false, reduce_motion: false, - max_toot_chars: limit + max_toot_chars: limit, + mascot: "/images/pleroma-fox-tan-smol.png" }, rights: %{ delete_others_notice: present?(user.info.is_moderator), From b655a8ea839d19443f44ff5b300a069d88ec7d58 Mon Sep 17 00:00:00 2001 From: href Date: Wed, 6 Feb 2019 10:33:05 +0100 Subject: [PATCH 03/26] Add recon --- mix.exs | 9 ++++++++- mix.lock | 8 ++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/mix.exs b/mix.exs index 88da6332a..8cb389248 100644 --- a/mix.exs +++ b/mix.exs @@ -94,7 +94,14 @@ defmodule Pleroma.Mixfile do {:auto_linker, git: "https://git.pleroma.social/pleroma/auto_linker.git", ref: "479dd343f4e563ff91215c8275f3b5c67e032850"}, - {:pleroma_job_queue, "~> 0.2.0"} + {:pleroma_job_queue, "~> 0.2.0"}, + {:telemetry, "~> 0.3"}, + {:prometheus_ex, "~> 3.0"}, + {:prometheus_plugs, "~> 1.1"}, + {:prometheus_phoenix, "~> 1.2"}, + {:prometheus_ecto, "~> 1.4"}, + {:prometheus_process_collector, "~> 1.4"}, + {:recon, github: "ferd/recon"} ] end diff --git a/mix.lock b/mix.lock index 9c454446a..0ece4b353 100644 --- a/mix.lock +++ b/mix.lock @@ -1,4 +1,5 @@ %{ + "accept": {:hex, :accept, "0.3.5", "b33b127abca7cc948bbe6caa4c263369abf1347cfa9d8e699c6d214660f10cd1", [:rebar3], [], "hexpm"}, "auto_linker": {:git, "https://git.pleroma.social/pleroma/auto_linker.git", "479dd343f4e563ff91215c8275f3b5c67e032850", [ref: "479dd343f4e563ff91215c8275f3b5c67e032850"]}, "base64url": {:hex, :base64url, "0.0.1", "36a90125f5948e3afd7be97662a1504b934dd5dac78451ca6e9abf85a10286be", [:rebar], [], "hexpm"}, "bunt": {:hex, :bunt, "0.2.0", "951c6e801e8b1d2cbe58ebbd3e616a869061ddadcc4863d0a2182541acae9a38", [:mix], [], "hexpm"}, @@ -57,7 +58,14 @@ "poison": {:hex, :poison, "3.1.0", "d9eb636610e096f86f25d9a46f35a9facac35609a7591b3be3326e99a0484665", [:mix], [], "hexpm"}, "poolboy": {:hex, :poolboy, "1.5.2", "392b007a1693a64540cead79830443abf5762f5d30cf50bc95cb2c1aaafa006b", [:rebar3], [], "hexpm"}, "postgrex": {:hex, :postgrex, "0.14.1", "63247d4a5ad6b9de57a0bac5d807e1c32d41e39c04b8a4156a26c63bcd8a2e49", [:mix], [{:connection, "~> 1.0", [hex: :connection, repo: "hexpm", optional: false]}, {:db_connection, "~> 2.0", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm"}, + "prometheus": {:hex, :prometheus, "4.2.2", "a830e77b79dc6d28183f4db050a7cac926a6c58f1872f9ef94a35cd989aceef8", [:mix, :rebar3], [], "hexpm"}, + "prometheus_ecto": {:hex, :prometheus_ecto, "1.4.1", "6c768ea9654de871e5b32fab2eac348467b3021604ebebbcbd8bcbe806a65ed5", [:mix], [{:ecto, "~> 2.0 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:prometheus_ex, "~> 1.1 or ~> 2.0 or ~> 3.0", [hex: :prometheus_ex, repo: "hexpm", optional: false]}], "hexpm"}, + "prometheus_ex": {:hex, :prometheus_ex, "3.0.5", "fa58cfd983487fc5ead331e9a3e0aa622c67232b3ec71710ced122c4c453a02f", [:mix], [{:prometheus, "~> 4.0", [hex: :prometheus, repo: "hexpm", optional: false]}], "hexpm"}, + "prometheus_phoenix": {:hex, :prometheus_phoenix, "1.2.1", "964a74dfbc055f781d3a75631e06ce3816a2913976d1df7830283aa3118a797a", [:mix], [{:phoenix, "~> 1.3", [hex: :phoenix, repo: "hexpm", optional: false]}, {:prometheus_ex, "~> 1.3 or ~> 2.0 or ~> 3.0", [hex: :prometheus_ex, repo: "hexpm", optional: false]}], "hexpm"}, + "prometheus_plugs": {:hex, :prometheus_plugs, "1.1.5", "25933d48f8af3a5941dd7b621c889749894d8a1082a6ff7c67cc99dec26377c5", [:mix], [{:accept, "~> 0.1", [hex: :accept, repo: "hexpm", optional: false]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: false]}, {:prometheus_ex, "~> 1.1 or ~> 2.0 or ~> 3.0", [hex: :prometheus_ex, repo: "hexpm", optional: false]}, {:prometheus_process_collector, "~> 1.1", [hex: :prometheus_process_collector, repo: "hexpm", optional: true]}], "hexpm"}, + "prometheus_process_collector": {:hex, :prometheus_process_collector, "1.4.0", "6dbd39e3165b9ef1c94a7a820e9ffe08479f949dcdd431ed4aaea7b250eebfde", [:rebar3], [{:prometheus, "~> 4.0", [hex: :prometheus, repo: "hexpm", optional: false]}], "hexpm"}, "ranch": {:hex, :ranch, "1.7.1", "6b1fab51b49196860b733a49c07604465a47bdb78aa10c1c16a3d199f7f8c881", [:rebar3], [], "hexpm"}, + "recon": {:git, "https://github.com/ferd/recon.git", "75d70c7c08926d2f24f1ee6de14ee50fe8a52763", []}, "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.4", "f0eafff810d2041e93f915ef59899c923f4568f4585904d010387ed74988e77b", [:make, :mix, :rebar3], [], "hexpm"}, "swoosh": {:hex, :swoosh, "0.20.0", "9a6c13822c9815993c03b6f8fccc370fcffb3c158d9754f67b1fdee6b3a5d928", [:mix], [{:cowboy, "~> 1.0.1 or ~> 1.1 or ~> 2.4", [hex: :cowboy, repo: "hexpm", optional: true]}, {:gen_smtp, "~> 0.12", [hex: :gen_smtp, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mime, "~> 1.1", [hex: :mime, repo: "hexpm", optional: false]}, {:plug, "~> 1.4", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm"}, "syslog": {:git, "https://github.com/Vagabond/erlang-syslog.git", "4a6c6f2c996483e86c1320e9553f91d337bcb6aa", [tag: "1.0.5"]}, From bc3618a38d2e37254e27f723d3dd61679eca9be5 Mon Sep 17 00:00:00 2001 From: href Date: Wed, 30 Jan 2019 16:32:30 +0100 Subject: [PATCH 04/26] Set up telemetry and prometheus --- config/config.exs | 5 +++++ lib/pleroma/application.ex | 8 ++++++++ lib/pleroma/repo.ex | 4 ++++ lib/pleroma/web/endpoint.ex | 20 ++++++++++++++++++++ 4 files changed, 37 insertions(+) diff --git a/config/config.exs b/config/config.exs index dccf7b263..1e086f44c 100644 --- a/config/config.exs +++ b/config/config.exs @@ -8,6 +8,10 @@ use Mix.Config # General application configuration config :pleroma, ecto_repos: [Pleroma.Repo] +config :pleroma, Pleroma.Repo, + types: Pleroma.PostgresTypes, + loggers: [Pleroma.Repo.Instrumenter, Ecto.LogEntry] + config :pleroma, Pleroma.Captcha, enabled: false, seconds_valid: 60, @@ -87,6 +91,7 @@ websocket_config = [ # Configures the endpoint config :pleroma, Pleroma.Web.Endpoint, + instrumenters: [Pleroma.Web.Endpoint.Instrumenter], url: [host: "localhost"], http: [ dispatch: [ diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 782d1d589..03dcbab1a 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -25,6 +25,7 @@ defmodule Pleroma.Application do import Cachex.Spec Pleroma.Config.DeprecationWarnings.warn() + setup_instrumenters() # Define workers and child supervisors to be supervised children = @@ -140,6 +141,13 @@ defmodule Pleroma.Application do end end + defp setup_instrumenters() do + Pleroma.Web.Endpoint.MetricsExporter.setup() + Pleroma.Web.Endpoint.PipelineInstrumenter.setup() + Pleroma.Web.Endpoint.Instrumenter.setup() + Pleroma.Repo.Instrumenter.setup() + end + if Mix.env() == :test do defp streamer_child, do: [] defp chat_child, do: [] diff --git a/lib/pleroma/repo.ex b/lib/pleroma/repo.ex index 4af1bde56..aa5d427ae 100644 --- a/lib/pleroma/repo.ex +++ b/lib/pleroma/repo.ex @@ -8,6 +8,10 @@ defmodule Pleroma.Repo do adapter: Ecto.Adapters.Postgres, migration_timestamps: [type: :naive_datetime_usec] + defmodule Instrumenter do + use Prometheus.EctoInstrumenter + end + @doc """ Dynamically loads the repository url from the DATABASE_URL environment variable. diff --git a/lib/pleroma/web/endpoint.ex b/lib/pleroma/web/endpoint.ex index fa2d1cbe7..6d9528c86 100644 --- a/lib/pleroma/web/endpoint.ex +++ b/lib/pleroma/web/endpoint.ex @@ -70,6 +70,26 @@ defmodule Pleroma.Web.Endpoint do extra: "SameSite=Strict" ) + # Note: the plug and its configuration is compile-time this can't be upstreamed yet + if proxies = Pleroma.Config.get([__MODULE__, :reverse_proxies]) do + plug(RemoteIp, proxies: proxies) + end + + defmodule Instrumenter do + use Prometheus.PhoenixInstrumenter + end + + defmodule PipelineInstrumenter do + use Prometheus.PlugPipelineInstrumenter + end + + defmodule MetricsExporter do + use Prometheus.PlugExporter + end + + plug(PipelineInstrumenter) + plug(MetricsExporter) + plug(Pleroma.Web.Router) @doc """ From 0b5c818cb78b8c23fb2ba7ef372d0688ea9f36b7 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Mon, 25 Mar 2019 15:29:04 +0700 Subject: [PATCH 05/26] [#1] fix telemetry --- config/config.exs | 2 +- lib/pleroma/application.ex | 25 ++++++++++++++++++------- 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/config/config.exs b/config/config.exs index 1e086f44c..4fd63f99d 100644 --- a/config/config.exs +++ b/config/config.exs @@ -10,7 +10,7 @@ config :pleroma, ecto_repos: [Pleroma.Repo] config :pleroma, Pleroma.Repo, types: Pleroma.PostgresTypes, - loggers: [Pleroma.Repo.Instrumenter, Ecto.LogEntry] + telemetry_event: [Pleroma.Repo.Instrumenter] config :pleroma, Pleroma.Captcha, enabled: false, diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 03dcbab1a..c3f3126c6 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -127,6 +127,24 @@ defmodule Pleroma.Application do Supervisor.start_link(children, opts) end + defp setup_instrumenters() do + require Prometheus.Registry + + :ok = + :telemetry.attach( + "prometheus-ecto", + [:pleroma, :repo, :query], + &Pleroma.Repo.Instrumenter.handle_event/4, + %{} + ) + + Prometheus.Registry.register_collector(:prometheus_process_collector) + Pleroma.Web.Endpoint.MetricsExporter.setup() + Pleroma.Web.Endpoint.PipelineInstrumenter.setup() + Pleroma.Web.Endpoint.Instrumenter.setup() + Pleroma.Repo.Instrumenter.setup() + end + def enabled_hackney_pools do [:media] ++ if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Hackney do @@ -141,13 +159,6 @@ defmodule Pleroma.Application do end end - defp setup_instrumenters() do - Pleroma.Web.Endpoint.MetricsExporter.setup() - Pleroma.Web.Endpoint.PipelineInstrumenter.setup() - Pleroma.Web.Endpoint.Instrumenter.setup() - Pleroma.Repo.Instrumenter.setup() - end - if Mix.env() == :test do defp streamer_child, do: [] defp chat_child, do: [] From 5564cd421dfc706208df0b7447b0d692dffe052e Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 4 Apr 2019 12:19:31 -0500 Subject: [PATCH 06/26] Document Prometheus --- docs/api/prometheus.md | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 docs/api/prometheus.md diff --git a/docs/api/prometheus.md b/docs/api/prometheus.md new file mode 100644 index 000000000..19c564e3c --- /dev/null +++ b/docs/api/prometheus.md @@ -0,0 +1,22 @@ +# Prometheus Metrics + +Pleroma includes support for exporting metrics via the [prometheus_ex](https://github.com/deadtrickster/prometheus.ex) library. + +## `/api/pleroma/app_metrics` +### Exports Prometheus application metrics +* Method: `GET` +* Authentication: not required +* Params: none +* Response: JSON + +## Grafana +### Config example +The following is a config example to use with [Grafana](https://grafana.com) + +``` + - job_name: 'beam' + metrics_path: /api/pleroma/app_metrics + scheme: https + static_configs: + - targets: ['pleroma.soykaf.com'] +``` From 7e930559fece1a86891645333cc79a18f440ef1d Mon Sep 17 00:00:00 2001 From: href Date: Wed, 30 Jan 2019 16:44:38 +0100 Subject: [PATCH 07/26] Serve metrics at `/api/pleroma/app_metrics` --- config/config.exs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/config/config.exs b/config/config.exs index 4fd63f99d..ebaf1aec5 100644 --- a/config/config.exs +++ b/config/config.exs @@ -353,6 +353,7 @@ config :pleroma, Pleroma.Web.Federator.RetryQueue, initial_timeout: 30, max_retries: 5 +<<<<<<< HEAD config :pleroma_job_queue, :queues, federator_incoming: 50, federator_outgoing: 50, @@ -385,6 +386,8 @@ config :pleroma, :ldap, config :pleroma, Pleroma.Mailer, adapter: Swoosh.Adapters.Sendmail +config :prometheus, Pleroma.Web.Endpoint.MetricsExporter, path: "/api/pleroma/app_metrics" + # Import environment specific config. This must remain at the bottom # of this file so it overrides the configuration defined above. import_config "#{Mix.env()}.exs" From 7222afe01b13586018b481172731309587191338 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 4 Apr 2019 12:29:10 -0500 Subject: [PATCH 08/26] Clean merge crumbs --- config/config.exs | 1 - 1 file changed, 1 deletion(-) diff --git a/config/config.exs b/config/config.exs index ebaf1aec5..b19b36b22 100644 --- a/config/config.exs +++ b/config/config.exs @@ -353,7 +353,6 @@ config :pleroma, Pleroma.Web.Federator.RetryQueue, initial_timeout: 30, max_retries: 5 -<<<<<<< HEAD config :pleroma_job_queue, :queues, federator_incoming: 50, federator_outgoing: 50, From 69038887b2930072356aa00841b889c59518e264 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 4 Apr 2019 12:36:57 -0500 Subject: [PATCH 09/26] Code readability tweak --- lib/pleroma/application.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index c3f3126c6..1fc3fb728 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -127,7 +127,7 @@ defmodule Pleroma.Application do Supervisor.start_link(children, opts) end - defp setup_instrumenters() do + defp setup_instrumenters do require Prometheus.Registry :ok = From 3b12eeda192e739e8328ef4202059bb482d1cff2 Mon Sep 17 00:00:00 2001 From: feld Date: Thu, 4 Apr 2019 19:52:22 +0000 Subject: [PATCH 10/26] Add ability to ship logs to a Slack channel --- config/config.exs | 5 +++++ docs/config.md | 20 +++++++++++++++++++- mix.exs | 5 +++-- mix.lock | 1 + 4 files changed, 28 insertions(+), 3 deletions(-) diff --git a/config/config.exs b/config/config.exs index dccf7b263..c143f79fc 100644 --- a/config/config.exs +++ b/config/config.exs @@ -118,6 +118,11 @@ config :logger, :ex_syslogger, format: "$metadata[$level] $message", metadata: [:request_id] +config :quack, + level: :warn, + meta: [:all], + webhook_url: "https://hooks.slack.com/services/YOUR-KEY-HERE" + config :mime, :types, %{ "application/xml" => ["xml"], "application/xrd+xml" => ["xrd+xml"], diff --git a/docs/config.md b/docs/config.md index 97a0e6ffa..06d6fd757 100644 --- a/docs/config.md +++ b/docs/config.md @@ -105,7 +105,7 @@ config :pleroma, Pleroma.Mailer, * `safe_dm_mentions`: If set to true, only mentions at the beginning of a post will be used to address people in direct messages. This is to prevent accidental mentioning of people when talking about them (e.g. "@friend hey i really don't like @enemy"). (Default: `false`) ## :logger -* `backends`: `:console` is used to send logs to stdout, `{ExSyslogger, :ex_syslogger}` to log to syslog +* `backends`: `:console` is used to send logs to stdout, `{ExSyslogger, :ex_syslogger}` to log to syslog, and `Quack.Logger` to log to Slack An example to enable ONLY ExSyslogger (f/ex in ``prod.secret.exs``) with info and debug suppressed: ``` @@ -128,6 +128,24 @@ config :logger, :ex_syslogger, See: [logger’s documentation](https://hexdocs.pm/logger/Logger.html) and [ex_syslogger’s documentation](https://hexdocs.pm/ex_syslogger/) +An example of logging info to local syslog, but warn to a Slack channel: +``` +config :logger, + backends: [ {ExSyslogger, :ex_syslogger}, Quack.Logger ], + level: :info + +config :logger, :ex_syslogger, + level: :info, + ident: "pleroma", + format: "$metadata[$level] $message" + +config :quack, + level: :warn, + meta: [:all], + webhook_url: "https://hooks.slack.com/services/YOUR-API-KEY-HERE" +``` + +See the [Quack Github](https://github.com/azohra/quack) for more details ## :frontend_configurations diff --git a/mix.exs b/mix.exs index 88da6332a..3a04e0060 100644 --- a/mix.exs +++ b/mix.exs @@ -41,7 +41,7 @@ defmodule Pleroma.Mixfile do def application do [ mod: {Pleroma.Application, []}, - extra_applications: [:logger, :runtime_tools, :comeonin], + extra_applications: [:logger, :runtime_tools, :comeonin, :quack], included_applications: [:ex_syslogger] ] end @@ -94,7 +94,8 @@ defmodule Pleroma.Mixfile do {:auto_linker, git: "https://git.pleroma.social/pleroma/auto_linker.git", ref: "479dd343f4e563ff91215c8275f3b5c67e032850"}, - {:pleroma_job_queue, "~> 0.2.0"} + {:pleroma_job_queue, "~> 0.2.0"}, + {:quack, "~> 0.1.1"} ] end diff --git a/mix.lock b/mix.lock index 9c454446a..d84d11049 100644 --- a/mix.lock +++ b/mix.lock @@ -57,6 +57,7 @@ "poison": {:hex, :poison, "3.1.0", "d9eb636610e096f86f25d9a46f35a9facac35609a7591b3be3326e99a0484665", [:mix], [], "hexpm"}, "poolboy": {:hex, :poolboy, "1.5.2", "392b007a1693a64540cead79830443abf5762f5d30cf50bc95cb2c1aaafa006b", [:rebar3], [], "hexpm"}, "postgrex": {:hex, :postgrex, "0.14.1", "63247d4a5ad6b9de57a0bac5d807e1c32d41e39c04b8a4156a26c63bcd8a2e49", [:mix], [{:connection, "~> 1.0", [hex: :connection, repo: "hexpm", optional: false]}, {:db_connection, "~> 2.0", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm"}, + "quack": {:hex, :quack, "0.1.1", "cca7b4da1a233757fdb44b3334fce80c94785b3ad5a602053b7a002b5a8967bf", [:mix], [{:poison, ">= 1.0.0", [hex: :poison, repo: "hexpm", optional: false]}, {:tesla, "~> 1.2.0", [hex: :tesla, repo: "hexpm", optional: false]}], "hexpm"}, "ranch": {:hex, :ranch, "1.7.1", "6b1fab51b49196860b733a49c07604465a47bdb78aa10c1c16a3d199f7f8c881", [:rebar3], [], "hexpm"}, "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.4", "f0eafff810d2041e93f915ef59899c923f4568f4585904d010387ed74988e77b", [:make, :mix, :rebar3], [], "hexpm"}, "swoosh": {:hex, :swoosh, "0.20.0", "9a6c13822c9815993c03b6f8fccc370fcffb3c158d9754f67b1fdee6b3a5d928", [:mix], [{:cowboy, "~> 1.0.1 or ~> 1.1 or ~> 2.4", [hex: :cowboy, repo: "hexpm", optional: true]}, {:gen_smtp, "~> 0.12", [hex: :gen_smtp, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mime, "~> 1.1", [hex: :mime, repo: "hexpm", optional: false]}, {:plug, "~> 1.4", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm"}, From f0f30019e1c9992cb420ba54457840cddaeb6a3a Mon Sep 17 00:00:00 2001 From: rinpatch Date: Fri, 5 Apr 2019 15:19:44 +0300 Subject: [PATCH 11/26] Refactor html caching functions to have a key instead of a module, use more correct terminology and fix summaries in mastoapi --- lib/pleroma/html.ex | 15 +++++++-------- lib/pleroma/web/mastodon_api/views/status_view.ex | 14 +++++++++++--- lib/pleroma/web/metadata/utils.ex | 2 +- .../web/twitter_api/views/activity_view.ex | 6 +++--- 4 files changed, 22 insertions(+), 15 deletions(-) diff --git a/lib/pleroma/html.ex b/lib/pleroma/html.ex index 1e48749a8..7f1dbe28c 100644 --- a/lib/pleroma/html.ex +++ b/lib/pleroma/html.ex @@ -28,21 +28,20 @@ defmodule Pleroma.HTML do def filter_tags(html), do: filter_tags(html, nil) def strip_tags(html), do: Scrubber.scrub(html, Scrubber.StripTags) - # TODO: rename object to activity because that's what it is really working with - def get_cached_scrubbed_html_for_object(content, scrubbers, object, module) do - key = "#{module}#{generate_scrubber_signature(scrubbers)}|#{object.id}" + def get_cached_scrubbed_html_for_activity(content, scrubbers, activity, key \\ "") do + key = "#{key}#{generate_scrubber_signature(scrubbers)}|#{activity.id}" Cachex.fetch!(:scrubber_cache, key, fn _key -> - ensure_scrubbed_html(content, scrubbers, object.data["object"]["fake"] || false) + ensure_scrubbed_html(content, scrubbers, activity.data["object"]["fake"] || false) end) end - def get_cached_stripped_html_for_object(content, object, module) do - get_cached_scrubbed_html_for_object( + def get_cached_stripped_html_for_activity(content, activity, key) do + get_cached_scrubbed_html_for_activity( content, HtmlSanitizeEx.Scrubber.StripTags, - object, - module + activity, + key ) end diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index 200bb453d..4c0b53bdd 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -147,10 +147,18 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do content = object |> render_content() - |> HTML.get_cached_scrubbed_html_for_object( + |> HTML.get_cached_scrubbed_html_for_activity( User.html_filter_policy(opts[:for]), activity, - __MODULE__ + "mastoapi:content" + ) + + summary = + (object["summary"] || "") + |> HTML.get_cached_scrubbed_html_for_activity( + User.html_filter_policy(opts[:for]), + activity, + "mastoapi:summary" ) card = render("card.json", Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity)) @@ -182,7 +190,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do muted: CommonAPI.thread_muted?(user, activity) || User.mutes?(opts[:for], user), pinned: pinned?(activity, user), sensitive: sensitive, - spoiler_text: object["summary"] || "", + spoiler_text: summary, visibility: get_visibility(object), media_attachments: attachments, mentions: mentions, diff --git a/lib/pleroma/web/metadata/utils.ex b/lib/pleroma/web/metadata/utils.ex index 23bbde1a6..58385a3d1 100644 --- a/lib/pleroma/web/metadata/utils.ex +++ b/lib/pleroma/web/metadata/utils.ex @@ -12,7 +12,7 @@ defmodule Pleroma.Web.Metadata.Utils do # html content comes from DB already encoded, decode first and scrub after |> HtmlEntities.decode() |> String.replace(~r//, " ") - |> HTML.get_cached_stripped_html_for_object(object, __MODULE__) + |> HTML.get_cached_stripped_html_for_activity(object, "metadata") |> Formatter.demojify() |> Formatter.truncate() end diff --git a/lib/pleroma/web/twitter_api/views/activity_view.ex b/lib/pleroma/web/twitter_api/views/activity_view.ex index aa1d41fa2..433322eb8 100644 --- a/lib/pleroma/web/twitter_api/views/activity_view.ex +++ b/lib/pleroma/web/twitter_api/views/activity_view.ex @@ -254,10 +254,10 @@ defmodule Pleroma.Web.TwitterAPI.ActivityView do html = content - |> HTML.get_cached_scrubbed_html_for_object( + |> HTML.get_cached_scrubbed_html_for_activity( User.html_filter_policy(opts[:for]), activity, - __MODULE__ + "twitterapi:content" ) |> Formatter.emojify(object["emoji"]) @@ -265,7 +265,7 @@ defmodule Pleroma.Web.TwitterAPI.ActivityView do if content do content |> String.replace(~r//, "\n") - |> HTML.get_cached_stripped_html_for_object(activity, __MODULE__) + |> HTML.get_cached_stripped_html_for_activity(activity, "twitterapi:content") else "" end From 7895ee37fae82de26b3c06e69a96788d8c88d139 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Sun, 16 Dec 2018 16:41:56 +0100 Subject: [PATCH 12/26] Add user following / unfollowing to the admin api. --- .../web/admin_api/admin_api_controller.ex | 20 ++++++++ lib/pleroma/web/router.ex | 4 ++ .../admin_api/admin_api_controller_test.exs | 46 +++++++++++++++++++ 3 files changed, 70 insertions(+) diff --git a/lib/pleroma/web/admin_api/admin_api_controller.ex b/lib/pleroma/web/admin_api/admin_api_controller.ex index b3a09e49e..84d0aabaf 100644 --- a/lib/pleroma/web/admin_api/admin_api_controller.ex +++ b/lib/pleroma/web/admin_api/admin_api_controller.ex @@ -25,6 +25,26 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do |> json(nickname) end + def user_follow(conn, %{"follower" => follower_nick, "followed" => followed_nick}) do + with %User{} = follower <- Repo.get_by(User, %{nickname: follower_nick}), + %User{} = followed <- Repo.get_by(User, %{nickname: followed_nick}) do + User.follow(follower, followed) + end + + conn + |> json("ok") + end + + def user_unfollow(conn, %{"follower" => follower_nick, "followed" => followed_nick}) do + with %User{} = follower <- Repo.get_by(User, %{nickname: follower_nick}), + %User{} = followed <- Repo.get_by(User, %{nickname: followed_nick}) do + User.unfollow(follower, followed) + end + + conn + |> json("ok") + end + def user_create( conn, %{"nickname" => nickname, "email" => email, "password" => password} diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 605a327fc..1c752e44c 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -140,8 +140,12 @@ defmodule Pleroma.Web.Router do scope "/api/pleroma/admin", Pleroma.Web.AdminAPI do pipe_through([:admin_api, :oauth_write]) + post("/user/follow", AdminAPIController, :user_follow) + post("/user/unfollow", AdminAPIController, :user_unfollow) + get("/users", AdminAPIController, :list_users) get("/users/:nickname", AdminAPIController, :user_show) + delete("/user", AdminAPIController, :user_delete) patch("/users/:nickname/toggle_activation", AdminAPIController, :user_toggle_activation) post("/user", AdminAPIController, :user_create) diff --git a/test/web/admin_api/admin_api_controller_test.exs b/test/web/admin_api/admin_api_controller_test.exs index acae64361..cedc907ec 100644 --- a/test/web/admin_api/admin_api_controller_test.exs +++ b/test/web/admin_api/admin_api_controller_test.exs @@ -74,6 +74,52 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do end end + describe "/api/pleroma/admin/user/follow" do + test "allows to force-follow another user" do + admin = insert(:user, info: %{is_admin: true}) + user = insert(:user) + follower = insert(:user) + + conn = + build_conn() + |> assign(:user, admin) + |> put_req_header("accept", "application/json") + |> post("/api/pleroma/admin/user/follow", %{ + "follower" => follower.nickname, + "followed" => user.nickname + }) + + user = Repo.get(User, user.id) + follower = Repo.get(User, follower.id) + + assert User.following?(follower, user) + end + end + + describe "/api/pleroma/admin/user/unfollow" do + test "allows to force-unfollow another user" do + admin = insert(:user, info: %{is_admin: true}) + user = insert(:user) + follower = insert(:user) + + User.follow(follower, user) + + conn = + build_conn() + |> assign(:user, admin) + |> put_req_header("accept", "application/json") + |> post("/api/pleroma/admin/user/unfollow", %{ + "follower" => follower.nickname, + "followed" => user.nickname + }) + + user = Repo.get(User, user.id) + follower = Repo.get(User, follower.id) + + refute User.following?(follower, user) + end + end + describe "PUT /api/pleroma/admin/users/tag" do setup do admin = insert(:user, info: %{is_admin: true}) From da64a5aece131d6bd8c0d17dcda61c626b44c4d0 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 5 Apr 2019 11:29:34 -0500 Subject: [PATCH 13/26] Document the admin API endpoints for controlling follow/unfollow --- docs/api/admin_api.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/docs/api/admin_api.md b/docs/api/admin_api.md index 53b68ffd4..86cacebb1 100644 --- a/docs/api/admin_api.md +++ b/docs/api/admin_api.md @@ -58,6 +58,26 @@ Authentication is required and the user must be an admin. - `password` - Response: User’s nickname +## `/api/pleroma/admin/user/follow` +### Make a user follow another user + +- Methods: `POST` +- Params: + - `follower`: The nickname of the follower + - `followed`: The nickname of the followed +- Response: + - "ok" + +## `/api/pleroma/admin/user/unfollow` +### Make a user unfollow another user + +- Methods: `POST` +- Params: + - `follower`: The nickname of the follower + - `followed`: The nickname of the followed +- Response: + - "ok" + ## `/api/pleroma/admin/users/:nickname/toggle_activation` ### Toggle user activation From b5a2d384f71de9f7ff33d99c95c5db4674141d9a Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 5 Apr 2019 11:41:41 -0500 Subject: [PATCH 14/26] Redundant Repo.get_by usage was recently removed from the codebase --- lib/pleroma/web/admin_api/admin_api_controller.ex | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/web/admin_api/admin_api_controller.ex b/lib/pleroma/web/admin_api/admin_api_controller.ex index 84d0aabaf..78bf31893 100644 --- a/lib/pleroma/web/admin_api/admin_api_controller.ex +++ b/lib/pleroma/web/admin_api/admin_api_controller.ex @@ -26,8 +26,8 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do end def user_follow(conn, %{"follower" => follower_nick, "followed" => followed_nick}) do - with %User{} = follower <- Repo.get_by(User, %{nickname: follower_nick}), - %User{} = followed <- Repo.get_by(User, %{nickname: followed_nick}) do + with %User{} = follower <- User.get_by_nickname(follower_nick), + %User{} = followed <- User.get_by_nickname(followed_nick) do User.follow(follower, followed) end @@ -36,8 +36,8 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do end def user_unfollow(conn, %{"follower" => follower_nick, "followed" => followed_nick}) do - with %User{} = follower <- Repo.get_by(User, %{nickname: follower_nick}), - %User{} = followed <- Repo.get_by(User, %{nickname: followed_nick}) do + with %User{} = follower <- User.get_by_nickname(follower_nick), + %User{} = followed <- User.get_by_nickname(followed_nick) do User.unfollow(follower, followed) end From c746087f570e366976b9b89c2aa6c2a5ff83c9ca Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 5 Apr 2019 11:59:56 -0500 Subject: [PATCH 15/26] Also remove Repo functions in the tests --- test/web/admin_api/admin_api_controller_test.exs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/web/admin_api/admin_api_controller_test.exs b/test/web/admin_api/admin_api_controller_test.exs index cedc907ec..9c1cae6b7 100644 --- a/test/web/admin_api/admin_api_controller_test.exs +++ b/test/web/admin_api/admin_api_controller_test.exs @@ -89,8 +89,8 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do "followed" => user.nickname }) - user = Repo.get(User, user.id) - follower = Repo.get(User, follower.id) + user = User.get_by_nickname(user.id) + follower = User.get_by_nickname(follower.id) assert User.following?(follower, user) end @@ -113,8 +113,8 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do "followed" => user.nickname }) - user = Repo.get(User, user.id) - follower = Repo.get(User, follower.id) + user = User.get_by_nickname(user.id) + follower = User.get_by_nickname(follower.id) refute User.following?(follower, user) end From fac76bfa35f735005249111e74ea6be8670f5755 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 5 Apr 2019 12:11:19 -0500 Subject: [PATCH 16/26] We actually want the user id not nickname in the test... --- test/web/admin_api/admin_api_controller_test.exs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/web/admin_api/admin_api_controller_test.exs b/test/web/admin_api/admin_api_controller_test.exs index 9c1cae6b7..dd2fbfb15 100644 --- a/test/web/admin_api/admin_api_controller_test.exs +++ b/test/web/admin_api/admin_api_controller_test.exs @@ -89,8 +89,8 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do "followed" => user.nickname }) - user = User.get_by_nickname(user.id) - follower = User.get_by_nickname(follower.id) + user = User.get_by_id(user.id) + follower = User.get_by_id(follower.id) assert User.following?(follower, user) end @@ -113,8 +113,8 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do "followed" => user.nickname }) - user = User.get_by_nickname(user.id) - follower = User.get_by_nickname(follower.id) + user = User.get_by_id(user.id) + follower = User.get_by_id(follower.id) refute User.following?(follower, user) end From fb1be1d79892a72b10af2c24479e81600603a6af Mon Sep 17 00:00:00 2001 From: optikfluffel Date: Fri, 5 Apr 2019 20:12:44 +0200 Subject: [PATCH 17/26] Use --cover option when running CI tests --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c07f1a5d3..0bd657d67 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -50,7 +50,7 @@ unit-testing: script: - mix ecto.create - mix ecto.migrate - - mix test --trace --preload-modules + - mix test --trace --preload-modules --cover lint: stage: test From e5df8cadeaa1ee3992e31e6ac00a0c391da7e4bd Mon Sep 17 00:00:00 2001 From: rinpatch Date: Fri, 5 Apr 2019 19:59:03 +0000 Subject: [PATCH 18/26] Revert "Merge branch 'test-coverage' into 'develop'" This reverts merge request !1027 --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0bd657d67..c07f1a5d3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -50,7 +50,7 @@ unit-testing: script: - mix ecto.create - mix ecto.migrate - - mix test --trace --preload-modules --cover + - mix test --trace --preload-modules lint: stage: test From e9c075d05c2f11b905d40ed86dd19818acf310ec Mon Sep 17 00:00:00 2001 From: Sergey Suprunenko Date: Fri, 5 Apr 2019 22:40:30 +0200 Subject: [PATCH 19/26] Mock :crypt.crypt/2 because otherwise the test fails on Mac OS --- test/plugs/legacy_authentication_plug_test.exs | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/test/plugs/legacy_authentication_plug_test.exs b/test/plugs/legacy_authentication_plug_test.exs index 302662797..8b0b06772 100644 --- a/test/plugs/legacy_authentication_plug_test.exs +++ b/test/plugs/legacy_authentication_plug_test.exs @@ -47,16 +47,18 @@ defmodule Pleroma.Plugs.LegacyAuthenticationPlugTest do |> assign(:auth_user, user) conn = - with_mock User, - reset_password: fn user, %{password: password, password_confirmation: password} -> - send(self(), :reset_password) - {:ok, user} - end do - conn - |> LegacyAuthenticationPlug.call(%{}) + with_mocks([ + {:crypt, [], [crypt: fn _password, password_hash -> password_hash end]}, + {User, [], + [ + reset_password: fn user, %{password: password, password_confirmation: password} -> + {:ok, user} + end + ]} + ]) do + LegacyAuthenticationPlug.call(conn, %{}) end - assert_received :reset_password assert conn.assigns.user == user end From 325a2680173f714a5875ed726f9171e7984f7f7a Mon Sep 17 00:00:00 2001 From: Sergey Suprunenko Date: Fri, 5 Apr 2019 23:36:42 +0000 Subject: [PATCH 20/26] Redirect to the referer url after mastofe authorization --- .../mastodon_api/mastodon_api_controller.ex | 19 ++++-- test/support/factory.ex | 10 +++ .../mastodon_api_controller_test.exs | 67 +++++++++++++++++++ 3 files changed, 90 insertions(+), 6 deletions(-) diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index 89fd7629a..bcc79b08a 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -1091,9 +1091,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do end def index(%{assigns: %{user: user}} = conn, _params) do - token = - conn - |> get_session(:oauth_token) + token = get_session(conn, :oauth_token) if user && token do mastodon_emoji = mastodonized_emoji() @@ -1194,6 +1192,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do |> render("index.html", %{initial_state: initial_state, flavour: flavour}) else conn + |> put_session(:return_to, conn.request_path) |> redirect(to: "/web/login") end end @@ -1278,12 +1277,20 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do scope: Enum.join(app.scopes, " ") ) - conn - |> redirect(to: path) + redirect(conn, to: path) end end - defp local_mastodon_root_path(conn), do: mastodon_api_path(conn, :index, ["getting-started"]) + defp local_mastodon_root_path(conn) do + case get_session(conn, :return_to) do + nil -> + mastodon_api_path(conn, :index, ["getting-started"]) + + return_to -> + delete_session(conn, :return_to) + return_to + end + end defp get_or_make_app do find_attrs = %{client_name: @local_mastodon_name, redirect_uris: "."} diff --git a/test/support/factory.ex b/test/support/factory.ex index e1a08315a..b37bc2c07 100644 --- a/test/support/factory.ex +++ b/test/support/factory.ex @@ -240,6 +240,16 @@ defmodule Pleroma.Factory do } end + def oauth_authorization_factory do + %Pleroma.Web.OAuth.Authorization{ + token: :crypto.strong_rand_bytes(32) |> Base.url_encode64(padding: false), + scopes: ["read", "write", "follow", "push"], + valid_until: NaiveDateTime.add(NaiveDateTime.utc_now(), 60 * 10), + user: build(:user), + app: build(:oauth_app) + } + end + def push_subscription_factory do %Pleroma.Web.Push.Subscription{ user: build(:user), diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index 6060cc97f..438e9507d 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -2340,4 +2340,71 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do refute acc_one == acc_two assert acc_two == acc_three end + + describe "index/2 redirections" do + setup %{conn: conn} do + session_opts = [ + store: :cookie, + key: "_test", + signing_salt: "cooldude" + ] + + conn = + conn + |> Plug.Session.call(Plug.Session.init(session_opts)) + |> fetch_session() + + test_path = "/web/statuses/test" + %{conn: conn, path: test_path} + end + + test "redirects not logged-in users to the login page", %{conn: conn, path: path} do + conn = get(conn, path) + + assert conn.status == 302 + assert redirected_to(conn) == "/web/login" + end + + test "does not redirect logged in users to the login page", %{conn: conn, path: path} do + token = insert(:oauth_token) + + conn = + conn + |> assign(:user, token.user) + |> put_session(:oauth_token, token.token) + |> get(path) + + assert conn.status == 200 + end + + test "saves referer path to session", %{conn: conn, path: path} do + conn = get(conn, path) + return_to = Plug.Conn.get_session(conn, :return_to) + + assert return_to == path + end + + test "redirects to the saved path after log in", %{conn: conn, path: path} do + app = insert(:oauth_app, client_name: "Mastodon-Local", redirect_uris: ".") + auth = insert(:oauth_authorization, app: app) + + conn = + conn + |> put_session(:return_to, path) + |> get("/web/login", %{code: auth.token}) + + assert conn.status == 302 + assert redirected_to(conn) == path + end + + test "redirects to the getting-started page when referer is not present", %{conn: conn} do + app = insert(:oauth_app, client_name: "Mastodon-Local", redirect_uris: ".") + auth = insert(:oauth_authorization, app: app) + + conn = get(conn, "/web/login", %{code: auth.token}) + + assert conn.status == 302 + assert redirected_to(conn) == "/web/getting-started" + end + end end From b395aebf2489f44bdb1a9c4905a51f0f26bf5fab Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sat, 6 Apr 2019 09:30:36 -0500 Subject: [PATCH 21/26] Pin recon dependency to 2.4.0 --- mix.exs | 2 +- mix.lock | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mix.exs b/mix.exs index 6e7cff413..ec0865c4f 100644 --- a/mix.exs +++ b/mix.exs @@ -101,7 +101,7 @@ defmodule Pleroma.Mixfile do {:prometheus_phoenix, "~> 1.2"}, {:prometheus_ecto, "~> 1.4"}, {:prometheus_process_collector, "~> 1.4"}, - {:recon, github: "ferd/recon"}, + {:recon, github: "ferd/recon", tag: "2.4.0"}, {:quack, "~> 0.1.1"} ] end diff --git a/mix.lock b/mix.lock index 662fd0c6e..7c7e322de 100644 --- a/mix.lock +++ b/mix.lock @@ -66,7 +66,7 @@ "prometheus_process_collector": {:hex, :prometheus_process_collector, "1.4.0", "6dbd39e3165b9ef1c94a7a820e9ffe08479f949dcdd431ed4aaea7b250eebfde", [:rebar3], [{:prometheus, "~> 4.0", [hex: :prometheus, repo: "hexpm", optional: false]}], "hexpm"}, "quack": {:hex, :quack, "0.1.1", "cca7b4da1a233757fdb44b3334fce80c94785b3ad5a602053b7a002b5a8967bf", [:mix], [{:poison, ">= 1.0.0", [hex: :poison, repo: "hexpm", optional: false]}, {:tesla, "~> 1.2.0", [hex: :tesla, repo: "hexpm", optional: false]}], "hexpm"}, "ranch": {:hex, :ranch, "1.7.1", "6b1fab51b49196860b733a49c07604465a47bdb78aa10c1c16a3d199f7f8c881", [:rebar3], [], "hexpm"}, - "recon": {:git, "https://github.com/ferd/recon.git", "75d70c7c08926d2f24f1ee6de14ee50fe8a52763", []}, + "recon": {:git, "https://github.com/ferd/recon.git", "75d70c7c08926d2f24f1ee6de14ee50fe8a52763", [tag: "2.4.0"]}, "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.4", "f0eafff810d2041e93f915ef59899c923f4568f4585904d010387ed74988e77b", [:make, :mix, :rebar3], [], "hexpm"}, "swoosh": {:hex, :swoosh, "0.20.0", "9a6c13822c9815993c03b6f8fccc370fcffb3c158d9754f67b1fdee6b3a5d928", [:mix], [{:cowboy, "~> 1.0.1 or ~> 1.1 or ~> 2.4", [hex: :cowboy, repo: "hexpm", optional: true]}, {:gen_smtp, "~> 0.12", [hex: :gen_smtp, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mime, "~> 1.1", [hex: :mime, repo: "hexpm", optional: false]}, {:plug, "~> 1.4", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm"}, "syslog": {:git, "https://github.com/Vagabond/erlang-syslog.git", "4a6c6f2c996483e86c1320e9553f91d337bcb6aa", [tag: "1.0.5"]}, From 7aa53d52bd982b5ab233a65048f5fb1823127d4a Mon Sep 17 00:00:00 2001 From: eugenijm Date: Sat, 6 Apr 2019 00:22:42 +0300 Subject: [PATCH 22/26] Return 403 on oauth token exchange for a deactivated user --- lib/pleroma/web/oauth/oauth_controller.ex | 6 ++++++ test/web/oauth/oauth_controller_test.exs | 26 +++++++++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/lib/pleroma/web/oauth/oauth_controller.ex b/lib/pleroma/web/oauth/oauth_controller.ex index 26d53df1a..aac8f97fc 100644 --- a/lib/pleroma/web/oauth/oauth_controller.ex +++ b/lib/pleroma/web/oauth/oauth_controller.ex @@ -152,6 +152,7 @@ defmodule Pleroma.Web.OAuth.OAuthController do with {_, {:ok, %User{} = user}} <- {:get_user, Authenticator.get_user(conn)}, %App{} = app <- get_app_from_request(conn, params), {:auth_active, true} <- {:auth_active, User.auth_active?(user)}, + {:user_active, true} <- {:user_active, !user.info.deactivated}, scopes <- oauth_scopes(params, app.scopes), [] <- scopes -- app.scopes, true <- Enum.any?(scopes), @@ -175,6 +176,11 @@ defmodule Pleroma.Web.OAuth.OAuthController do |> put_status(:forbidden) |> json(%{error: "Your login is missing a confirmed e-mail address"}) + {:user_active, false} -> + conn + |> put_status(:forbidden) + |> json(%{error: "Your account is currently disabled"}) + _error -> put_status(conn, 400) |> json(%{error: "Invalid credentials"}) diff --git a/test/web/oauth/oauth_controller_test.exs b/test/web/oauth/oauth_controller_test.exs index a9a0b9ed4..a68528420 100644 --- a/test/web/oauth/oauth_controller_test.exs +++ b/test/web/oauth/oauth_controller_test.exs @@ -327,6 +327,32 @@ defmodule Pleroma.Web.OAuth.OAuthControllerTest do refute Map.has_key?(resp, "access_token") end + test "rejects token exchange for valid credentials belonging to deactivated user" do + password = "testpassword" + + user = + insert(:user, + password_hash: Comeonin.Pbkdf2.hashpwsalt(password), + info: %{deactivated: true} + ) + + app = insert(:oauth_app) + + conn = + build_conn() + |> post("/oauth/token", %{ + "grant_type" => "password", + "username" => user.nickname, + "password" => password, + "client_id" => app.client_id, + "client_secret" => app.client_secret + }) + + assert resp = json_response(conn, 403) + assert %{"error" => _} = resp + refute Map.has_key?(resp, "access_token") + end + test "rejects an invalid authorization code" do app = insert(:oauth_app) From 7bf622ce736af12db9b4865d8d3c2db5792d6f03 Mon Sep 17 00:00:00 2001 From: eugenijm Date: Thu, 28 Mar 2019 12:39:10 +0300 Subject: [PATCH 23/26] Add scheduled activities --- lib/pleroma/scheduled_activity.ex | 74 +++++++++++++ lib/pleroma/web/mastodon_api/mastodon_api.ex | 7 ++ .../mastodon_api/mastodon_api_controller.ex | 47 ++++++++ .../views/scheduled_activity_view.ex | 23 ++++ lib/pleroma/web/router.ex | 6 + ...0328053912_create_scheduled_activities.exs | 15 +++ test/support/factory.ex | 8 ++ .../mastodon_api_controller_test.exs | 104 ++++++++++++++++++ 8 files changed, 284 insertions(+) create mode 100644 lib/pleroma/scheduled_activity.ex create mode 100644 lib/pleroma/web/mastodon_api/views/scheduled_activity_view.ex create mode 100644 priv/repo/migrations/20190328053912_create_scheduled_activities.exs diff --git a/lib/pleroma/scheduled_activity.ex b/lib/pleroma/scheduled_activity.ex new file mode 100644 index 000000000..0c1b26a33 --- /dev/null +++ b/lib/pleroma/scheduled_activity.ex @@ -0,0 +1,74 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.ScheduledActivity do + use Ecto.Schema + + alias Pleroma.Repo + alias Pleroma.ScheduledActivity + alias Pleroma.User + + import Ecto.Query + import Ecto.Changeset + + schema "scheduled_activities" do + belongs_to(:user, User, type: Pleroma.FlakeId) + field(:scheduled_at, :naive_datetime) + field(:params, :map) + + timestamps() + end + + def changeset(%ScheduledActivity{} = scheduled_activity, attrs) do + scheduled_activity + |> cast(attrs, [:scheduled_at, :params]) + end + + def update_changeset(%ScheduledActivity{} = scheduled_activity, attrs) do + scheduled_activity + |> cast(attrs, [:scheduled_at]) + end + + def new(%User{} = user, attrs) do + %ScheduledActivity{user_id: user.id} + |> changeset(attrs) + end + + def create(%User{} = user, attrs) do + user + |> new(attrs) + |> Repo.insert() + end + + def get(%User{} = user, scheduled_activity_id) do + ScheduledActivity + |> where(user_id: ^user.id) + |> where(id: ^scheduled_activity_id) + |> Repo.one() + end + + def update(%User{} = user, scheduled_activity_id, attrs) do + with %ScheduledActivity{} = scheduled_activity <- get(user, scheduled_activity_id) do + scheduled_activity + |> update_changeset(attrs) + |> Repo.update() + else + nil -> {:error, :not_found} + end + end + + def delete(%User{} = user, scheduled_activity_id) do + with %ScheduledActivity{} = scheduled_activity <- get(user, scheduled_activity_id) do + scheduled_activity + |> Repo.delete() + else + nil -> {:error, :not_found} + end + end + + def for_user_query(%User{} = user) do + ScheduledActivity + |> where(user_id: ^user.id) + end +end diff --git a/lib/pleroma/web/mastodon_api/mastodon_api.ex b/lib/pleroma/web/mastodon_api/mastodon_api.ex index 08ea5f967..382f07e6b 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api.ex @@ -5,6 +5,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPI do alias Pleroma.Activity alias Pleroma.Notification alias Pleroma.Pagination + alias Pleroma.ScheduledActivity alias Pleroma.User def get_followers(user, params \\ %{}) do @@ -28,6 +29,12 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPI do |> Pagination.fetch_paginated(params) end + def get_scheduled_activities(user, params \\ %{}) do + user + |> ScheduledActivity.for_user_query() + |> Pagination.fetch_paginated(params) + end + defp cast_params(params) do param_types = %{ exclude_types: {:array, :string} diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index bcc79b08a..0916d84dc 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -11,6 +11,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do alias Pleroma.Notification alias Pleroma.Object alias Pleroma.Repo + alias Pleroma.ScheduledActivity alias Pleroma.Stats alias Pleroma.User alias Pleroma.Web @@ -25,6 +26,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do alias Pleroma.Web.MastodonAPI.MastodonView alias Pleroma.Web.MastodonAPI.NotificationView alias Pleroma.Web.MastodonAPI.ReportView + alias Pleroma.Web.MastodonAPI.ScheduledActivityView alias Pleroma.Web.MastodonAPI.StatusView alias Pleroma.Web.MediaProxy alias Pleroma.Web.OAuth.App @@ -364,6 +366,45 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do end end + def scheduled_statuses(%{assigns: %{user: user}} = conn, params) do + with scheduled_activities <- MastodonAPI.get_scheduled_activities(user, params) do + conn + |> add_link_headers(:scheduled_statuses, scheduled_activities) + |> put_view(ScheduledActivityView) + |> render("index.json", %{scheduled_activities: scheduled_activities}) + end + end + + def show_scheduled_status(%{assigns: %{user: user}} = conn, %{"id" => scheduled_activity_id}) do + with %ScheduledActivity{} = scheduled_activity <- + ScheduledActivity.get(user, scheduled_activity_id) do + conn + |> put_view(ScheduledActivityView) + |> render("show.json", %{scheduled_activity: scheduled_activity}) + else + _ -> {:error, :not_found} + end + end + + def update_scheduled_status( + %{assigns: %{user: user}} = conn, + %{"id" => scheduled_activity_id} = params + ) do + with {:ok, scheduled_activity} <- + ScheduledActivity.update(user, scheduled_activity_id, params) do + conn + |> put_view(ScheduledActivityView) + |> render("show.json", %{scheduled_activity: scheduled_activity}) + end + end + + def delete_scheduled_status(%{assigns: %{user: user}} = conn, %{"id" => scheduled_activity_id}) do + with {:ok, %ScheduledActivity{}} <- ScheduledActivity.delete(user, scheduled_activity_id) do + conn + |> json(%{}) + end + end + def post_status(conn, %{"status" => "", "media_ids" => media_ids} = params) when length(media_ids) > 0 do params = @@ -1406,6 +1447,12 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do # fallback action # + def errors(conn, {:error, :not_found}) do + conn + |> put_status(404) + |> json(%{error: "Record not found"}) + end + def errors(conn, _) do conn |> put_status(500) diff --git a/lib/pleroma/web/mastodon_api/views/scheduled_activity_view.ex b/lib/pleroma/web/mastodon_api/views/scheduled_activity_view.ex new file mode 100644 index 000000000..87aa3729e --- /dev/null +++ b/lib/pleroma/web/mastodon_api/views/scheduled_activity_view.ex @@ -0,0 +1,23 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.MastodonAPI.ScheduledActivityView do + use Pleroma.Web, :view + + alias Pleroma.ScheduledActivity + alias Pleroma.Web.CommonAPI + alias Pleroma.Web.MastodonAPI.ScheduledActivityView + + def render("index.json", %{scheduled_activities: scheduled_activities}) do + render_many(scheduled_activities, ScheduledActivityView, "show.json") + end + + def render("show.json", %{scheduled_activity: %ScheduledActivity{} = scheduled_activity}) do + %{ + id: scheduled_activity.id |> to_string, + scheduled_at: scheduled_activity.scheduled_at |> CommonAPI.Utils.to_masto_date(), + params: scheduled_activity.params + } + end +end diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 1c752e44c..3b5ac6fdd 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -244,6 +244,9 @@ defmodule Pleroma.Web.Router do get("/notifications", MastodonAPIController, :notifications) get("/notifications/:id", MastodonAPIController, :get_notification) + get("/scheduled_statuses", MastodonAPIController, :scheduled_statuses) + get("/scheduled_statuses/:id", MastodonAPIController, :show_scheduled_status) + get("/lists", MastodonAPIController, :get_lists) get("/lists/:id", MastodonAPIController, :get_list) get("/lists/:id/accounts", MastodonAPIController, :list_accounts) @@ -278,6 +281,9 @@ defmodule Pleroma.Web.Router do post("/statuses/:id/mute", MastodonAPIController, :mute_conversation) post("/statuses/:id/unmute", MastodonAPIController, :unmute_conversation) + put("/scheduled_statuses/:id", MastodonAPIController, :update_scheduled_status) + delete("/scheduled_statuses/:id", MastodonAPIController, :delete_scheduled_status) + post("/media", MastodonAPIController, :upload) put("/media/:id", MastodonAPIController, :update_media) diff --git a/priv/repo/migrations/20190328053912_create_scheduled_activities.exs b/priv/repo/migrations/20190328053912_create_scheduled_activities.exs new file mode 100644 index 000000000..dc2436dce --- /dev/null +++ b/priv/repo/migrations/20190328053912_create_scheduled_activities.exs @@ -0,0 +1,15 @@ +defmodule Pleroma.Repo.Migrations.CreateScheduledActivities do + use Ecto.Migration + + def change do + create table(:scheduled_activities) do + add(:user_id, references(:users, type: :uuid, on_delete: :delete_all)) + add(:scheduled_at, :naive_datetime, null: false) + add(:params, :map, null: false) + + timestamps() + end + + create(index(:scheduled_activities, [:scheduled_at])) + end +end diff --git a/test/support/factory.ex b/test/support/factory.ex index b37bc2c07..667f59e8c 100644 --- a/test/support/factory.ex +++ b/test/support/factory.ex @@ -23,6 +23,14 @@ defmodule Pleroma.Factory do } end + def scheduled_activity_factory do + %Pleroma.ScheduledActivity{ + user: build(:user), + scheduled_at: NaiveDateTime.add(NaiveDateTime.utc_now(), :timer.minutes(60), :millisecond), + params: build(:note) |> Map.from_struct() |> Map.get(:data) + } + end + def note_factory(attrs \\ %{}) do text = sequence(:text, &"This is :moominmamma: note #{&1}") diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index 438e9507d..864c0ad4d 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -10,6 +10,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do alias Pleroma.Notification alias Pleroma.Object alias Pleroma.Repo + alias Pleroma.ScheduledActivity alias Pleroma.User alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.CommonAPI @@ -2407,4 +2408,107 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do assert redirected_to(conn) == "/web/getting-started" end end + + describe "scheduled activities" do + test "shows scheduled activities", %{conn: conn} do + user = insert(:user) + scheduled_activity_id1 = insert(:scheduled_activity, user: user).id |> to_string() + scheduled_activity_id2 = insert(:scheduled_activity, user: user).id |> to_string() + scheduled_activity_id3 = insert(:scheduled_activity, user: user).id |> to_string() + scheduled_activity_id4 = insert(:scheduled_activity, user: user).id |> to_string() + + conn = + conn + |> assign(:user, user) + + # min_id + conn_res = + conn + |> get("/api/v1/scheduled_statuses?limit=2&min_id=#{scheduled_activity_id1}") + + result = json_response(conn_res, 200) + assert [%{"id" => ^scheduled_activity_id3}, %{"id" => ^scheduled_activity_id2}] = result + + # since_id + conn_res = + conn + |> get("/api/v1/scheduled_statuses?limit=2&since_id=#{scheduled_activity_id1}") + + result = json_response(conn_res, 200) + assert [%{"id" => ^scheduled_activity_id4}, %{"id" => ^scheduled_activity_id3}] = result + + # max_id + conn_res = + conn + |> get("/api/v1/scheduled_statuses?limit=2&max_id=#{scheduled_activity_id4}") + + result = json_response(conn_res, 200) + assert [%{"id" => ^scheduled_activity_id3}, %{"id" => ^scheduled_activity_id2}] = result + end + + test "shows a scheduled activity", %{conn: conn} do + user = insert(:user) + scheduled_activity = insert(:scheduled_activity, user: user) + + res_conn = + conn + |> assign(:user, user) + |> get("/api/v1/scheduled_statuses/#{scheduled_activity.id}") + + assert %{"id" => scheduled_activity_id} = json_response(res_conn, 200) + assert scheduled_activity_id == scheduled_activity.id |> to_string() + + res_conn = + conn + |> assign(:user, user) + |> get("/api/v1/scheduled_statuses/404") + + assert %{"error" => "Record not found"} = json_response(res_conn, 404) + end + + test "updates a scheduled activity", %{conn: conn} do + user = insert(:user) + scheduled_activity = insert(:scheduled_activity, user: user) + + new_scheduled_at = + NaiveDateTime.add(NaiveDateTime.utc_now(), :timer.minutes(120), :millisecond) + + res_conn = + conn + |> assign(:user, user) + |> put("/api/v1/scheduled_statuses/#{scheduled_activity.id}", %{ + scheduled_at: new_scheduled_at + }) + + assert %{"scheduled_at" => expected_scheduled_at} = json_response(res_conn, 200) + assert expected_scheduled_at == Pleroma.Web.CommonAPI.Utils.to_masto_date(new_scheduled_at) + + res_conn = + conn + |> assign(:user, user) + |> put("/api/v1/scheduled_statuses/404", %{scheduled_at: new_scheduled_at}) + + assert %{"error" => "Record not found"} = json_response(res_conn, 404) + end + + test "deletes a scheduled activity", %{conn: conn} do + user = insert(:user) + scheduled_activity = insert(:scheduled_activity, user: user) + + res_conn = + conn + |> assign(:user, user) + |> delete("/api/v1/scheduled_statuses/#{scheduled_activity.id}") + + assert %{} = json_response(res_conn, 200) + assert nil == Repo.get(ScheduledActivity, scheduled_activity.id) + + res_conn = + conn + |> assign(:user, user) + |> delete("/api/v1/scheduled_statuses/#{scheduled_activity.id}") + + assert %{"error" => "Record not found"} = json_response(res_conn, 404) + end + end end From b3870df51fb2f35c3e51bea435134fe3fb692ef8 Mon Sep 17 00:00:00 2001 From: eugenijm Date: Sat, 30 Mar 2019 12:58:40 +0300 Subject: [PATCH 24/26] Handle `scheduled_at` on status creation. --- lib/pleroma/activity.ex | 2 +- lib/pleroma/scheduled_activity.ex | 16 +++++++++ .../mastodon_api/mastodon_api_controller.ex | 27 +++++++++++--- .../mastodon_api_controller_test.exs | 36 +++++++++++++++++++ 4 files changed, 75 insertions(+), 6 deletions(-) diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex index bc3f8caba..ab8861b27 100644 --- a/lib/pleroma/activity.ex +++ b/lib/pleroma/activity.ex @@ -31,7 +31,7 @@ defmodule Pleroma.Activity do field(:data, :map) field(:local, :boolean, default: true) field(:actor, :string) - field(:recipients, {:array, :string}) + field(:recipients, {:array, :string}, default: []) has_many(:notifications, Notification, on_delete: :delete_all) # Attention: this is a fake relation, don't try to preload it blindly and expect it to work! diff --git a/lib/pleroma/scheduled_activity.ex b/lib/pleroma/scheduled_activity.ex index 0c1b26a33..9fdc13990 100644 --- a/lib/pleroma/scheduled_activity.ex +++ b/lib/pleroma/scheduled_activity.ex @@ -12,6 +12,8 @@ defmodule Pleroma.ScheduledActivity do import Ecto.Query import Ecto.Changeset + @min_offset :timer.minutes(5) + schema "scheduled_activities" do belongs_to(:user, User, type: Pleroma.FlakeId) field(:scheduled_at, :naive_datetime) @@ -30,6 +32,20 @@ defmodule Pleroma.ScheduledActivity do |> cast(attrs, [:scheduled_at]) end + def far_enough?(scheduled_at) when is_binary(scheduled_at) do + with {:ok, scheduled_at} <- Ecto.Type.cast(:naive_datetime, scheduled_at) do + far_enough?(scheduled_at) + else + _ -> false + end + end + + def far_enough?(scheduled_at) do + now = NaiveDateTime.utc_now() + diff = NaiveDateTime.diff(scheduled_at, now, :millisecond) + diff > @min_offset + end + def new(%User{} = user, attrs) do %ScheduledActivity{user_id: user.id} |> changeset(attrs) diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index 0916d84dc..863fc3954 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -425,12 +425,29 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do _ -> Ecto.UUID.generate() end - {:ok, activity} = - Cachex.fetch!(:idempotency_cache, idempotency_key, fn _ -> CommonAPI.post(user, params) end) + scheduled_at = params["scheduled_at"] - conn - |> put_view(StatusView) - |> try_render("status.json", %{activity: activity, for: user, as: :activity}) + if scheduled_at && ScheduledActivity.far_enough?(scheduled_at) do + {:ok, scheduled_activity} = + Cachex.fetch!(:idempotency_cache, idempotency_key, fn _ -> + ScheduledActivity.create(user, %{"params" => params, "scheduled_at" => scheduled_at}) + end) + + conn + |> put_view(ScheduledActivityView) + |> render("show.json", %{scheduled_activity: scheduled_activity}) + else + params = Map.drop(params, ["scheduled_at"]) + + {:ok, activity} = + Cachex.fetch!(:idempotency_cache, idempotency_key, fn _ -> + CommonAPI.post(user, params) + end) + + conn + |> put_view(StatusView) + |> try_render("status.json", %{activity: activity, for: user, as: :activity}) + end end def delete_status(%{assigns: %{user: user}} = conn, %{"id" => id}) do diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index 864c0ad4d..0ec66ab73 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -2410,6 +2410,42 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do end describe "scheduled activities" do + test "creates a scheduled activity", %{conn: conn} do + user = insert(:user) + scheduled_at = NaiveDateTime.add(NaiveDateTime.utc_now(), :timer.minutes(120), :millisecond) + + conn = + conn + |> assign(:user, user) + |> post("/api/v1/statuses", %{ + "status" => "scheduled", + "scheduled_at" => scheduled_at + }) + + assert %{"scheduled_at" => expected_scheduled_at} = json_response(conn, 200) + assert expected_scheduled_at == Pleroma.Web.CommonAPI.Utils.to_masto_date(scheduled_at) + assert [] == Repo.all(Activity) + end + + test "skips the scheduling and creates the activity if scheduled_at is earlier than 5 minutes from now", + %{conn: conn} do + user = insert(:user) + + scheduled_at = + NaiveDateTime.add(NaiveDateTime.utc_now(), :timer.minutes(5) - 1, :millisecond) + + conn = + conn + |> assign(:user, user) + |> post("/api/v1/statuses", %{ + "status" => "not scheduled", + "scheduled_at" => scheduled_at + }) + + assert %{"content" => "not scheduled"} = json_response(conn, 200) + assert [] == Repo.all(ScheduledActivity) + end + test "shows scheduled activities", %{conn: conn} do user = insert(:user) scheduled_activity_id1 = insert(:scheduled_activity, user: user).id |> to_string() From fc92a0fd8d5be0352f4791b79bda04960f36f707 Mon Sep 17 00:00:00 2001 From: eugenijm Date: Tue, 2 Apr 2019 01:31:01 +0300 Subject: [PATCH 25/26] Added limits and media attachments for scheduled activities. --- config/config.exs | 4 + docs/config.md | 9 +- lib/pleroma/object.ex | 8 ++ lib/pleroma/scheduled_activity.ex | 83 ++++++++++++++--- .../mastodon_api/mastodon_api_controller.ex | 18 +++- .../views/scheduled_activity_view.ex | 32 ++++++- ...0328053912_create_scheduled_activities.exs | 1 + test/scheduled_activity_test.exs | 93 +++++++++++++++++++ test/support/factory.ex | 16 ++-- .../mastodon_api_controller_test.exs | 25 +++++ .../scheduled_activity_view_test.exs | 68 ++++++++++++++ 11 files changed, 327 insertions(+), 30 deletions(-) create mode 100644 test/scheduled_activity_test.exs create mode 100644 test/web/mastodon_api/scheduled_activity_view_test.exs diff --git a/config/config.exs b/config/config.exs index 61e799f33..79cef87e6 100644 --- a/config/config.exs +++ b/config/config.exs @@ -367,6 +367,10 @@ config :pleroma, :fetch_initial_posts, enabled: false, pages: 5 +config :pleroma, Pleroma.ScheduledActivity, + daily_user_limit: 25, + total_user_limit: 100 + config :auto_linker, opts: [ scheme: true, diff --git a/docs/config.md b/docs/config.md index 06d6fd757..df21beff3 100644 --- a/docs/config.md +++ b/docs/config.md @@ -218,14 +218,14 @@ This section is used to configure Pleroma-FE, unless ``:managed_config`` in ``:i - `port` * `url` - a list containing the configuration for generating urls, accepts - `host` - the host without the scheme and a post (e.g `example.com`, not `https://example.com:2020`) - - `scheme` - e.g `http`, `https` + - `scheme` - e.g `http`, `https` - `port` - `path` **Important note**: if you modify anything inside these lists, default `config.exs` values will be overwritten, which may result in breakage, to make sure this does not happen please copy the default value for the list from `config.exs` and modify/add only what you need -Example: +Example: ```elixir config :pleroma, Pleroma.Web.Endpoint, url: [host: "example.com", port: 2020, scheme: "https"], @@ -412,3 +412,8 @@ Pleroma account will be created with the same name as the LDAP user name. * `Pleroma.Web.Auth.PleromaAuthenticator`: default database authenticator * `Pleroma.Web.Auth.LDAPAuthenticator`: LDAP authentication + +## Pleroma.ScheduledActivity + +* `daily_user_limit`: the number of scheduled activities a user is allowed to create in a single day +* `total_user_limit`: the number of scheduled activities a user is allowed to create in total diff --git a/lib/pleroma/object.ex b/lib/pleroma/object.ex index 013d62157..786d6296c 100644 --- a/lib/pleroma/object.ex +++ b/lib/pleroma/object.ex @@ -184,4 +184,12 @@ defmodule Pleroma.Object do _ -> {:error, "Not found"} end end + + def enforce_user_objects(user, object_ids) do + Object + |> where([o], fragment("?->>'actor' = ?", o.data, ^user.ap_id)) + |> where([o], o.id in ^object_ids) + |> select([o], o.id) + |> Repo.all() + end end diff --git a/lib/pleroma/scheduled_activity.ex b/lib/pleroma/scheduled_activity.ex index 9fdc13990..723eb6dc3 100644 --- a/lib/pleroma/scheduled_activity.ex +++ b/lib/pleroma/scheduled_activity.ex @@ -5,9 +5,12 @@ defmodule Pleroma.ScheduledActivity do use Ecto.Schema + alias Pleroma.Config + alias Pleroma.Object alias Pleroma.Repo alias Pleroma.ScheduledActivity alias Pleroma.User + alias Pleroma.Web.CommonAPI.Utils import Ecto.Query import Ecto.Changeset @@ -25,11 +28,69 @@ defmodule Pleroma.ScheduledActivity do def changeset(%ScheduledActivity{} = scheduled_activity, attrs) do scheduled_activity |> cast(attrs, [:scheduled_at, :params]) + |> validate_required([:scheduled_at, :params]) + |> validate_scheduled_at() + |> with_media_attachments() end + defp with_media_attachments( + %{changes: %{params: %{"media_ids" => media_ids} = params}} = changeset + ) + when is_list(media_ids) do + user = User.get_cached_by_id(changeset.data.user_id) + media_ids = Object.enforce_user_objects(user, media_ids) |> Enum.map(&to_string(&1)) + media_attachments = Utils.attachments_from_ids(%{"media_ids" => media_ids}) + + params = + params + |> Map.put("media_attachments", media_attachments) + |> Map.put("media_ids", media_ids) + + put_change(changeset, :params, params) + end + + defp with_media_attachments(changeset), do: changeset + def update_changeset(%ScheduledActivity{} = scheduled_activity, attrs) do scheduled_activity |> cast(attrs, [:scheduled_at]) + |> validate_required([:scheduled_at]) + |> validate_scheduled_at() + end + + def validate_scheduled_at(changeset) do + validate_change(changeset, :scheduled_at, fn _, scheduled_at -> + cond do + not far_enough?(scheduled_at) -> + [scheduled_at: "must be at least 5 minutes from now"] + + exceeds_daily_user_limit?(changeset.data.user_id, scheduled_at) -> + [scheduled_at: "daily limit exceeded"] + + exceeds_total_user_limit?(changeset.data.user_id) -> + [scheduled_at: "total limit exceeded"] + + true -> + [] + end + end) + end + + def exceeds_daily_user_limit?(user_id, scheduled_at) do + ScheduledActivity + |> where(user_id: ^user_id) + |> where([s], type(s.scheduled_at, :date) == type(^scheduled_at, :date)) + |> select([u], count(u.id)) + |> Repo.one() + |> Kernel.>=(Config.get([ScheduledActivity, :daily_user_limit])) + end + + def exceeds_total_user_limit?(user_id) do + ScheduledActivity + |> where(user_id: ^user_id) + |> select([u], count(u.id)) + |> Repo.one() + |> Kernel.>=(Config.get([ScheduledActivity, :total_user_limit])) end def far_enough?(scheduled_at) when is_binary(scheduled_at) do @@ -64,23 +125,15 @@ defmodule Pleroma.ScheduledActivity do |> Repo.one() end - def update(%User{} = user, scheduled_activity_id, attrs) do - with %ScheduledActivity{} = scheduled_activity <- get(user, scheduled_activity_id) do - scheduled_activity - |> update_changeset(attrs) - |> Repo.update() - else - nil -> {:error, :not_found} - end + def update(scheduled_activity, attrs) do + scheduled_activity + |> update_changeset(attrs) + |> Repo.update() end - def delete(%User{} = user, scheduled_activity_id) do - with %ScheduledActivity{} = scheduled_activity <- get(user, scheduled_activity_id) do - scheduled_activity - |> Repo.delete() - else - nil -> {:error, :not_found} - end + def delete(scheduled_activity) do + scheduled_activity + |> Repo.delete() end def for_user_query(%User{} = user) do diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index 863fc3954..6cb5df378 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -390,18 +390,28 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do %{assigns: %{user: user}} = conn, %{"id" => scheduled_activity_id} = params ) do - with {:ok, scheduled_activity} <- - ScheduledActivity.update(user, scheduled_activity_id, params) do + with %ScheduledActivity{} = scheduled_activity <- + ScheduledActivity.get(user, scheduled_activity_id), + {:ok, scheduled_activity} <- ScheduledActivity.update(scheduled_activity, params) do conn |> put_view(ScheduledActivityView) |> render("show.json", %{scheduled_activity: scheduled_activity}) + else + nil -> {:error, :not_found} + error -> error end end def delete_scheduled_status(%{assigns: %{user: user}} = conn, %{"id" => scheduled_activity_id}) do - with {:ok, %ScheduledActivity{}} <- ScheduledActivity.delete(user, scheduled_activity_id) do + with %ScheduledActivity{} = scheduled_activity <- + ScheduledActivity.get(user, scheduled_activity_id), + {:ok, scheduled_activity} <- ScheduledActivity.delete(scheduled_activity) do conn - |> json(%{}) + |> put_view(ScheduledActivityView) + |> render("show.json", %{scheduled_activity: scheduled_activity}) + else + nil -> {:error, :not_found} + error -> error end end diff --git a/lib/pleroma/web/mastodon_api/views/scheduled_activity_view.ex b/lib/pleroma/web/mastodon_api/views/scheduled_activity_view.ex index 87aa3729e..1ebff7aba 100644 --- a/lib/pleroma/web/mastodon_api/views/scheduled_activity_view.ex +++ b/lib/pleroma/web/mastodon_api/views/scheduled_activity_view.ex @@ -8,6 +8,7 @@ defmodule Pleroma.Web.MastodonAPI.ScheduledActivityView do alias Pleroma.ScheduledActivity alias Pleroma.Web.CommonAPI alias Pleroma.Web.MastodonAPI.ScheduledActivityView + alias Pleroma.Web.MastodonAPI.StatusView def render("index.json", %{scheduled_activities: scheduled_activities}) do render_many(scheduled_activities, ScheduledActivityView, "show.json") @@ -17,7 +18,36 @@ defmodule Pleroma.Web.MastodonAPI.ScheduledActivityView do %{ id: scheduled_activity.id |> to_string, scheduled_at: scheduled_activity.scheduled_at |> CommonAPI.Utils.to_masto_date(), - params: scheduled_activity.params + params: status_params(scheduled_activity.params) } + |> with_media_attachments(scheduled_activity) + end + + defp with_media_attachments(data, %{params: %{"media_attachments" => media_attachments}}) do + attachments = render_many(media_attachments, StatusView, "attachment.json", as: :attachment) + Map.put(data, :media_attachments, attachments) + end + + defp with_media_attachments(data, _), do: data + + defp status_params(params) do + data = %{ + text: params["status"], + sensitive: params["sensitive"], + spoiler_text: params["spoiler_text"], + visibility: params["visibility"], + scheduled_at: params["scheduled_at"], + poll: params["poll"], + in_reply_to_id: params["in_reply_to_id"] + } + + data = + if media_ids = params["media_ids"] do + Map.put(data, :media_ids, media_ids) + else + data + end + + data end end diff --git a/priv/repo/migrations/20190328053912_create_scheduled_activities.exs b/priv/repo/migrations/20190328053912_create_scheduled_activities.exs index dc2436dce..dd737e25a 100644 --- a/priv/repo/migrations/20190328053912_create_scheduled_activities.exs +++ b/priv/repo/migrations/20190328053912_create_scheduled_activities.exs @@ -11,5 +11,6 @@ defmodule Pleroma.Repo.Migrations.CreateScheduledActivities do end create(index(:scheduled_activities, [:scheduled_at])) + create(index(:scheduled_activities, [:user_id])) end end diff --git a/test/scheduled_activity_test.exs b/test/scheduled_activity_test.exs new file mode 100644 index 000000000..c49c65c0a --- /dev/null +++ b/test/scheduled_activity_test.exs @@ -0,0 +1,93 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2018 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.ScheduledActivityTest do + use Pleroma.DataCase + alias Pleroma.Config + alias Pleroma.DataCase + alias Pleroma.ScheduledActivity + alias Pleroma.Web.ActivityPub.ActivityPub + import Pleroma.Factory + + setup context do + Config.put([ScheduledActivity, :daily_user_limit], 2) + Config.put([ScheduledActivity, :total_user_limit], 3) + DataCase.ensure_local_uploader(context) + end + + describe "creation" do + test "when daily user limit is exceeded" do + user = insert(:user) + + today = + NaiveDateTime.utc_now() + |> NaiveDateTime.add(:timer.minutes(6), :millisecond) + |> NaiveDateTime.to_iso8601() + + attrs = %{params: %{}, scheduled_at: today} + {:ok, _} = ScheduledActivity.create(user, attrs) + {:ok, _} = ScheduledActivity.create(user, attrs) + {:error, changeset} = ScheduledActivity.create(user, attrs) + assert changeset.errors == [scheduled_at: {"daily limit exceeded", []}] + end + + test "when total user limit is exceeded" do + user = insert(:user) + + today = + NaiveDateTime.utc_now() + |> NaiveDateTime.add(:timer.minutes(6), :millisecond) + |> NaiveDateTime.to_iso8601() + + tomorrow = + NaiveDateTime.utc_now() + |> NaiveDateTime.add(:timer.hours(24), :millisecond) + |> NaiveDateTime.to_iso8601() + + {:ok, _} = ScheduledActivity.create(user, %{params: %{}, scheduled_at: today}) + {:ok, _} = ScheduledActivity.create(user, %{params: %{}, scheduled_at: today}) + {:ok, _} = ScheduledActivity.create(user, %{params: %{}, scheduled_at: tomorrow}) + {:error, changeset} = ScheduledActivity.create(user, %{params: %{}, scheduled_at: tomorrow}) + assert changeset.errors == [scheduled_at: {"total limit exceeded", []}] + end + + test "when scheduled_at is earlier than 5 minute from now" do + user = insert(:user) + + scheduled_at = + NaiveDateTime.utc_now() + |> NaiveDateTime.add(:timer.minutes(4), :millisecond) + |> NaiveDateTime.to_iso8601() + + attrs = %{params: %{}, scheduled_at: scheduled_at} + {:error, changeset} = ScheduledActivity.create(user, attrs) + assert changeset.errors == [scheduled_at: {"must be at least 5 minutes from now", []}] + end + + test "excludes attachments belonging to another user" do + user = insert(:user) + another_user = insert(:user) + + scheduled_at = + NaiveDateTime.utc_now() + |> NaiveDateTime.add(:timer.minutes(10), :millisecond) + |> NaiveDateTime.to_iso8601() + + file = %Plug.Upload{ + content_type: "image/jpg", + path: Path.absname("test/fixtures/image.jpg"), + filename: "an_image.jpg" + } + + {:ok, user_upload} = ActivityPub.upload(file, actor: user.ap_id) + {:ok, another_user_upload} = ActivityPub.upload(file, actor: another_user.ap_id) + + media_ids = [user_upload.id, another_user_upload.id] + attrs = %{params: %{"media_ids" => media_ids}, scheduled_at: scheduled_at} + {:ok, scheduled_activity} = ScheduledActivity.create(user, attrs) + assert to_string(user_upload.id) in scheduled_activity.params["media_ids"] + refute to_string(another_user_upload.id) in scheduled_activity.params["media_ids"] + end + end +end diff --git a/test/support/factory.ex b/test/support/factory.ex index 667f59e8c..608f8d46b 100644 --- a/test/support/factory.ex +++ b/test/support/factory.ex @@ -23,14 +23,6 @@ defmodule Pleroma.Factory do } end - def scheduled_activity_factory do - %Pleroma.ScheduledActivity{ - user: build(:user), - scheduled_at: NaiveDateTime.add(NaiveDateTime.utc_now(), :timer.minutes(60), :millisecond), - params: build(:note) |> Map.from_struct() |> Map.get(:data) - } - end - def note_factory(attrs \\ %{}) do text = sequence(:text, &"This is :moominmamma: note #{&1}") @@ -275,4 +267,12 @@ defmodule Pleroma.Factory do user: build(:user) } end + + def scheduled_activity_factory do + %Pleroma.ScheduledActivity{ + user: build(:user), + scheduled_at: NaiveDateTime.add(NaiveDateTime.utc_now(), :timer.minutes(60), :millisecond), + params: build(:note) |> Map.from_struct() |> Map.get(:data) + } + end end diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index 0ec66ab73..ae2375696 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -2427,6 +2427,31 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do assert [] == Repo.all(Activity) end + test "creates a scheduled activity with a media attachment", %{conn: conn} do + user = insert(:user) + scheduled_at = NaiveDateTime.add(NaiveDateTime.utc_now(), :timer.minutes(120), :millisecond) + + file = %Plug.Upload{ + content_type: "image/jpg", + path: Path.absname("test/fixtures/image.jpg"), + filename: "an_image.jpg" + } + + {:ok, upload} = ActivityPub.upload(file, actor: user.ap_id) + + conn = + conn + |> assign(:user, user) + |> post("/api/v1/statuses", %{ + "media_ids" => [to_string(upload.id)], + "status" => "scheduled", + "scheduled_at" => scheduled_at + }) + + assert %{"media_attachments" => [media_attachment]} = json_response(conn, 200) + assert %{"type" => "image"} = media_attachment + end + test "skips the scheduling and creates the activity if scheduled_at is earlier than 5 minutes from now", %{conn: conn} do user = insert(:user) diff --git a/test/web/mastodon_api/scheduled_activity_view_test.exs b/test/web/mastodon_api/scheduled_activity_view_test.exs new file mode 100644 index 000000000..26747a0c0 --- /dev/null +++ b/test/web/mastodon_api/scheduled_activity_view_test.exs @@ -0,0 +1,68 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2018 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.MastodonAPI.ScheduledActivityViewTest do + use Pleroma.DataCase + alias Pleroma.ScheduledActivity + alias Pleroma.Web.ActivityPub.ActivityPub + alias Pleroma.Web.CommonAPI + alias Pleroma.Web.CommonAPI.Utils + alias Pleroma.Web.MastodonAPI.ScheduledActivityView + alias Pleroma.Web.MastodonAPI.StatusView + import Pleroma.Factory + + test "A scheduled activity with a media attachment" do + user = insert(:user) + {:ok, activity} = CommonAPI.post(user, %{"status" => "hi"}) + + scheduled_at = + NaiveDateTime.utc_now() + |> NaiveDateTime.add(:timer.minutes(10), :millisecond) + |> NaiveDateTime.to_iso8601() + + file = %Plug.Upload{ + content_type: "image/jpg", + path: Path.absname("test/fixtures/image.jpg"), + filename: "an_image.jpg" + } + + {:ok, upload} = ActivityPub.upload(file, actor: user.ap_id) + + attrs = %{ + params: %{ + "media_ids" => [upload.id], + "status" => "hi", + "sensitive" => true, + "spoiler_text" => "spoiler", + "visibility" => "unlisted", + "in_reply_to_id" => to_string(activity.id) + }, + scheduled_at: scheduled_at + } + + {:ok, scheduled_activity} = ScheduledActivity.create(user, attrs) + result = ScheduledActivityView.render("show.json", %{scheduled_activity: scheduled_activity}) + + expected = %{ + id: to_string(scheduled_activity.id), + media_attachments: + %{"media_ids" => [upload.id]} + |> Utils.attachments_from_ids() + |> Enum.map(&StatusView.render("attachment.json", %{attachment: &1})), + params: %{ + in_reply_to_id: to_string(activity.id), + media_ids: [to_string(upload.id)], + poll: nil, + scheduled_at: nil, + sensitive: true, + spoiler_text: "spoiler", + text: "hi", + visibility: "unlisted" + }, + scheduled_at: Utils.to_masto_date(scheduled_activity.scheduled_at) + } + + assert expected == result + end +end From 2056efa714460faaf25f6bc03ab643f5a2e8cd3d Mon Sep 17 00:00:00 2001 From: eugenijm Date: Wed, 3 Apr 2019 18:55:04 +0300 Subject: [PATCH 26/26] Add scheduler for sending scheduled activities to the queue --- config/config.exs | 12 ++-- config/test.exs | 5 ++ docs/config.md | 8 ++- lib/pleroma/application.ex | 3 +- lib/pleroma/object.ex | 8 --- lib/pleroma/scheduled_activity.ex | 34 ++++++++--- lib/pleroma/scheduled_activity_worker.ex | 58 +++++++++++++++++++ .../mastodon_api/mastodon_api_controller.ex | 26 ++++++--- .../views/scheduled_activity_view.ex | 12 ++-- test/scheduled_activity_test.exs | 31 +--------- test/scheduled_activity_worker_test.exs | 19 ++++++ .../mastodon_api_controller_test.exs | 46 +++++++++++++++ .../scheduled_activity_view_test.exs | 2 +- 13 files changed, 196 insertions(+), 68 deletions(-) create mode 100644 lib/pleroma/scheduled_activity_worker.ex create mode 100644 test/scheduled_activity_worker_test.exs diff --git a/config/config.exs b/config/config.exs index 79cef87e6..8a977ece5 100644 --- a/config/config.exs +++ b/config/config.exs @@ -361,16 +361,13 @@ config :pleroma, Pleroma.Web.Federator.RetryQueue, config :pleroma_job_queue, :queues, federator_incoming: 50, federator_outgoing: 50, - mailer: 10 + mailer: 10, + scheduled_activities: 10 config :pleroma, :fetch_initial_posts, enabled: false, pages: 5 -config :pleroma, Pleroma.ScheduledActivity, - daily_user_limit: 25, - total_user_limit: 100 - config :auto_linker, opts: [ scheme: true, @@ -396,6 +393,11 @@ config :pleroma, Pleroma.Mailer, adapter: Swoosh.Adapters.Sendmail config :prometheus, Pleroma.Web.Endpoint.MetricsExporter, path: "/api/pleroma/app_metrics" +config :pleroma, Pleroma.ScheduledActivity, + daily_user_limit: 25, + total_user_limit: 300, + enabled: true + # Import environment specific config. This must remain at the bottom # of this file so it overrides the configuration defined above. import_config "#{Mix.env()}.exs" diff --git a/config/test.exs b/config/test.exs index 6a7b9067e..894fa8d3d 100644 --- a/config/test.exs +++ b/config/test.exs @@ -50,6 +50,11 @@ config :web_push_encryption, :http_client, Pleroma.Web.WebPushHttpClientMock config :pleroma_job_queue, disabled: true +config :pleroma, Pleroma.ScheduledActivity, + daily_user_limit: 2, + total_user_limit: 3, + enabled: false + try do import_config "test.secret.exs" rescue diff --git a/docs/config.md b/docs/config.md index df21beff3..ba0759e87 100644 --- a/docs/config.md +++ b/docs/config.md @@ -317,6 +317,7 @@ Pleroma has the following queues: * `federator_outgoing` - Outgoing federation * `federator_incoming` - Incoming federation * `mailer` - Email sender, see [`Pleroma.Mailer`](#pleroma-mailer) +* `scheduled_activities` - Scheduled activities, see [`Pleroma.ScheduledActivities`](#pleromascheduledactivity) Example: @@ -413,7 +414,8 @@ Pleroma account will be created with the same name as the LDAP user name. * `Pleroma.Web.Auth.PleromaAuthenticator`: default database authenticator * `Pleroma.Web.Auth.LDAPAuthenticator`: LDAP authentication -## Pleroma.ScheduledActivity +## Pleroma.ScheduledActivity -* `daily_user_limit`: the number of scheduled activities a user is allowed to create in a single day -* `total_user_limit`: the number of scheduled activities a user is allowed to create in total +* `daily_user_limit`: the number of scheduled activities a user is allowed to create in a single day (Default: `25`) +* `total_user_limit`: the number of scheduled activities a user is allowed to create in total (Default: `300`) +* `enabled`: whether scheduled activities are sent to the job queue to be executed diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 1fc3fb728..f0cb7d9a8 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -104,7 +104,8 @@ defmodule Pleroma.Application do ], id: :cachex_idem ), - worker(Pleroma.FlakeId, []) + worker(Pleroma.FlakeId, []), + worker(Pleroma.ScheduledActivityWorker, []) ] ++ hackney_pool_children() ++ [ diff --git a/lib/pleroma/object.ex b/lib/pleroma/object.ex index 786d6296c..013d62157 100644 --- a/lib/pleroma/object.ex +++ b/lib/pleroma/object.ex @@ -184,12 +184,4 @@ defmodule Pleroma.Object do _ -> {:error, "Not found"} end end - - def enforce_user_objects(user, object_ids) do - Object - |> where([o], fragment("?->>'actor' = ?", o.data, ^user.ap_id)) - |> where([o], o.id in ^object_ids) - |> select([o], o.id) - |> Repo.all() - end end diff --git a/lib/pleroma/scheduled_activity.ex b/lib/pleroma/scheduled_activity.ex index 723eb6dc3..de0e54699 100644 --- a/lib/pleroma/scheduled_activity.ex +++ b/lib/pleroma/scheduled_activity.ex @@ -6,7 +6,6 @@ defmodule Pleroma.ScheduledActivity do use Ecto.Schema alias Pleroma.Config - alias Pleroma.Object alias Pleroma.Repo alias Pleroma.ScheduledActivity alias Pleroma.User @@ -37,8 +36,6 @@ defmodule Pleroma.ScheduledActivity do %{changes: %{params: %{"media_ids" => media_ids} = params}} = changeset ) when is_list(media_ids) do - user = User.get_cached_by_id(changeset.data.user_id) - media_ids = Object.enforce_user_objects(user, media_ids) |> Enum.map(&to_string(&1)) media_attachments = Utils.attachments_from_ids(%{"media_ids" => media_ids}) params = @@ -79,8 +76,8 @@ defmodule Pleroma.ScheduledActivity do def exceeds_daily_user_limit?(user_id, scheduled_at) do ScheduledActivity |> where(user_id: ^user_id) - |> where([s], type(s.scheduled_at, :date) == type(^scheduled_at, :date)) - |> select([u], count(u.id)) + |> where([sa], type(sa.scheduled_at, :date) == type(^scheduled_at, :date)) + |> select([sa], count(sa.id)) |> Repo.one() |> Kernel.>=(Config.get([ScheduledActivity, :daily_user_limit])) end @@ -88,7 +85,7 @@ defmodule Pleroma.ScheduledActivity do def exceeds_total_user_limit?(user_id) do ScheduledActivity |> where(user_id: ^user_id) - |> select([u], count(u.id)) + |> select([sa], count(sa.id)) |> Repo.one() |> Kernel.>=(Config.get([ScheduledActivity, :total_user_limit])) end @@ -125,19 +122,40 @@ defmodule Pleroma.ScheduledActivity do |> Repo.one() end - def update(scheduled_activity, attrs) do + def update(%ScheduledActivity{} = scheduled_activity, attrs) do scheduled_activity |> update_changeset(attrs) |> Repo.update() end - def delete(scheduled_activity) do + def delete(%ScheduledActivity{} = scheduled_activity) do scheduled_activity |> Repo.delete() end + def delete(id) when is_binary(id) or is_integer(id) do + ScheduledActivity + |> where(id: ^id) + |> select([sa], sa) + |> Repo.delete_all() + |> case do + {1, [scheduled_activity]} -> {:ok, scheduled_activity} + _ -> :error + end + end + def for_user_query(%User{} = user) do ScheduledActivity |> where(user_id: ^user.id) end + + def due_activities(offset \\ 0) do + naive_datetime = + NaiveDateTime.utc_now() + |> NaiveDateTime.add(offset, :millisecond) + + ScheduledActivity + |> where([sa], sa.scheduled_at < ^naive_datetime) + |> Repo.all() + end end diff --git a/lib/pleroma/scheduled_activity_worker.ex b/lib/pleroma/scheduled_activity_worker.ex new file mode 100644 index 000000000..65b38622f --- /dev/null +++ b/lib/pleroma/scheduled_activity_worker.ex @@ -0,0 +1,58 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.ScheduledActivityWorker do + @moduledoc """ + Sends scheduled activities to the job queue. + """ + + alias Pleroma.Config + alias Pleroma.ScheduledActivity + alias Pleroma.User + alias Pleroma.Web.CommonAPI + use GenServer + require Logger + + @schedule_interval :timer.minutes(1) + + def start_link do + GenServer.start_link(__MODULE__, nil) + end + + def init(_) do + if Config.get([ScheduledActivity, :enabled]) do + schedule_next() + {:ok, nil} + else + :ignore + end + end + + def perform(:execute, scheduled_activity_id) do + try do + {:ok, scheduled_activity} = ScheduledActivity.delete(scheduled_activity_id) + %User{} = user = User.get_cached_by_id(scheduled_activity.user_id) + {:ok, _result} = CommonAPI.post(user, scheduled_activity.params) + rescue + error -> + Logger.error( + "#{__MODULE__} Couldn't create a status from the scheduled activity: #{inspect(error)}" + ) + end + end + + def handle_info(:perform, state) do + ScheduledActivity.due_activities(@schedule_interval) + |> Enum.each(fn scheduled_activity -> + PleromaJobQueue.enqueue(:scheduled_activities, __MODULE__, [:execute, scheduled_activity.id]) + end) + + schedule_next() + {:noreply, state} + end + + defp schedule_next do + Process.send_after(self(), :perform, @schedule_interval) + end +end diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index 6cb5df378..fc8a2458c 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -5,6 +5,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do use Pleroma.Web, :controller + alias Ecto.Changeset alias Pleroma.Activity alias Pleroma.Config alias Pleroma.Filter @@ -438,14 +439,12 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do scheduled_at = params["scheduled_at"] if scheduled_at && ScheduledActivity.far_enough?(scheduled_at) do - {:ok, scheduled_activity} = - Cachex.fetch!(:idempotency_cache, idempotency_key, fn _ -> - ScheduledActivity.create(user, %{"params" => params, "scheduled_at" => scheduled_at}) - end) - - conn - |> put_view(ScheduledActivityView) - |> render("show.json", %{scheduled_activity: scheduled_activity}) + with {:ok, scheduled_activity} <- + ScheduledActivity.create(user, %{"params" => params, "scheduled_at" => scheduled_at}) do + conn + |> put_view(ScheduledActivityView) + |> render("show.json", %{scheduled_activity: scheduled_activity}) + end else params = Map.drop(params, ["scheduled_at"]) @@ -1474,6 +1473,17 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do # fallback action # + def errors(conn, {:error, %Changeset{} = changeset}) do + error_message = + changeset + |> Changeset.traverse_errors(fn {message, _opt} -> message end) + |> Enum.map_join(", ", fn {_k, v} -> v end) + + conn + |> put_status(422) + |> json(%{error: error_message}) + end + def errors(conn, {:error, :not_found}) do conn |> put_status(404) diff --git a/lib/pleroma/web/mastodon_api/views/scheduled_activity_view.ex b/lib/pleroma/web/mastodon_api/views/scheduled_activity_view.ex index 1ebff7aba..0aae15ab9 100644 --- a/lib/pleroma/web/mastodon_api/views/scheduled_activity_view.ex +++ b/lib/pleroma/web/mastodon_api/views/scheduled_activity_view.ex @@ -16,16 +16,20 @@ defmodule Pleroma.Web.MastodonAPI.ScheduledActivityView do def render("show.json", %{scheduled_activity: %ScheduledActivity{} = scheduled_activity}) do %{ - id: scheduled_activity.id |> to_string, - scheduled_at: scheduled_activity.scheduled_at |> CommonAPI.Utils.to_masto_date(), + id: to_string(scheduled_activity.id), + scheduled_at: CommonAPI.Utils.to_masto_date(scheduled_activity.scheduled_at), params: status_params(scheduled_activity.params) } |> with_media_attachments(scheduled_activity) end defp with_media_attachments(data, %{params: %{"media_attachments" => media_attachments}}) do - attachments = render_many(media_attachments, StatusView, "attachment.json", as: :attachment) - Map.put(data, :media_attachments, attachments) + try do + attachments = render_many(media_attachments, StatusView, "attachment.json", as: :attachment) + Map.put(data, :media_attachments, attachments) + rescue + _ -> data + end end defp with_media_attachments(data, _), do: data diff --git a/test/scheduled_activity_test.exs b/test/scheduled_activity_test.exs index c49c65c0a..edc7cc3f9 100644 --- a/test/scheduled_activity_test.exs +++ b/test/scheduled_activity_test.exs @@ -4,15 +4,11 @@ defmodule Pleroma.ScheduledActivityTest do use Pleroma.DataCase - alias Pleroma.Config alias Pleroma.DataCase alias Pleroma.ScheduledActivity - alias Pleroma.Web.ActivityPub.ActivityPub import Pleroma.Factory setup context do - Config.put([ScheduledActivity, :daily_user_limit], 2) - Config.put([ScheduledActivity, :total_user_limit], 3) DataCase.ensure_local_uploader(context) end @@ -42,7 +38,7 @@ defmodule Pleroma.ScheduledActivityTest do tomorrow = NaiveDateTime.utc_now() - |> NaiveDateTime.add(:timer.hours(24), :millisecond) + |> NaiveDateTime.add(:timer.hours(36), :millisecond) |> NaiveDateTime.to_iso8601() {:ok, _} = ScheduledActivity.create(user, %{params: %{}, scheduled_at: today}) @@ -64,30 +60,5 @@ defmodule Pleroma.ScheduledActivityTest do {:error, changeset} = ScheduledActivity.create(user, attrs) assert changeset.errors == [scheduled_at: {"must be at least 5 minutes from now", []}] end - - test "excludes attachments belonging to another user" do - user = insert(:user) - another_user = insert(:user) - - scheduled_at = - NaiveDateTime.utc_now() - |> NaiveDateTime.add(:timer.minutes(10), :millisecond) - |> NaiveDateTime.to_iso8601() - - file = %Plug.Upload{ - content_type: "image/jpg", - path: Path.absname("test/fixtures/image.jpg"), - filename: "an_image.jpg" - } - - {:ok, user_upload} = ActivityPub.upload(file, actor: user.ap_id) - {:ok, another_user_upload} = ActivityPub.upload(file, actor: another_user.ap_id) - - media_ids = [user_upload.id, another_user_upload.id] - attrs = %{params: %{"media_ids" => media_ids}, scheduled_at: scheduled_at} - {:ok, scheduled_activity} = ScheduledActivity.create(user, attrs) - assert to_string(user_upload.id) in scheduled_activity.params["media_ids"] - refute to_string(another_user_upload.id) in scheduled_activity.params["media_ids"] - end end end diff --git a/test/scheduled_activity_worker_test.exs b/test/scheduled_activity_worker_test.exs new file mode 100644 index 000000000..b9c91dda6 --- /dev/null +++ b/test/scheduled_activity_worker_test.exs @@ -0,0 +1,19 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2018 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.ScheduledActivityWorkerTest do + use Pleroma.DataCase + alias Pleroma.ScheduledActivity + import Pleroma.Factory + + test "creates a status from the scheduled activity" do + user = insert(:user) + scheduled_activity = insert(:scheduled_activity, user: user, params: %{status: "hi"}) + Pleroma.ScheduledActivityWorker.perform(:execute, scheduled_activity.id) + + refute Repo.get(ScheduledActivity, scheduled_activity.id) + activity = Repo.all(Pleroma.Activity) |> Enum.find(&(&1.actor == user.ap_id)) + assert activity.data["object"]["content"] == "hi" + end +end diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index ae2375696..cd01116e2 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -2471,6 +2471,52 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do assert [] == Repo.all(ScheduledActivity) end + test "returns error when daily user limit is exceeded", %{conn: conn} do + user = insert(:user) + + today = + NaiveDateTime.utc_now() + |> NaiveDateTime.add(:timer.minutes(6), :millisecond) + |> NaiveDateTime.to_iso8601() + + attrs = %{params: %{}, scheduled_at: today} + {:ok, _} = ScheduledActivity.create(user, attrs) + {:ok, _} = ScheduledActivity.create(user, attrs) + + conn = + conn + |> assign(:user, user) + |> post("/api/v1/statuses", %{"status" => "scheduled", "scheduled_at" => today}) + + assert %{"error" => "daily limit exceeded"} == json_response(conn, 422) + end + + test "returns error when total user limit is exceeded", %{conn: conn} do + user = insert(:user) + + today = + NaiveDateTime.utc_now() + |> NaiveDateTime.add(:timer.minutes(6), :millisecond) + |> NaiveDateTime.to_iso8601() + + tomorrow = + NaiveDateTime.utc_now() + |> NaiveDateTime.add(:timer.hours(36), :millisecond) + |> NaiveDateTime.to_iso8601() + + attrs = %{params: %{}, scheduled_at: today} + {:ok, _} = ScheduledActivity.create(user, attrs) + {:ok, _} = ScheduledActivity.create(user, attrs) + {:ok, _} = ScheduledActivity.create(user, %{params: %{}, scheduled_at: tomorrow}) + + conn = + conn + |> assign(:user, user) + |> post("/api/v1/statuses", %{"status" => "scheduled", "scheduled_at" => tomorrow}) + + assert %{"error" => "total limit exceeded"} == json_response(conn, 422) + end + test "shows scheduled activities", %{conn: conn} do user = insert(:user) scheduled_activity_id1 = insert(:scheduled_activity, user: user).id |> to_string() diff --git a/test/web/mastodon_api/scheduled_activity_view_test.exs b/test/web/mastodon_api/scheduled_activity_view_test.exs index 26747a0c0..ecbb855d4 100644 --- a/test/web/mastodon_api/scheduled_activity_view_test.exs +++ b/test/web/mastodon_api/scheduled_activity_view_test.exs @@ -52,7 +52,7 @@ defmodule Pleroma.Web.MastodonAPI.ScheduledActivityViewTest do |> Enum.map(&StatusView.render("attachment.json", %{attachment: &1})), params: %{ in_reply_to_id: to_string(activity.id), - media_ids: [to_string(upload.id)], + media_ids: [upload.id], poll: nil, scheduled_at: nil, sensitive: true,