Merge branch 'develop' of git.pleroma.social:pleroma/pleroma into frontend-bundles-downloads

This commit is contained in:
lain 2020-08-06 15:22:20 +02:00
commit bf95dfb240
31 changed files with 358 additions and 61 deletions

View file

@ -104,6 +104,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
- Fix CSP policy generation to include remote Captcha services
- Fix edge case where MediaProxy truncates media, usually caused when Caddy is serving content for the other Federated instance.
- Emoji Packs could not be listed when instance was set to `public: false`
- Fix whole_word always returning false on filter get requests
## [Unreleased (patch)]

View file

@ -120,6 +120,8 @@ config :pleroma, Pleroma.Uploaders.S3,
config :tzdata, :autoupdate, :disabled
config :pleroma, :mrf, policies: []
if File.exists?("./config/test.secret.exs") do
import_config "test.secret.exs"
else

View file

@ -47,6 +47,7 @@ defmodule Pleroma.Application do
Pleroma.ApplicationRequirements.verify!()
setup_instrumenters()
load_custom_modules()
check_system_commands()
Pleroma.Docs.JSON.compile()
adapter = Application.get_env(:tesla, :adapter)
@ -249,4 +250,21 @@ defmodule Pleroma.Application do
end
defp http_children(_, _), do: []
defp check_system_commands do
filters = Config.get([Pleroma.Upload, :filters])
check_filter = fn filter, command_required ->
with true <- filter in filters,
false <- Pleroma.Utils.command_available?(command_required) do
Logger.error(
"#{filter} is specified in list of Pleroma.Upload filters, but the #{command_required} command is not found"
)
end
end
check_filter.(Pleroma.Upload.Filters.Exiftool, "exiftool")
check_filter.(Pleroma.Upload.Filters.Mogrify, "mogrify")
check_filter.(Pleroma.Upload.Filters.Mogrifun, "mogrify")
end
end

View file

@ -11,12 +11,10 @@ defmodule Pleroma.Config do
def get([key], default), do: get(key, default)
def get([parent_key | keys], default) do
case :pleroma
|> Application.get_env(parent_key)
|> get_in(keys) do
nil -> default
any -> any
def get([_ | _] = path, default) do
case fetch(path) do
{:ok, value} -> value
:error -> default
end
end
@ -34,6 +32,24 @@ defmodule Pleroma.Config do
end
end
def fetch(key) when is_atom(key), do: fetch([key])
def fetch([root_key | keys]) do
Enum.reduce_while(keys, Application.fetch_env(:pleroma, root_key), fn
key, {:ok, config} when is_map(config) or is_list(config) ->
case Access.fetch(config, key) do
:error ->
{:halt, :error}
value ->
{:cont, value}
end
_key, _config ->
{:halt, :error}
end)
end
def put([key], value), do: put(key, value)
def put([parent_key | keys], value) do
@ -50,12 +66,15 @@ defmodule Pleroma.Config do
def delete([key]), do: delete(key)
def delete([parent_key | keys]) do
{_, parent} =
Application.get_env(:pleroma, parent_key)
|> get_and_update_in(keys, fn _ -> :pop end)
def delete([parent_key | keys] = path) do
with {:ok, _} <- fetch(path) do
{_, parent} =
parent_key
|> get()
|> get_and_update_in(keys, fn _ -> :pop end)
Application.put_env(:pleroma, parent_key, parent)
Application.put_env(:pleroma, parent_key, parent)
end
end
def delete(key) do

View file

@ -9,9 +9,17 @@ defmodule Pleroma.Upload.Filter.Exiftool do
"""
@behaviour Pleroma.Upload.Filter
@spec filter(Pleroma.Upload.t()) :: :ok | {:error, String.t()}
def filter(%Pleroma.Upload{tempfile: file, content_type: "image" <> _}) do
System.cmd("exiftool", ["-overwrite_original", "-gps:all=", file], parallelism: true)
:ok
try do
case System.cmd("exiftool", ["-overwrite_original", "-gps:all=", file], parallelism: true) do
{_response, 0} -> :ok
{error, 1} -> {:error, error}
end
rescue
_e in ErlangError ->
{:error, "exiftool command not found"}
end
end
def filter(_), do: :ok

View file

@ -34,10 +34,15 @@ defmodule Pleroma.Upload.Filter.Mogrifun do
[{"fill", "yellow"}, {"tint", "40"}]
]
@spec filter(Pleroma.Upload.t()) :: :ok | {:error, String.t()}
def filter(%Pleroma.Upload{tempfile: file, content_type: "image" <> _}) do
Filter.Mogrify.do_filter(file, [Enum.random(@filters)])
:ok
try do
Filter.Mogrify.do_filter(file, [Enum.random(@filters)])
:ok
rescue
_e in ErlangError ->
{:error, "mogrify command not found"}
end
end
def filter(_), do: :ok

View file

@ -8,11 +8,15 @@ defmodule Pleroma.Upload.Filter.Mogrify do
@type conversion :: action :: String.t() | {action :: String.t(), opts :: String.t()}
@type conversions :: conversion() | [conversion()]
@spec filter(Pleroma.Upload.t()) :: :ok | {:error, String.t()}
def filter(%Pleroma.Upload{tempfile: file, content_type: "image" <> _}) do
filters = Pleroma.Config.get!([__MODULE__, :args])
do_filter(file, filters)
:ok
try do
do_filter(file, Pleroma.Config.get!([__MODULE__, :args]))
:ok
rescue
_e in ErlangError ->
{:error, "mogrify command not found"}
end
end
def filter(_), do: :ok

View file

@ -9,4 +9,19 @@ defmodule Pleroma.Utils do
|> Enum.map(&Path.join(dir, &1))
|> Kernel.ParallelCompiler.compile()
end
@doc """
POSIX-compliant check if command is available in the system
## Examples
iex> command_available?("git")
true
iex> command_available?("wrongcmd")
false
"""
@spec command_available?(String.t()) :: boolean()
def command_available?(command) do
match?({_output, 0}, System.cmd("sh", ["-c", "command -v #{command}"]))
end
end

View file

@ -21,8 +21,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.ActivityExpirationPolicy do
@impl true
def describe, do: {:ok, %{}}
defp local?(%{"id" => id}) do
String.starts_with?(id, Pleroma.Web.Endpoint.url())
defp local?(%{"actor" => actor}) do
String.starts_with?(actor, Pleroma.Web.Endpoint.url())
end
defp note?(activity) do

View file

@ -34,10 +34,15 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations do
cng
|> validate_change(field_name, fn field_name, actor ->
if User.get_cached_by_ap_id(actor) do
[]
else
[{field_name, "can't find user"}]
case User.get_cached_by_ap_id(actor) do
%User{deactivated: true} ->
[{field_name, "user is deactivated"}]
%User{} ->
[]
_ ->
[{field_name, "can't find user"}]
end
end)
end

View file

@ -25,7 +25,7 @@ defmodule Pleroma.Web.MastodonAPI.FilterView do
context: filter.context,
expires_at: expires_at,
irreversible: filter.hide,
whole_word: false
whole_word: filter.whole_word
}
end
end

View file

@ -9,6 +9,11 @@ defmodule Pleroma.Web.RichMedia.Helpers do
alias Pleroma.Object
alias Pleroma.Web.RichMedia.Parser
@rich_media_options [
pool: :media,
max_body: 2_000_000
]
@spec validate_page_url(URI.t() | binary()) :: :ok | :error
defp validate_page_url(page_url) when is_binary(page_url) do
validate_tld = Pleroma.Config.get([Pleroma.Formatter, :validate_tld])
@ -77,4 +82,20 @@ defmodule Pleroma.Web.RichMedia.Helpers do
fetch_data_for_activity(activity)
:ok
end
def rich_media_get(url) do
headers = [{"user-agent", Pleroma.Application.user_agent() <> "; Bot"}]
options =
if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Hackney do
Keyword.merge(@rich_media_options,
recv_timeout: 2_000,
with_body: true
)
else
@rich_media_options
end
Pleroma.HTTP.get(url, headers, options)
end
end

View file

@ -3,11 +3,6 @@
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.RichMedia.Parser do
@options [
pool: :media,
max_body: 2_000_000
]
defp parsers do
Pleroma.Config.get([:rich_media, :parsers])
end
@ -75,21 +70,8 @@ defmodule Pleroma.Web.RichMedia.Parser do
end
defp parse_url(url) do
opts =
if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Hackney do
Keyword.merge(@options,
recv_timeout: 2_000,
with_body: true
)
else
@options
end
try do
rich_media_agent = Pleroma.Application.user_agent() <> "; Bot"
{:ok, %Tesla.Env{body: html}} =
Pleroma.HTTP.get(url, [{"user-agent", rich_media_agent}], adapter: opts)
{:ok, %Tesla.Env{body: html}} = Pleroma.Web.RichMedia.Helpers.rich_media_get(url)
html
|> parse_html()

View file

@ -22,7 +22,7 @@ defmodule Pleroma.Web.RichMedia.Parsers.OEmbed do
end
defp get_oembed_data(url) do
with {:ok, %Tesla.Env{body: json}} <- Pleroma.HTTP.get(url, [], adapter: [pool: :media]) do
with {:ok, %Tesla.Env{body: json}} <- Pleroma.Web.RichMedia.Helpers.rich_media_get(url) do
Jason.decode(json)
end
end

View file

@ -37,7 +37,7 @@
}
a {
color: color: #d8a070;
color: #d8a070;
text-decoration: none;
}

View file

@ -214,7 +214,8 @@ defmodule Pleroma.Mixfile do
"ecto.setup": ["ecto.create", "ecto.migrate", "run priv/repo/seeds.exs"],
"ecto.reset": ["ecto.drop", "ecto.setup"],
test: ["ecto.create --quiet", "ecto.migrate", "test"],
docs: ["pleroma.docs", "docs"]
docs: ["pleroma.docs", "docs"],
analyze: ["credo --strict --only=warnings,todo,fixme,consistency,readability"]
]
end
@ -228,10 +229,10 @@ defmodule Pleroma.Mixfile do
defp version(version) do
identifier_filter = ~r/[^0-9a-z\-]+/i
{_cmdgit, cmdgit_err} = System.cmd("sh", ["-c", "command -v git"])
git_available? = match?({_output, 0}, System.cmd("sh", ["-c", "command -v git"]))
git_pre_release =
if cmdgit_err == 0 do
if git_available? do
{tag, tag_err} =
System.cmd("git", ["describe", "--tags", "--abbrev=0"], stderr_to_stdout: true)
@ -257,7 +258,7 @@ defmodule Pleroma.Mixfile do
# Branch name as pre-release version component, denoted with a dot
branch_name =
with 0 <- cmdgit_err,
with true <- git_available?,
{branch_name, 0} <- System.cmd("git", ["rev-parse", "--abbrev-ref", "HEAD"]),
branch_name <- String.trim(branch_name),
branch_name <- System.get_env("PLEROMA_BUILD_BRANCH") || branch_name,

View file

@ -0,0 +1,37 @@
# Fix legacy tags set by AdminFE that don't align with TagPolicy MRF
defmodule Pleroma.Repo.Migrations.FixLegacyTags do
use Ecto.Migration
alias Pleroma.Repo
alias Pleroma.User
import Ecto.Query
@old_new_map %{
"force_nsfw" => "mrf_tag:media-force-nsfw",
"strip_media" => "mrf_tag:media-strip",
"force_unlisted" => "mrf_tag:force-unlisted",
"sandbox" => "mrf_tag:sandbox",
"disable_remote_subscription" => "mrf_tag:disable-remote-subscription",
"disable_any_subscription" => "mrf_tag:disable-any-subscription"
}
def change do
legacy_tags = Map.keys(@old_new_map)
from(u in User, where: fragment("? && ?", u.tags, ^legacy_tags))
|> Repo.all()
|> Enum.each(fn user ->
fix_tags_changeset(user)
|> Repo.update()
end)
end
defp fix_tags_changeset(%User{tags: tags} = user) do
new_tags =
Enum.map(tags, fn tag ->
Map.get(@old_new_map, tag, tag)
end)
Ecto.Changeset.change(user, tags: new_tags)
end
end

View file

@ -0,0 +1,19 @@
defmodule Pleroma.Repo.Migrations.RemoveNonlocalExpirations do
use Ecto.Migration
def up do
statement = """
DELETE FROM
activity_expirations A USING activities B
WHERE
A.activity_id = B.id
AND B.local = false;
"""
execute(statement)
end
def down do
:ok
end
end

View file

@ -0,0 +1,7 @@
defmodule Pleroma.Repo.Migrations.AddUniqueIndexToAppClientId do
use Ecto.Migration
def change do
create(unique_index(:apps, [:client_id]))
end
end

View file

@ -28,6 +28,34 @@ defmodule Pleroma.ConfigTest do
assert Pleroma.Config.get([:azerty, :uiop], true) == true
end
describe "nil values" do
setup do
Pleroma.Config.put(:lorem, nil)
Pleroma.Config.put(:ipsum, %{dolor: [sit: nil]})
Pleroma.Config.put(:dolor, sit: %{amet: nil})
on_exit(fn -> Enum.each(~w(lorem ipsum dolor)a, &Pleroma.Config.delete/1) end)
end
test "get/1 with an atom for nil value" do
assert Pleroma.Config.get(:lorem) == nil
end
test "get/2 with an atom for nil value" do
assert Pleroma.Config.get(:lorem, true) == nil
end
test "get/1 with a list of keys for nil value" do
assert Pleroma.Config.get([:ipsum, :dolor, :sit]) == nil
assert Pleroma.Config.get([:dolor, :sit, :amet]) == nil
end
test "get/2 with a list of keys for nil value" do
assert Pleroma.Config.get([:ipsum, :dolor, :sit], true) == nil
assert Pleroma.Config.get([:dolor, :sit, :amet], true) == nil
end
end
test "get/1 when value is false" do
Pleroma.Config.put([:instance, :false_test], false)
Pleroma.Config.put([:instance, :nested], [])
@ -89,5 +117,23 @@ defmodule Pleroma.ConfigTest do
Pleroma.Config.put([:delete_me, :delete_me], hello: "world", world: "Hello")
Pleroma.Config.delete([:delete_me, :delete_me, :world])
assert Pleroma.Config.get([:delete_me, :delete_me]) == [hello: "world"]
assert Pleroma.Config.delete([:this_key_does_not_exist])
assert Pleroma.Config.delete([:non, :existing, :key])
end
test "fetch/1" do
Pleroma.Config.put([:lorem], :ipsum)
Pleroma.Config.put([:ipsum], dolor: :sit)
assert Pleroma.Config.fetch([:lorem]) == {:ok, :ipsum}
assert Pleroma.Config.fetch(:lorem) == {:ok, :ipsum}
assert Pleroma.Config.fetch([:ipsum, :dolor]) == {:ok, :sit}
assert Pleroma.Config.fetch([:lorem, :ipsum]) == :error
assert Pleroma.Config.fetch([:loremipsum]) == :error
assert Pleroma.Config.fetch(:loremipsum) == :error
Pleroma.Config.delete([:lorem])
Pleroma.Config.delete([:ipsum])
end
end

View file

@ -0,0 +1,24 @@
defmodule Pleroma.Repo.Migrations.FixLegacyTagsTest do
alias Pleroma.User
use Pleroma.DataCase
import Pleroma.Factory
import Pleroma.Tests.Helpers
setup_all do: require_migration("20200802170532_fix_legacy_tags")
test "change/0 converts legacy user tags into correct values", %{migration: migration} do
user = insert(:user, tags: ["force_nsfw", "force_unlisted", "verified"])
user2 = insert(:user)
assert :ok == migration.change()
fixed_user = User.get_by_id(user.id)
fixed_user2 = User.get_by_id(user2.id)
assert fixed_user.tags == ["mrf_tag:media-force-nsfw", "mrf_tag:force-unlisted", "verified"]
assert fixed_user2.tags == []
# user2 should not have been updated
assert fixed_user2.updated_at == fixed_user2.inserted_at
end
end

View file

@ -17,9 +17,19 @@ defmodule Pleroma.Tests.Helpers do
defmacro clear_config(config_path, do: yield) do
quote do
initial_setting = Config.get(unquote(config_path))
initial_setting = Config.fetch(unquote(config_path))
unquote(yield)
on_exit(fn -> Config.put(unquote(config_path), initial_setting) end)
on_exit(fn ->
case initial_setting do
:error ->
Config.delete(unquote(config_path))
{:ok, value} ->
Config.put(unquote(config_path), value)
end
end)
:ok
end
end

View file

@ -50,13 +50,13 @@ defmodule Mix.Tasks.Pleroma.AppTest do
defp assert_app(name, redirect, scopes) do
app = Repo.get_by(Pleroma.Web.OAuth.App, client_name: name)
assert_received {:mix_shell, :info, [message]}
assert_receive {:mix_shell, :info, [message]}
assert message == "#{name} successfully created:"
assert_received {:mix_shell, :info, [message]}
assert_receive {:mix_shell, :info, [message]}
assert message == "App client_id: #{app.client_id}"
assert_received {:mix_shell, :info, [message]}
assert_receive {:mix_shell, :info, [message]}
assert message == "App client_secret: #{app.client_secret}"
assert app.scopes == scopes

View file

@ -7,6 +7,8 @@ defmodule Pleroma.Upload.Filter.ExiftoolTest do
alias Pleroma.Upload.Filter
test "apply exiftool filter" do
assert Pleroma.Utils.command_available?("exiftool")
File.cp!(
"test/fixtures/DSCN0010.jpg",
"test/fixtures/DSCN0010_tmp.jpg"

View file

@ -7,11 +7,13 @@ defmodule Pleroma.Web.ActivityPub.MRF.ActivityExpirationPolicyTest do
alias Pleroma.Web.ActivityPub.MRF.ActivityExpirationPolicy
@id Pleroma.Web.Endpoint.url() <> "/activities/cofe"
@local_actor Pleroma.Web.Endpoint.url() <> "/users/cofe"
test "adds `expires_at` property" do
assert {:ok, %{"type" => "Create", "expires_at" => expires_at}} =
ActivityExpirationPolicy.filter(%{
"id" => @id,
"actor" => @local_actor,
"type" => "Create",
"object" => %{"type" => "Note"}
})
@ -25,6 +27,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.ActivityExpirationPolicyTest do
assert {:ok, %{"type" => "Create", "expires_at" => ^expires_at}} =
ActivityExpirationPolicy.filter(%{
"id" => @id,
"actor" => @local_actor,
"type" => "Create",
"expires_at" => expires_at,
"object" => %{"type" => "Note"}
@ -37,6 +40,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.ActivityExpirationPolicyTest do
assert {:ok, %{"type" => "Create", "expires_at" => expires_at}} =
ActivityExpirationPolicy.filter(%{
"id" => @id,
"actor" => @local_actor,
"type" => "Create",
"expires_at" => too_distant_future,
"object" => %{"type" => "Note"}
@ -49,6 +53,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.ActivityExpirationPolicyTest do
assert {:ok, activity} =
ActivityExpirationPolicy.filter(%{
"id" => "https://example.com/123",
"actor" => "https://example.com/users/cofe",
"type" => "Create",
"object" => %{"type" => "Note"}
})
@ -60,6 +65,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.ActivityExpirationPolicyTest do
assert {:ok, activity} =
ActivityExpirationPolicy.filter(%{
"id" => "https://example.com/123",
"actor" => "https://example.com/users/cofe",
"type" => "Follow"
})
@ -68,6 +74,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.ActivityExpirationPolicyTest do
assert {:ok, activity} =
ActivityExpirationPolicy.filter(%{
"id" => "https://example.com/123",
"actor" => "https://example.com/users/cofe",
"type" => "Create",
"object" => %{"type" => "Cofe"}
})

View file

@ -124,6 +124,24 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier.ChatMessageTest do
{:ok, %Activity{} = _activity} = Transmogrifier.handle_incoming(data)
end
test "it doesn't work for deactivated users" do
data =
File.read!("test/fixtures/create-chat-message.json")
|> Poison.decode!()
_author =
insert(:user,
ap_id: data["actor"],
local: false,
last_refreshed_at: DateTime.utc_now(),
deactivated: true
)
_recipient = insert(:user, ap_id: List.first(data["to"]), local: true)
assert {:error, _} = Transmogrifier.handle_incoming(data)
end
test "it inserts it and creates a chat" do
data =
File.read!("test/fixtures/create-chat-message.json")

View file

@ -163,6 +163,14 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do
end) =~ "[warn] Couldn't fetch \"https://404.site/whatever\", error: nil"
end
test "it does not work for deactivated users" do
data = File.read!("test/fixtures/mastodon-post-activity.json") |> Poison.decode!()
insert(:user, ap_id: data["actor"], deactivated: true)
assert {:error, _} = Transmogrifier.handle_incoming(data)
end
test "it works for incoming notices" do
data = File.read!("test/fixtures/mastodon-post-activity.json") |> Poison.decode!()

View file

@ -458,6 +458,11 @@ defmodule Pleroma.Web.CommonAPITest do
end
describe "posting" do
test "deactivated users can't post" do
user = insert(:user, deactivated: true)
assert {:error, _} = CommonAPI.post(user, %{status: "ye"})
end
test "it supports explicit addressing" do
user = insert(:user)
user_two = insert(:user)

View file

@ -64,11 +64,13 @@ defmodule Pleroma.Web.MastodonAPI.FilterControllerTest do
test "get a filter" do
%{user: user, conn: conn} = oauth_access(["read:filters"])
# check whole_word false
query = %Pleroma.Filter{
user_id: user.id,
filter_id: 2,
phrase: "knight",
context: ["home"]
context: ["home"],
whole_word: false
}
{:ok, filter} = Pleroma.Filter.create(query)
@ -76,6 +78,25 @@ defmodule Pleroma.Web.MastodonAPI.FilterControllerTest do
conn = get(conn, "/api/v1/filters/#{filter.filter_id}")
assert response = json_response_and_validate_schema(conn, 200)
assert response["whole_word"] == false
# check whole_word true
%{user: user, conn: conn} = oauth_access(["read:filters"])
query = %Pleroma.Filter{
user_id: user.id,
filter_id: 3,
phrase: "knight",
context: ["home"],
whole_word: true
}
{:ok, filter} = Pleroma.Filter.create(query)
conn = get(conn, "/api/v1/filters/#{filter.filter_id}")
assert response = json_response_and_validate_schema(conn, 200)
assert response["whole_word"] == true
end
test "update a filter" do
@ -86,7 +107,8 @@ defmodule Pleroma.Web.MastodonAPI.FilterControllerTest do
filter_id: 2,
phrase: "knight",
context: ["home"],
hide: true
hide: true,
whole_word: true
}
{:ok, _filter} = Pleroma.Filter.create(query)
@ -108,6 +130,7 @@ defmodule Pleroma.Web.MastodonAPI.FilterControllerTest do
assert response["phrase"] == new.phrase
assert response["context"] == new.context
assert response["irreversible"] == true
assert response["whole_word"] == true
end
test "delete a filter" do

View file

@ -17,8 +17,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPITest do
test "returns error when followed user is deactivated" do
follower = insert(:user)
user = insert(:user, local: true, deactivated: true)
{:error, error} = MastodonAPI.follow(follower, user)
assert error == :rejected
assert {:error, _error} = MastodonAPI.follow(follower, user)
end
test "following for user" do

View file

@ -29,5 +29,16 @@ defmodule Pleroma.Web.OAuth.AppTest do
assert exist_app.id == app.id
assert exist_app.scopes == ["read", "write", "follow", "push"]
end
test "has unique client_id" do
insert(:oauth_app, client_name: "", redirect_uris: "", client_id: "boop")
error =
catch_error(insert(:oauth_app, client_name: "", redirect_uris: "", client_id: "boop"))
assert %Ecto.ConstraintError{} = error
assert error.constraint == "apps_client_id_index"
assert error.type == :unique
end
end
end