diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..c5ef89b86 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,12 @@ +.* +*.md +AGPL-3 +CC-BY-SA-4.0 +COPYING +*file +elixir_buildpack.config +docs/ +test/ + +# Required to get version +!.git diff --git a/CHANGELOG.md b/CHANGELOG.md index e77fe4f3d..069974e44 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## [Unreleased] +### Security +- OStatus: eliminate the possibility of a protocol downgrade attack. +- OStatus: prevent following locked accounts, bypassing the approval process. + ### Changed - **Breaking:** Configuration: A setting to explicitly disable the mailer was added, defaulting to true, if you are using a mailer add `config :pleroma, Pleroma.Emails.Mailer, enabled: true` to your config - **Breaking:** Configuration: `/media/` is now removed when `base_url` is configured, append `/media/` to your `base_url` config to keep the old behaviour if desired @@ -17,6 +21,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Fixed - Not being able to pin unlisted posts +- Objects being re-embedded to activities after being updated (e.g faved/reposted). Running 'mix pleroma.database prune_objects' again is advised. - Metadata rendering errors resulting in the entire page being inaccessible - Federation/MediaProxy not working with instances that have wrong certificate order - Mastodon API: Handling of search timeouts (`/api/v1/search` and `/api/v2/search`) @@ -30,6 +35,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - ActivityPub S2S: sharedInbox usage has been mostly aligned with the rules in the AP specification. - ActivityPub S2S: remote user deletions now work the same as local user deletions. - Not being able to access the Mastodon FE login page on private instances +- Invalid SemVer version generation, when the current branch does not have commits ahead of tag/checked out on a tag +- Pleroma.Upload base_url was not automatically whitelisted by MediaProxy. Now your custom CDN or file hosting will be accessed directly as expected. +- Report email not being sent to admins when the reporter is a remote user ### Added - MRF: Support for priming the mediaproxy cache (`Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy`) @@ -53,6 +61,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Admin API: Return avatar and display name when querying users - Admin API: Allow querying user by ID - Admin API: Added support for `tuples`. +- Admin API: Added endpoints to run mix tasks pleroma.config migrate_to_db & pleroma.config migrate_from_db - Added synchronization of following/followers counters for external users - Configuration: `enabled` option for `Pleroma.Emails.Mailer`, defaulting to `false`. - Configuration: Pleroma.Plugs.RateLimiter `bucket_name`, `params` options. @@ -62,6 +71,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - ActivityPub: Optional signing of ActivityPub object fetches. - Admin API: Endpoint for fetching latest user's statuses - Pleroma API: Add `/api/v1/pleroma/accounts/confirmation_resend?email=` for resending account confirmation. +- Relays: Added a task to list relay subscriptions. ### Changed - Configuration: Filter.AnonymizeFilename added ability to retain file extension with custom text @@ -69,6 +79,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - RichMedia: parsers and their order are configured in `rich_media` config. - RichMedia: add the rich media ttl based on image expiration time. +### Removed +- Emoji: Remove longfox emojis. +- Remove `Reply-To` header from report emails for admins. + ## [1.0.1] - 2019-07-14 ### Security - OStatus: fix an object spoofing vulnerability. diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..268ec61dc --- /dev/null +++ b/Dockerfile @@ -0,0 +1,39 @@ +FROM rinpatch/elixir:1.9.0-rc.0-alpine as build + +COPY . . + +ENV MIX_ENV=prod + +RUN apk add git gcc g++ musl-dev make &&\ + echo "import Mix.Config" > config/prod.secret.exs &&\ + mix local.hex --force &&\ + mix local.rebar --force &&\ + mix deps.get --only prod &&\ + mkdir release &&\ + mix release --path release + +FROM alpine:latest + +ARG HOME=/opt/pleroma +ARG DATA=/var/lib/pleroma + +RUN echo "http://nl.alpinelinux.org/alpine/latest-stable/community" >> /etc/apk/repositories &&\ + apk update &&\ + apk add ncurses postgresql-client &&\ + adduser --system --shell /bin/false --home ${HOME} pleroma &&\ + mkdir -p ${DATA}/uploads &&\ + mkdir -p ${DATA}/static &&\ + chown -R pleroma ${DATA} &&\ + mkdir -p /etc/pleroma &&\ + chown -R pleroma /etc/pleroma + +USER pleroma + +COPY --from=build --chown=pleroma:0 /release ${HOME} + +COPY ./config/docker.exs /etc/pleroma/config.exs +COPY ./docker-entrypoint.sh ${HOME} + +EXPOSE 4000 + +ENTRYPOINT ["/opt/pleroma/docker-entrypoint.sh"] diff --git a/README.md b/README.md index 41d454a03..5aad34ccc 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ If you want to run your own server, feel free to contact us at @lain@pleroma.soy Currently Pleroma is not packaged by any OS/Distros, but feel free to reach out to us at [#pleroma-dev on freenode](https://webchat.freenode.net/?channels=%23pleroma-dev) or via matrix at for assistance. If you want to change default options in your Pleroma package, please **discuss it with us first**. ### Docker -While we don’t provide docker files, other people have written very good ones. Take a look at or . +While we don’t provide docker files, other people have written very good ones. Take a look at or . ### Dependencies diff --git a/config/docker.exs b/config/docker.exs new file mode 100644 index 000000000..63ab4cdee --- /dev/null +++ b/config/docker.exs @@ -0,0 +1,68 @@ +import Config + +config :pleroma, Pleroma.Web.Endpoint, + url: [host: System.get_env("DOMAIN", "localhost"), scheme: "https", port: 443], + http: [ip: {0, 0, 0, 0}, port: 4000] + +config :pleroma, :instance, + name: System.get_env("INSTANCE_NAME", "Pleroma"), + email: System.get_env("ADMIN_EMAIL"), + notify_email: System.get_env("NOTIFY_EMAIL"), + limit: 5000, + registrations_open: false, + dynamic_configuration: true + +config :pleroma, Pleroma.Repo, + adapter: Ecto.Adapters.Postgres, + username: System.get_env("DB_USER", "pleroma"), + password: System.fetch_env!("DB_PASS"), + database: System.get_env("DB_NAME", "pleroma"), + hostname: System.get_env("DB_HOST", "db"), + pool_size: 10 + +# Configure web push notifications +config :web_push_encryption, :vapid_details, subject: "mailto:#{System.get_env("NOTIFY_EMAIL")}" + +config :pleroma, :database, rum_enabled: false +config :pleroma, :instance, static_dir: "/var/lib/pleroma/static" +config :pleroma, Pleroma.Uploaders.Local, uploads: "/var/lib/pleroma/uploads" + +# We can't store the secrets in this file, since this is baked into the docker image +if not File.exists?("/var/lib/pleroma/secret.exs") do + secret = :crypto.strong_rand_bytes(64) |> Base.encode64() |> binary_part(0, 64) + signing_salt = :crypto.strong_rand_bytes(8) |> Base.encode64() |> binary_part(0, 8) + {web_push_public_key, web_push_private_key} = :crypto.generate_key(:ecdh, :prime256v1) + + secret_file = + EEx.eval_string( + """ + import Config + + config :pleroma, Pleroma.Web.Endpoint, + secret_key_base: "<%= secret %>", + signing_salt: "<%= signing_salt %>" + + config :web_push_encryption, :vapid_details, + public_key: "<%= web_push_public_key %>", + private_key: "<%= web_push_private_key %>" + """, + secret: secret, + signing_salt: signing_salt, + web_push_public_key: Base.url_encode64(web_push_public_key, padding: false), + web_push_private_key: Base.url_encode64(web_push_private_key, padding: false) + ) + + File.write("/var/lib/pleroma/secret.exs", secret_file) +end + +import_config("/var/lib/pleroma/secret.exs") + +# For additional user config +if File.exists?("/var/lib/pleroma/config.exs"), + do: import_config("/var/lib/pleroma/config.exs"), + else: + File.write("/var/lib/pleroma/config.exs", """ + import Config + + # For additional configuration outside of environmental variables + """) diff --git a/config/emoji.txt b/config/emoji.txt index 79246f239..200768ad1 100644 --- a/config/emoji.txt +++ b/config/emoji.txt @@ -1,30 +1,2 @@ firefox, /emoji/Firefox.gif, Gif,Fun blank, /emoji/blank.png, Fun -f_00b, /emoji/f_00b.png -f_00b11b, /emoji/f_00b11b.png -f_00b33b, /emoji/f_00b33b.png -f_00h, /emoji/f_00h.png -f_00t, /emoji/f_00t.png -f_01b, /emoji/f_01b.png -f_03b, /emoji/f_03b.png -f_10b, /emoji/f_10b.png -f_11b, /emoji/f_11b.png -f_11b00b, /emoji/f_11b00b.png -f_11b22b, /emoji/f_11b22b.png -f_11h, /emoji/f_11h.png -f_11t, /emoji/f_11t.png -f_12b, /emoji/f_12b.png -f_21b, /emoji/f_21b.png -f_22b, /emoji/f_22b.png -f_22b11b, /emoji/f_22b11b.png -f_22b33b, /emoji/f_22b33b.png -f_22h, /emoji/f_22h.png -f_22t, /emoji/f_22t.png -f_23b, /emoji/f_23b.png -f_30b, /emoji/f_30b.png -f_32b, /emoji/f_32b.png -f_33b, /emoji/f_33b.png -f_33b00b, /emoji/f_33b00b.png -f_33b22b, /emoji/f_33b22b.png -f_33h, /emoji/f_33h.png -f_33t, /emoji/f_33t.png diff --git a/config/test.exs b/config/test.exs index 92dca18bc..aded8600d 100644 --- a/config/test.exs +++ b/config/test.exs @@ -29,7 +29,8 @@ config :pleroma, :instance, email: "admin@example.com", notify_email: "noreply@example.com", skip_thread_containment: false, - federating: false + federating: false, + external_user_synchronization: false config :pleroma, :activitypub, sign_object_fetches: false diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh new file mode 100755 index 000000000..f56f8c50a --- /dev/null +++ b/docker-entrypoint.sh @@ -0,0 +1,14 @@ +#!/bin/ash + +set -e + +echo "-- Waiting for database..." +while ! pg_isready -U ${DB_USER:-pleroma} -d postgres://${DB_HOST:-db}:5432/${DB_NAME:-pleroma} -t 1; do + sleep 1s +done + +echo "-- Running migrations..." +$HOME/bin/pleroma_ctl migrate + +echo "-- Starting!" +exec $HOME/bin/pleroma start diff --git a/docs/api/admin_api.md b/docs/api/admin_api.md index ca9303227..7ccb90836 100644 --- a/docs/api/admin_api.md +++ b/docs/api/admin_api.md @@ -575,6 +575,29 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret - 404 Not Found `"Not found"` - On success: 200 OK `{}` + +## `/api/pleroma/admin/config/migrate_to_db` +### Run mix task pleroma.config migrate_to_db +Copy settings on key `:pleroma` to DB. +- Method `GET` +- Params: none +- Response: + +```json +{} +``` + +## `/api/pleroma/admin/config/migrate_from_db` +### Run mix task pleroma.config migrate_from_db +Copy all settings from DB to `config/prod.exported_from_db.secret.exs` with deletion from DB. +- Method `GET` +- Params: none +- Response: + +```json +{} +``` + ## `/api/pleroma/admin/config` ### List config settings List config settings only works with `:pleroma => :instance => :dynamic_configuration` setting to `true`. @@ -604,6 +627,9 @@ Tuples can be passed as `{"tuple": ["first_val", Pleroma.Module, []]}`. Keywords can be passed as lists with 2 child tuples, e.g. `[{"tuple": ["first_val", Pleroma.Module]}, {"tuple": ["second_val", true]}]`. +If value contains list of settings `[subkey: val1, subkey2: val2, subkey3: val3]`, it's possible to remove only subkeys instead of all settings passing `subkeys` parameter. E.g.: +{"group": "pleroma", "key": "some_key", "delete": "true", "subkeys": [":subkey", ":subkey3"]}. + Compile time settings (need instance reboot): - all settings by this keys: - `:hackney_pools` @@ -622,6 +648,7 @@ Compile time settings (need instance reboot): - `key` (string or string with leading `:` for atoms) - `value` (string, [], {} or {"tuple": []}) - `delete` = true (optional, if parameter must be deleted) + - `subkeys` [(string with leading `:` for atoms)] (optional, works only if `delete=true` parameter is passed, otherwise will be ignored) ] - Request (example): diff --git a/docs/config.md b/docs/config.md index 02f86dc16..50d6bfed4 100644 --- a/docs/config.md +++ b/docs/config.md @@ -25,7 +25,7 @@ At this time, write CNAME to CDN in public_endpoint. ## Pleroma.Upload.Filter.Mogrify -* `args`: List of actions for the `mogrify` command like `"strip"` or `["strip", "auto-orient", {"impode", "1"}]`. +* `args`: List of actions for the `mogrify` command like `"strip"` or `["strip", "auto-orient", {"implode", "1"}]`. ## Pleroma.Upload.Filter.Dedupe diff --git a/docs/config/howto_mediaproxy.md b/docs/config/howto_mediaproxy.md index ed70c3ed4..16c40c5db 100644 --- a/docs/config/howto_mediaproxy.md +++ b/docs/config/howto_mediaproxy.md @@ -1,8 +1,8 @@ # How to activate mediaproxy ## Explanation -Without the `mediaproxy` function, Pleroma don't store any remote content like pictures, video etc. locally. So every time you open Pleroma, the content is loaded from the source server, from where the post is coming. This can result in slowly loading content or/and increased bandwidth usage on the source server. -With the `mediaproxy` function you can use the cache ability of nginx, to cache these content, so user can access it faster, cause it's loaded from your server. +Without the `mediaproxy` function, Pleroma doesn't store any remote content like pictures, video etc. locally. So every time you open Pleroma, the content is loaded from the source server, from where the post is coming. This can result in slowly loading content or/and increased bandwidth usage on the source server. +With the `mediaproxy` function you can use nginx to cache this content, so users can access it faster, because it's loaded from your server. ## Activate it diff --git a/lib/mix/tasks/pleroma/config.ex b/lib/mix/tasks/pleroma/config.ex index a7d0fac5d..462940e7e 100644 --- a/lib/mix/tasks/pleroma/config.ex +++ b/lib/mix/tasks/pleroma/config.ex @@ -15,7 +15,7 @@ defmodule Mix.Tasks.Pleroma.Config do mix pleroma.config migrate_to_db - ## Transfers config from DB to file. + ## Transfers config from DB to file `config/env.exported_from_db.secret.exs` mix pleroma.config migrate_from_db ENV """ diff --git a/lib/mix/tasks/pleroma/relay.ex b/lib/mix/tasks/pleroma/relay.ex index 83ed0ed02..c7324fff6 100644 --- a/lib/mix/tasks/pleroma/relay.ex +++ b/lib/mix/tasks/pleroma/relay.ex @@ -5,6 +5,7 @@ defmodule Mix.Tasks.Pleroma.Relay do use Mix.Task import Mix.Pleroma + alias Pleroma.User alias Pleroma.Web.ActivityPub.Relay @shortdoc "Manages remote relays" @@ -22,6 +23,10 @@ defmodule Mix.Tasks.Pleroma.Relay do ``mix pleroma.relay unfollow `` Example: ``mix pleroma.relay unfollow https://example.org/relay`` + + ## List relay subscriptions + + ``mix pleroma.relay list`` """ def run(["follow", target]) do start_pleroma() @@ -44,4 +49,19 @@ defmodule Mix.Tasks.Pleroma.Relay do {:error, e} -> shell_error("Error while following #{target}: #{inspect(e)}") end end + + def run(["list"]) do + start_pleroma() + + with %User{} = user <- Relay.get_actor() do + user.following + |> Enum.each(fn entry -> + URI.parse(entry) + |> Map.get(:host) + |> shell_info() + end) + else + e -> shell_error("Error while fetching relay subscription list: #{inspect(e)}") + end + end end diff --git a/lib/mix/tasks/pleroma/user.ex b/lib/mix/tasks/pleroma/user.ex index c9b84b8f9..a3f8bc945 100644 --- a/lib/mix/tasks/pleroma/user.ex +++ b/lib/mix/tasks/pleroma/user.ex @@ -31,8 +31,8 @@ defmodule Mix.Tasks.Pleroma.User do mix pleroma.user invite [OPTION...] Options: - - `--expires_at DATE` - last day on which token is active (e.g. "2019-04-05") - - `--max_use NUMBER` - maximum numbers of token uses + - `--expires-at DATE` - last day on which token is active (e.g. "2019-04-05") + - `--max-use NUMBER` - maximum numbers of token uses ## List generated invites diff --git a/lib/pleroma/emails/admin_email.ex b/lib/pleroma/emails/admin_email.ex index d0e254362..c14be02dd 100644 --- a/lib/pleroma/emails/admin_email.ex +++ b/lib/pleroma/emails/admin_email.ex @@ -63,7 +63,6 @@ defmodule Pleroma.Emails.AdminEmail do new() |> to({to.name, to.email}) |> from({instance_name(), instance_notify_email()}) - |> reply_to({reporter.name, reporter.email}) |> subject("#{instance_name()} Report") |> html_body(html_body) end diff --git a/lib/pleroma/flake_id.ex b/lib/pleroma/flake_id.ex index 58ab3650d..ca0610abc 100644 --- a/lib/pleroma/flake_id.ex +++ b/lib/pleroma/flake_id.ex @@ -66,6 +66,16 @@ defmodule Pleroma.FlakeId do @spec get :: binary def get, do: to_string(:gen_server.call(:flake, :get)) + # checks that ID is is valid FlakeID + # + @spec is_flake_id?(String.t()) :: boolean + def is_flake_id?(id), do: is_flake_id?(String.to_charlist(id), true) + defp is_flake_id?([c | cs], true) when c >= ?0 and c <= ?9, do: is_flake_id?(cs, true) + defp is_flake_id?([c | cs], true) when c >= ?A and c <= ?Z, do: is_flake_id?(cs, true) + defp is_flake_id?([c | cs], true) when c >= ?a and c <= ?z, do: is_flake_id?(cs, true) + defp is_flake_id?([], true), do: true + defp is_flake_id?(_, _), do: false + # -- Ecto.Type API @impl Ecto.Type def type, do: :uuid diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex index 305ce8357..8d79ddb1f 100644 --- a/lib/pleroma/object/fetcher.ex +++ b/lib/pleroma/object/fetcher.ex @@ -114,7 +114,7 @@ defmodule Pleroma.Object.Fetcher do end end - def fetch_and_contain_remote_object_from_id(id) do + def fetch_and_contain_remote_object_from_id(id) when is_binary(id) do Logger.info("Fetching object #{id} via AP") date = @@ -141,4 +141,9 @@ defmodule Pleroma.Object.Fetcher do {:error, e} end end + + def fetch_and_contain_remote_object_from_id(%{"id" => id}), + do: fetch_and_contain_remote_object_from_id(id) + + def fetch_and_contain_remote_object_from_id(_id), do: {:error, "id must be a string"} end diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index a021e77f0..302adb1bc 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -114,7 +114,9 @@ defmodule Pleroma.User do def user_info(%User{} = user, args \\ %{}) do following_count = - if args[:following_count], do: args[:following_count], else: following_count(user) + if args[:following_count], + do: args[:following_count], + else: user.info.following_count || following_count(user) follower_count = if args[:follower_count], do: args[:follower_count], else: user.info.follower_count @@ -226,6 +228,7 @@ defmodule Pleroma.User do |> put_password_hash end + @spec reset_password(User.t(), map) :: {:ok, User.t()} | {:error, Ecto.Changeset.t()} def reset_password(%User{id: user_id} = user, data) do multi = Multi.new() @@ -330,6 +333,7 @@ defmodule Pleroma.User do def needs_update?(_), do: true + @spec maybe_direct_follow(User.t(), User.t()) :: {:ok, User.t()} | {:error, String.t()} def maybe_direct_follow(%User{} = follower, %User{local: true, info: %{locked: true}}) do {:ok, follower} end @@ -404,6 +408,8 @@ defmodule Pleroma.User do {1, [follower]} = Repo.update_all(q, []) + follower = maybe_update_following_count(follower) + {:ok, _} = update_follower_count(followed) set_cache(follower) @@ -423,6 +429,8 @@ defmodule Pleroma.User do {1, [follower]} = Repo.update_all(q, []) + follower = maybe_update_following_count(follower) + {:ok, followed} = update_follower_count(followed) set_cache(follower) @@ -714,32 +722,73 @@ defmodule Pleroma.User do |> update_and_set_cache() end - def update_follower_count(%User{} = user) do - follower_count_query = - User.Query.build(%{followers: user, deactivated: false}) - |> select([u], %{count: count(u.id)}) + def maybe_fetch_follow_information(user) do + with {:ok, user} <- fetch_follow_information(user) do + user + else + e -> + Logger.error("Follower/Following counter update for #{user.ap_id} failed.\n#{inspect(e)}") - User - |> where(id: ^user.id) - |> join(:inner, [u], s in subquery(follower_count_query)) - |> update([u, s], - set: [ - info: - fragment( - "jsonb_set(?, '{follower_count}', ?::varchar::jsonb, true)", - u.info, - s.count - ) - ] - ) - |> select([u], u) - |> Repo.update_all([]) - |> case do - {1, [user]} -> set_cache(user) - _ -> {:error, user} + user end end + def fetch_follow_information(user) do + with {:ok, info} <- ActivityPub.fetch_follow_information_for_user(user) do + info_cng = User.Info.follow_information_update(user.info, info) + + changeset = + user + |> change() + |> put_embed(:info, info_cng) + + update_and_set_cache(changeset) + else + {:error, _} = e -> e + e -> {:error, e} + end + end + + def update_follower_count(%User{} = user) do + if user.local or !Pleroma.Config.get([:instance, :external_user_synchronization]) do + follower_count_query = + User.Query.build(%{followers: user, deactivated: false}) + |> select([u], %{count: count(u.id)}) + + User + |> where(id: ^user.id) + |> join(:inner, [u], s in subquery(follower_count_query)) + |> update([u, s], + set: [ + info: + fragment( + "jsonb_set(?, '{follower_count}', ?::varchar::jsonb, true)", + u.info, + s.count + ) + ] + ) + |> select([u], u) + |> Repo.update_all([]) + |> case do + {1, [user]} -> set_cache(user) + _ -> {:error, user} + end + else + {:ok, maybe_fetch_follow_information(user)} + end + end + + def maybe_update_following_count(%User{local: false} = user) do + if Pleroma.Config.get([:instance, :external_user_synchronization]) do + {:ok, maybe_fetch_follow_information(user)} + else + user + end + end + + def maybe_update_following_count(user), do: user + def remove_duplicated_following(%User{following: following} = user) do uniq_following = Enum.uniq(following) diff --git a/lib/pleroma/user/info.ex b/lib/pleroma/user/info.ex index 9beb3ddbd..b03e705c3 100644 --- a/lib/pleroma/user/info.ex +++ b/lib/pleroma/user/info.ex @@ -16,6 +16,8 @@ defmodule Pleroma.User.Info do field(:source_data, :map, default: %{}) field(:note_count, :integer, default: 0) field(:follower_count, :integer, default: 0) + # Should be filled in only for remote users + field(:following_count, :integer, default: nil) field(:locked, :boolean, default: false) field(:confirmation_pending, :boolean, default: false) field(:confirmation_token, :string, default: nil) @@ -223,7 +225,11 @@ defmodule Pleroma.User.Info do :uri, :hub, :topic, - :salmon + :salmon, + :hide_followers, + :hide_follows, + :follower_count, + :following_count ]) end @@ -234,7 +240,11 @@ defmodule Pleroma.User.Info do :source_data, :banner, :locked, - :magic_key + :magic_key, + :follower_count, + :following_count, + :hide_follows, + :hide_followers ]) end @@ -348,4 +358,14 @@ defmodule Pleroma.User.Info do cast(info, params, [:muted_reblogs]) end + + def follow_information_update(info, params) do + info + |> cast(params, [ + :hide_followers, + :hide_follows, + :follower_count, + :following_count + ]) + end end diff --git a/lib/pleroma/user/search.ex b/lib/pleroma/user/search.ex index 46620b89a..6fb2c2352 100644 --- a/lib/pleroma/user/search.ex +++ b/lib/pleroma/user/search.ex @@ -44,7 +44,7 @@ defmodule Pleroma.User.Search do query_string = String.trim_leading(query_string, "@") with [name, domain] <- String.split(query_string, "@"), - formatted_domain <- String.replace(domain, ~r/[!-\-|@|[-`|{-~|\/|:]+/, "") do + formatted_domain <- String.replace(domain, ~r/[!-\-|@|[-`|{-~|\/|:|\s]+/, "") do name <> "@" <> to_string(:idna.encode(formatted_domain)) else _ -> query_string diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 6fd7fef92..1a279a7df 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -267,6 +267,9 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do else {:fake, true, activity} -> {:ok, activity} + + {:error, message} -> + {:error, message} end end @@ -746,8 +749,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do defp restrict_favorited_by(query, %{"favorited_by" => ap_id}) do from( - activity in query, - where: fragment(~s(? <@ (? #> '{"object","likes"}'\)), ^ap_id, activity.data) + [_activity, object] in query, + where: fragment("(?)->'likes' \\? (?)", object.data, ^ap_id) ) end @@ -1009,10 +1012,10 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do user_data = %{ ap_id: data["id"], info: %{ - "ap_enabled" => true, - "source_data" => data, - "banner" => banner, - "locked" => locked + ap_enabled: true, + source_data: data, + banner: banner, + locked: locked }, avatar: avatar, name: data["name"], @@ -1036,6 +1039,71 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do {:ok, user_data} end + def fetch_follow_information_for_user(user) do + with {:ok, following_data} <- + Fetcher.fetch_and_contain_remote_object_from_id(user.following_address), + following_count when is_integer(following_count) <- following_data["totalItems"], + {:ok, hide_follows} <- collection_private(following_data), + {:ok, followers_data} <- + Fetcher.fetch_and_contain_remote_object_from_id(user.follower_address), + followers_count when is_integer(followers_count) <- followers_data["totalItems"], + {:ok, hide_followers} <- collection_private(followers_data) do + {:ok, + %{ + hide_follows: hide_follows, + follower_count: followers_count, + following_count: following_count, + hide_followers: hide_followers + }} + else + {:error, _} = e -> + e + + e -> + {:error, e} + end + end + + defp maybe_update_follow_information(data) do + with {:enabled, true} <- + {:enabled, Pleroma.Config.get([:instance, :external_user_synchronization])}, + {:ok, info} <- fetch_follow_information_for_user(data) do + info = Map.merge(data.info, info) + Map.put(data, :info, info) + else + {:enabled, false} -> + data + + e -> + Logger.error( + "Follower/Following counter update for #{data.ap_id} failed.\n" <> inspect(e) + ) + + data + end + end + + defp collection_private(data) do + if is_map(data["first"]) and + data["first"]["type"] in ["CollectionPage", "OrderedCollectionPage"] do + {:ok, false} + else + with {:ok, %{"type" => type}} when type in ["CollectionPage", "OrderedCollectionPage"] <- + Fetcher.fetch_and_contain_remote_object_from_id(data["first"]) do + {:ok, false} + else + {:error, {:ok, %{status: code}}} when code in [401, 403] -> + {:ok, true} + + {:error, _} = e -> + e + + e -> + {:error, e} + end + end + end + def user_data_from_user_object(data) do with {:ok, data} <- MRF.filter(data), {:ok, data} <- object_to_user_data(data) do @@ -1047,7 +1115,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do def fetch_and_prepare_user_from_ap_id(ap_id) do with {:ok, data} <- Fetcher.fetch_and_contain_remote_object_from_id(ap_id), - {:ok, data} <- user_data_from_user_object(data) do + {:ok, data} <- user_data_from_user_object(data), + data <- maybe_update_follow_information(data) do {:ok, data} else e -> Logger.error("Could not decode user at fetch #{ap_id}, #{inspect(e)}") diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 44bb1cb9a..5403b71d8 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -608,13 +608,13 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do with %User{ap_id: ^actor_id} = actor <- User.get_cached_by_ap_id(object["id"]) do {:ok, new_user_data} = ActivityPub.user_data_from_user_object(object) - banner = new_user_data[:info]["banner"] - locked = new_user_data[:info]["locked"] || false + banner = new_user_data[:info][:banner] + locked = new_user_data[:info][:locked] || false update_data = new_user_data |> Map.take([:name, :bio, :avatar]) - |> Map.put(:info, %{"banner" => banner, "locked" => locked}) + |> Map.put(:info, %{banner: banner, locked: locked}) actor |> User.upgrade_changeset(update_data) @@ -1076,10 +1076,6 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do PleromaJobQueue.enqueue(:transmogrifier, __MODULE__, [:user_upgrade, user]) end - if Pleroma.Config.get([:instance, :external_user_synchronization]) do - update_following_followers_counters(user) - end - {:ok, user} else %User{} = user -> {:ok, user} @@ -1112,27 +1108,4 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do data |> maybe_fix_user_url end - - def update_following_followers_counters(user) do - info = %{} - - following = fetch_counter(user.following_address) - info = if following, do: Map.put(info, :following_count, following), else: info - - followers = fetch_counter(user.follower_address) - info = if followers, do: Map.put(info, :follower_count, followers), else: info - - User.set_info_cache(user, info) - end - - defp fetch_counter(url) do - with {:ok, %{body: body, status: code}} when code in 200..299 <- - Pleroma.HTTP.get( - url, - [{:Accept, "application/activity+json"}] - ), - {:ok, data} <- Jason.decode(body) do - data["totalItems"] - end - end end diff --git a/lib/pleroma/web/activity_pub/utils.ex b/lib/pleroma/web/activity_pub/utils.ex index 39074888b..fc5305c58 100644 --- a/lib/pleroma/web/activity_pub/utils.ex +++ b/lib/pleroma/web/activity_pub/utils.ex @@ -251,20 +251,6 @@ defmodule Pleroma.Web.ActivityPub.Utils do def insert_full_object(map), do: {:ok, map, nil} - def update_object_in_activities(%{data: %{"id" => id}} = object) do - # TODO - # Update activities that already had this. Could be done in a seperate process. - # Alternatively, just don't do this and fetch the current object each time. Most - # could probably be taken from cache. - relevant_activities = Activity.get_all_create_by_object_ap_id(id) - - Enum.map(relevant_activities, fn activity -> - new_activity_data = activity.data |> Map.put("object", object.data) - changeset = Changeset.change(activity, data: new_activity_data) - Repo.update(changeset) - end) - end - #### Like-related helpers @doc """ @@ -347,8 +333,7 @@ defmodule Pleroma.Web.ActivityPub.Utils do |> Map.put("#{property}_count", length(element)) |> Map.put("#{property}s", element), changeset <- Changeset.change(object, data: new_data), - {:ok, object} <- Object.update_and_set_cache(changeset), - _ <- update_object_in_activities(object) do + {:ok, object} <- Object.update_and_set_cache(changeset) do {:ok, object} end end diff --git a/lib/pleroma/web/activity_pub/views/object_view.ex b/lib/pleroma/web/activity_pub/views/object_view.ex index 6028b773c..94d05f49b 100644 --- a/lib/pleroma/web/activity_pub/views/object_view.ex +++ b/lib/pleroma/web/activity_pub/views/object_view.ex @@ -66,8 +66,10 @@ defmodule Pleroma.Web.ActivityPub.ObjectView do "orderedItems" => items } - if offset < total do + if offset + length(items) < total do Map.put(map, "next", "#{iri}?page=#{page + 1}") + else + map end end end diff --git a/lib/pleroma/web/activity_pub/views/user_view.ex b/lib/pleroma/web/activity_pub/views/user_view.ex index 639519e0a..06c9e1c71 100644 --- a/lib/pleroma/web/activity_pub/views/user_view.ex +++ b/lib/pleroma/web/activity_pub/views/user_view.ex @@ -65,7 +65,7 @@ defmodule Pleroma.Web.ActivityPub.UserView do do: render("service.json", %{user: user}) def render("user.json", %{user: %User{nickname: "internal." <> _} = user}), - do: render("service.json", %{user: user}) + do: render("service.json", %{user: user}) |> Map.put("preferredUsername", user.nickname) def render("user.json", %{user: user}) do {:ok, user} = User.ensure_keys_present(user) diff --git a/lib/pleroma/web/admin_api/admin_api_controller.ex b/lib/pleroma/web/admin_api/admin_api_controller.ex index 1ae5acd91..2d3d0adc4 100644 --- a/lib/pleroma/web/admin_api/admin_api_controller.ex +++ b/lib/pleroma/web/admin_api/admin_api_controller.ex @@ -379,6 +379,16 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do end end + def migrate_to_db(conn, _params) do + Mix.Tasks.Pleroma.Config.run(["migrate_to_db"]) + json(conn, %{}) + end + + def migrate_from_db(conn, _params) do + Mix.Tasks.Pleroma.Config.run(["migrate_from_db", Pleroma.Config.get(:env), "true"]) + json(conn, %{}) + end + def config_show(conn, _params) do configs = Pleroma.Repo.all(Config) @@ -392,9 +402,9 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do if Pleroma.Config.get([:instance, :dynamic_configuration]) do updated = Enum.map(configs, fn - %{"group" => group, "key" => key, "delete" => "true"} -> - {:ok, _} = Config.delete(%{group: group, key: key}) - nil + %{"group" => group, "key" => key, "delete" => "true"} = params -> + {:ok, config} = Config.delete(%{group: group, key: key, subkeys: params["subkeys"]}) + config %{"group" => group, "key" => key, "value" => value} -> {:ok, config} = Config.update_or_create(%{group: group, key: key, value: value}) diff --git a/lib/pleroma/web/admin_api/config.ex b/lib/pleroma/web/admin_api/config.ex index dde05ea7b..a10cc779b 100644 --- a/lib/pleroma/web/admin_api/config.ex +++ b/lib/pleroma/web/admin_api/config.ex @@ -55,8 +55,19 @@ defmodule Pleroma.Web.AdminAPI.Config do @spec delete(map()) :: {:ok, Config.t()} | {:error, Changeset.t()} def delete(params) do - with %Config{} = config <- Config.get_by_params(params) do - Repo.delete(config) + with %Config{} = config <- Config.get_by_params(Map.delete(params, :subkeys)) do + if params[:subkeys] do + updated_value = + Keyword.drop( + :erlang.binary_to_term(config.value), + Enum.map(params[:subkeys], &do_transform_string(&1)) + ) + + Config.update(config, %{value: updated_value}) + else + Repo.delete(config) + {:ok, nil} + end else nil -> err = diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex index 425b6d656..61b96aba9 100644 --- a/lib/pleroma/web/common_api/utils.ex +++ b/lib/pleroma/web/common_api/utils.ex @@ -25,7 +25,12 @@ defmodule Pleroma.Web.CommonAPI.Utils do # This is a hack for twidere. def get_by_id_or_ap_id(id) do activity = - Activity.get_by_id_with_object(id) || Activity.get_create_by_object_ap_id_with_object(id) + with true <- Pleroma.FlakeId.is_flake_id?(id), + %Activity{} = activity <- Activity.get_by_id_with_object(id) do + activity + else + _ -> Activity.get_create_by_object_ap_id_with_object(id) + end activity && if activity.data["type"] == "Create" do @@ -43,26 +48,43 @@ defmodule Pleroma.Web.CommonAPI.Utils do def get_replied_to_activity(_), do: nil - def attachments_from_ids(data) do - if Map.has_key?(data, "descriptions") do - attachments_from_ids_descs(data["media_ids"], data["descriptions"]) - else - attachments_from_ids_no_descs(data["media_ids"]) - end + def attachments_from_ids(%{"media_ids" => ids, "descriptions" => desc} = _) do + attachments_from_ids_descs(ids, desc) end - def attachments_from_ids_no_descs(ids) do - Enum.map(ids || [], fn media_id -> - Repo.get(Object, media_id).data - end) + def attachments_from_ids(%{"media_ids" => ids} = _) do + attachments_from_ids_no_descs(ids) end + def attachments_from_ids(_), do: [] + + def attachments_from_ids_no_descs([]), do: [] + + def attachments_from_ids_no_descs(ids) do + Enum.map(ids, fn media_id -> + case Repo.get(Object, media_id) do + %Object{data: data} = _ -> data + _ -> nil + end + end) + |> Enum.filter(& &1) + end + + def attachments_from_ids_descs([], _), do: [] + def attachments_from_ids_descs(ids, descs_str) do {_, descs} = Jason.decode(descs_str) - Enum.map(ids || [], fn media_id -> - Map.put(Repo.get(Object, media_id).data, "name", descs[media_id]) + Enum.map(ids, fn media_id -> + case Repo.get(Object, media_id) do + %Object{data: data} = _ -> + Map.put(data, "name", descs[media_id]) + + _ -> + nil + end end) + |> Enum.filter(& &1) end @spec get_to_and_cc( @@ -259,20 +281,18 @@ defmodule Pleroma.Web.CommonAPI.Utils do end def add_attachments(text, attachments) do - attachment_text = - Enum.map(attachments, fn - %{"url" => [%{"href" => href} | _]} = attachment -> - name = attachment["name"] || URI.decode(Path.basename(href)) - href = MediaProxy.url(href) - "#{shortname(name)}" - - _ -> - "" - end) - + attachment_text = Enum.map(attachments, &build_attachment_link/1) Enum.join([text | attachment_text], "
") end + defp build_attachment_link(%{"url" => [%{"href" => href} | _]} = attachment) do + name = attachment["name"] || URI.decode(Path.basename(href)) + href = MediaProxy.url(href) + "#{shortname(name)}" + end + + defp build_attachment_link(_), do: "" + def format_input(text, format, options \\ []) @doc """ @@ -332,7 +352,7 @@ defmodule Pleroma.Web.CommonAPI.Utils do sensitive \\ false, merge \\ %{} ) do - object = %{ + %{ "type" => "Note", "to" => to, "cc" => cc, @@ -342,18 +362,20 @@ defmodule Pleroma.Web.CommonAPI.Utils do "context" => context, "attachment" => attachments, "actor" => actor, - "tag" => tags |> Enum.map(fn {_, tag} -> tag end) |> Enum.uniq() + "tag" => Keyword.values(tags) |> Enum.uniq() } + |> add_in_reply_to(in_reply_to) + |> Map.merge(merge) + end - object = - with false <- is_nil(in_reply_to), - %Object{} = in_reply_to_object <- Object.normalize(in_reply_to) do - Map.put(object, "inReplyTo", in_reply_to_object.data["id"]) - else - _ -> object - end + defp add_in_reply_to(object, nil), do: object - Map.merge(object, merge) + defp add_in_reply_to(object, in_reply_to) do + with %Object{} = in_reply_to_object <- Object.normalize(in_reply_to) do + Map.put(object, "inReplyTo", in_reply_to_object.data["id"]) + else + _ -> object + end end def format_naive_asctime(date) do @@ -385,17 +407,16 @@ defmodule Pleroma.Web.CommonAPI.Utils do |> String.replace(~r/(\.\d+)?$/, ".000Z", global: false) end - def to_masto_date(date) do - try do - date - |> NaiveDateTime.from_iso8601!() - |> NaiveDateTime.to_iso8601() - |> String.replace(~r/(\.\d+)?$/, ".000Z", global: false) - rescue - _e -> "" + def to_masto_date(date) when is_binary(date) do + with {:ok, date} <- NaiveDateTime.from_iso8601(date) do + to_masto_date(date) + else + _ -> "" end end + def to_masto_date(_), do: "" + defp shortname(name) do if String.length(name) < 30 do name @@ -440,7 +461,7 @@ defmodule Pleroma.Web.CommonAPI.Utils do object_data = cond do - !is_nil(object) -> + not is_nil(object) -> object.data is_map(data["object"]) -> @@ -484,9 +505,9 @@ defmodule Pleroma.Web.CommonAPI.Utils do def maybe_extract_mentions(%{"tag" => tag}) do tag - |> Enum.filter(fn x -> is_map(x) end) - |> Enum.filter(fn x -> x["type"] == "Mention" end) + |> Enum.filter(fn x -> is_map(x) && x["type"] == "Mention" end) |> Enum.map(fn x -> x["href"] end) + |> Enum.uniq() end def maybe_extract_mentions(_), do: [] diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex index b2b06eeb9..de084fd6e 100644 --- a/lib/pleroma/web/mastodon_api/views/account_view.ex +++ b/lib/pleroma/web/mastodon_api/views/account_view.ex @@ -28,7 +28,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do id: to_string(user.id), acct: user.nickname, username: username_from_nickname(user.nickname), - url: user.ap_id + url: User.profile_url(user) } end @@ -106,7 +106,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do following_count: user_info.following_count, statuses_count: user_info.note_count, note: bio || "", - url: user.ap_id, + url: User.profile_url(user), avatar: image, avatar_static: image, header: header, diff --git a/lib/pleroma/web/media_proxy/media_proxy.ex b/lib/pleroma/web/media_proxy/media_proxy.ex index a661e9bb7..1725ab071 100644 --- a/lib/pleroma/web/media_proxy/media_proxy.ex +++ b/lib/pleroma/web/media_proxy/media_proxy.ex @@ -4,6 +4,7 @@ defmodule Pleroma.Web.MediaProxy do alias Pleroma.Config + alias Pleroma.Upload alias Pleroma.Web @base64_opts [padding: false] @@ -26,7 +27,18 @@ defmodule Pleroma.Web.MediaProxy do defp whitelisted?(url) do %{host: domain} = URI.parse(url) - Enum.any?(Config.get([:media_proxy, :whitelist]), fn pattern -> + mediaproxy_whitelist = Config.get([:media_proxy, :whitelist]) + + upload_base_url_domain = + if !is_nil(Config.get([Upload, :base_url])) do + [URI.parse(Config.get([Upload, :base_url])).host] + else + [] + end + + whitelist = mediaproxy_whitelist ++ upload_base_url_domain + + Enum.any?(whitelist, fn pattern -> String.equivalent?(domain, pattern) end) end diff --git a/lib/pleroma/web/ostatus/activity_representer.ex b/lib/pleroma/web/ostatus/activity_representer.ex index 760345301..8e55b9f0b 100644 --- a/lib/pleroma/web/ostatus/activity_representer.ex +++ b/lib/pleroma/web/ostatus/activity_representer.ex @@ -183,6 +183,7 @@ defmodule Pleroma.Web.OStatus.ActivityRepresenter do author = if with_author, do: [{:author, UserRepresenter.to_simple_form(user)}], else: [] retweeted_activity = Activity.get_create_by_object_ap_id(activity.data["object"]) + retweeted_object = Object.normalize(retweeted_activity) retweeted_user = User.get_cached_by_ap_id(retweeted_activity.data["actor"]) retweeted_xml = to_simple_form(retweeted_activity, retweeted_user, true) @@ -197,7 +198,7 @@ defmodule Pleroma.Web.OStatus.ActivityRepresenter do {:"activity:verb", ['http://activitystrea.ms/schema/1.0/share']}, {:id, h.(activity.data["id"])}, {:title, ['#{user.nickname} repeated a notice']}, - {:content, [type: 'html'], ['RT #{retweeted_activity.data["object"]["content"]}']}, + {:content, [type: 'html'], ['RT #{retweeted_object.data["content"]}']}, {:published, h.(inserted_at)}, {:updated, h.(updated_at)}, {:"ostatus:conversation", [ref: h.(activity.data["context"])], diff --git a/lib/pleroma/web/ostatus/handlers/follow_handler.ex b/lib/pleroma/web/ostatus/handlers/follow_handler.ex index 263d3b2dc..24513972e 100644 --- a/lib/pleroma/web/ostatus/handlers/follow_handler.ex +++ b/lib/pleroma/web/ostatus/handlers/follow_handler.ex @@ -9,14 +9,18 @@ defmodule Pleroma.Web.OStatus.FollowHandler do alias Pleroma.Web.XML def handle(entry, doc) do - with {:ok, actor} <- OStatus.find_make_or_update_user(doc), + with {:ok, actor} <- OStatus.find_make_or_update_actor(doc), id when not is_nil(id) <- XML.string_from_xpath("/entry/id", entry), followed_uri when not is_nil(followed_uri) <- XML.string_from_xpath("/entry/activity:object/id", entry), {:ok, followed} <- OStatus.find_or_make_user(followed_uri), + {:locked, false} <- {:locked, followed.info.locked}, {:ok, activity} <- ActivityPub.follow(actor, followed, id, false) do User.follow(actor, followed) {:ok, activity} + else + {:locked, true} -> + {:error, "It's not possible to follow locked accounts over OStatus"} end end end diff --git a/lib/pleroma/web/ostatus/handlers/note_handler.ex b/lib/pleroma/web/ostatus/handlers/note_handler.ex index 3005e8f57..7fae14f7b 100644 --- a/lib/pleroma/web/ostatus/handlers/note_handler.ex +++ b/lib/pleroma/web/ostatus/handlers/note_handler.ex @@ -111,7 +111,7 @@ defmodule Pleroma.Web.OStatus.NoteHandler do with id <- XML.string_from_xpath("//id", entry), activity when is_nil(activity) <- Activity.get_create_by_object_ap_id_with_object(id), [author] <- :xmerl_xpath.string('//author[1]', doc), - {:ok, actor} <- OStatus.find_make_or_update_user(author), + {:ok, actor} <- OStatus.find_make_or_update_actor(author), content_html <- OStatus.get_content(entry), cw <- OStatus.get_cw(entry), in_reply_to <- XML.string_from_xpath("//thr:in-reply-to[1]/@ref", entry), diff --git a/lib/pleroma/web/ostatus/handlers/unfollow_handler.ex b/lib/pleroma/web/ostatus/handlers/unfollow_handler.ex index 6596ada3b..2062432e3 100644 --- a/lib/pleroma/web/ostatus/handlers/unfollow_handler.ex +++ b/lib/pleroma/web/ostatus/handlers/unfollow_handler.ex @@ -9,7 +9,7 @@ defmodule Pleroma.Web.OStatus.UnfollowHandler do alias Pleroma.Web.XML def handle(entry, doc) do - with {:ok, actor} <- OStatus.find_make_or_update_user(doc), + with {:ok, actor} <- OStatus.find_make_or_update_actor(doc), id when not is_nil(id) <- XML.string_from_xpath("/entry/id", entry), followed_uri when not is_nil(followed_uri) <- XML.string_from_xpath("/entry/activity:object/id", entry), diff --git a/lib/pleroma/web/ostatus/ostatus.ex b/lib/pleroma/web/ostatus/ostatus.ex index 502410c83..331cbc0b7 100644 --- a/lib/pleroma/web/ostatus/ostatus.ex +++ b/lib/pleroma/web/ostatus/ostatus.ex @@ -56,7 +56,7 @@ defmodule Pleroma.Web.OStatus do def handle_incoming(xml_string, options \\ []) do with doc when doc != :error <- parse_document(xml_string) do - with {:ok, actor_user} <- find_make_or_update_user(doc), + with {:ok, actor_user} <- find_make_or_update_actor(doc), do: Pleroma.Instances.set_reachable(actor_user.ap_id) entries = :xmerl_xpath.string('//entry', doc) @@ -120,7 +120,7 @@ defmodule Pleroma.Web.OStatus do end def make_share(entry, doc, retweeted_activity) do - with {:ok, actor} <- find_make_or_update_user(doc), + with {:ok, actor} <- find_make_or_update_actor(doc), %Object{} = object <- Object.normalize(retweeted_activity), id when not is_nil(id) <- string_from_xpath("/entry/id", entry), {:ok, activity, _object} = ActivityPub.announce(actor, object, id, false) do @@ -138,7 +138,7 @@ defmodule Pleroma.Web.OStatus do end def make_favorite(entry, doc, favorited_activity) do - with {:ok, actor} <- find_make_or_update_user(doc), + with {:ok, actor} <- find_make_or_update_actor(doc), %Object{} = object <- Object.normalize(favorited_activity), id when not is_nil(id) <- string_from_xpath("/entry/id", entry), {:ok, activity, _object} = ActivityPub.like(actor, object, id, false) do @@ -264,11 +264,18 @@ defmodule Pleroma.Web.OStatus do end end - def find_make_or_update_user(doc) do + def find_make_or_update_actor(doc) do uri = string_from_xpath("//author/uri[1]", doc) - with {:ok, user} <- find_or_make_user(uri) do + with {:ok, %User{} = user} <- find_or_make_user(uri), + {:ap_enabled, false} <- {:ap_enabled, User.ap_enabled?(user)} do maybe_update(doc, user) + else + {:ap_enabled, true} -> + {:error, :invalid_protocol} + + _ -> + {:error, :unknown_user} end end diff --git a/lib/pleroma/web/rich_media/parsers/twitter_card.ex b/lib/pleroma/web/rich_media/parsers/twitter_card.ex index e4efe2dd0..afaa98f3d 100644 --- a/lib/pleroma/web/rich_media/parsers/twitter_card.ex +++ b/lib/pleroma/web/rich_media/parsers/twitter_card.ex @@ -3,13 +3,20 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.RichMedia.Parsers.TwitterCard do + alias Pleroma.Web.RichMedia.Parsers.MetaTagsParser + + @spec parse(String.t(), map()) :: {:ok, map()} | {:error, String.t()} def parse(html, data) do - Pleroma.Web.RichMedia.Parsers.MetaTagsParser.parse( - html, - data, - "twitter", - "No twitter card metadata found", - "name" - ) + data + |> parse_name_attrs(html) + |> parse_property_attrs(html) + end + + defp parse_name_attrs(data, html) do + MetaTagsParser.parse(html, data, "twitter", %{}, "name") + end + + defp parse_property_attrs({_, data}, html) do + MetaTagsParser.parse(html, data, "twitter", "No twitter card metadata found", "property") end end diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 6cdef7e2f..c835f06b4 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -196,6 +196,8 @@ defmodule Pleroma.Web.Router do get("/config", AdminAPIController, :config_show) post("/config", AdminAPIController, :config_update) + get("/config/migrate_to_db", AdminAPIController, :migrate_to_db) + get("/config/migrate_from_db", AdminAPIController, :migrate_from_db) end scope "/", Pleroma.Web.TwitterAPI do diff --git a/lib/pleroma/web/templates/layout/app.html.eex b/lib/pleroma/web/templates/layout/app.html.eex index b3cf9ed11..5836ec1e0 100644 --- a/lib/pleroma/web/templates/layout/app.html.eex +++ b/lib/pleroma/web/templates/layout/app.html.eex @@ -36,6 +36,11 @@ margin-bottom: 20px; } + a { + color: color: #d8a070; + text-decoration: none; + } + form { width: 100%; } diff --git a/lib/pleroma/web/twitter_api/controllers/util_controller.ex b/lib/pleroma/web/twitter_api/controllers/util_controller.ex index 5c73a615d..3405bd3b7 100644 --- a/lib/pleroma/web/twitter_api/controllers/util_controller.ex +++ b/lib/pleroma/web/twitter_api/controllers/util_controller.ex @@ -18,6 +18,8 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do alias Pleroma.Web.CommonAPI alias Pleroma.Web.WebFinger + plug(Pleroma.Plugs.SetFormatPlug when action in [:config, :version]) + def help_test(conn, _params) do json(conn, "ok") end @@ -58,27 +60,25 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do %Activity{id: activity_id} = Activity.get_create_by_object_ap_id(object.data["id"]) redirect(conn, to: "/notice/#{activity_id}") else - {err, followee} = User.get_or_fetch(acct) - avatar = User.avatar_url(followee) - name = followee.nickname - id = followee.id - - if !!user do + with {:ok, followee} <- User.get_or_fetch(acct) do conn - |> render("follow.html", %{error: err, acct: acct, avatar: avatar, name: name, id: id}) - else - conn - |> render("follow_login.html", %{ + |> render(follow_template(user), %{ error: false, acct: acct, - avatar: avatar, - name: name, - id: id + avatar: User.avatar_url(followee), + name: followee.nickname, + id: followee.id }) + else + {:error, _reason} -> + render(conn, follow_template(user), %{error: :error}) end end end + defp follow_template(%User{} = _user), do: "follow.html" + defp follow_template(_), do: "follow_login.html" + defp is_status?(acct) do case Pleroma.Object.Fetcher.fetch_and_contain_remote_object_from_id(acct) do {:ok, %{"type" => type}} when type in ["Article", "Note", "Video", "Page", "Question"] -> @@ -92,48 +92,53 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do def do_remote_follow(conn, %{ "authorization" => %{"name" => username, "password" => password, "id" => id} }) do - followee = User.get_cached_by_id(id) - avatar = User.avatar_url(followee) - name = followee.nickname - - with %User{} = user <- User.get_cached_by_nickname(username), - true <- AuthenticationPlug.checkpw(password, user.password_hash), - %User{} = _followed <- User.get_cached_by_id(id), + with %User{} = followee <- User.get_cached_by_id(id), + {_, %User{} = user, _} <- {:auth, User.get_cached_by_nickname(username), followee}, + {_, true, _} <- { + :auth, + AuthenticationPlug.checkpw(password, user.password_hash), + followee + }, {:ok, _follower, _followee, _activity} <- CommonAPI.follow(user, followee) do conn |> render("followed.html", %{error: false}) else # Was already following user {:error, "Could not follow user:" <> _rest} -> - render(conn, "followed.html", %{error: false}) + render(conn, "followed.html", %{error: "Error following account"}) - _e -> + {:auth, _, followee} -> conn |> render("follow_login.html", %{ error: "Wrong username or password", id: id, - name: name, - avatar: avatar + name: followee.nickname, + avatar: User.avatar_url(followee) }) + + e -> + Logger.debug("Remote follow failed with error #{inspect(e)}") + render(conn, "followed.html", %{error: "Something went wrong."}) end end def do_remote_follow(%{assigns: %{user: user}} = conn, %{"user" => %{"id" => id}}) do - with %User{} = followee <- User.get_cached_by_id(id), + with {:fetch_user, %User{} = followee} <- {:fetch_user, User.get_cached_by_id(id)}, {:ok, _follower, _followee, _activity} <- CommonAPI.follow(user, followee) do conn |> render("followed.html", %{error: false}) else # Was already following user {:error, "Could not follow user:" <> _rest} -> - conn - |> render("followed.html", %{error: false}) + render(conn, "followed.html", %{error: "Error following account"}) + + {:fetch_user, error} -> + Logger.debug("Remote follow failed with error #{inspect(error)}") + render(conn, "followed.html", %{error: "Could not find user"}) e -> Logger.debug("Remote follow failed with error #{inspect(e)}") - - conn - |> render("followed.html", %{error: inspect(e)}) + render(conn, "followed.html", %{error: "Something went wrong."}) end end @@ -148,67 +153,70 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do end end + def config(%{assigns: %{format: "xml"}} = conn, _params) do + instance = Pleroma.Config.get(:instance) + + response = """ + + + #{Keyword.get(instance, :name)} + #{Web.base_url()} + #{Keyword.get(instance, :limit)} + #{!Keyword.get(instance, :registrations_open)} + + + """ + + conn + |> put_resp_content_type("application/xml") + |> send_resp(200, response) + end + def config(conn, _params) do instance = Pleroma.Config.get(:instance) - case get_format(conn) do - "xml" -> - response = """ - - - #{Keyword.get(instance, :name)} - #{Web.base_url()} - #{Keyword.get(instance, :limit)} - #{!Keyword.get(instance, :registrations_open)} - - - """ + vapid_public_key = Keyword.get(Pleroma.Web.Push.vapid_config(), :public_key) - conn - |> put_resp_content_type("application/xml") - |> send_resp(200, response) + uploadlimit = %{ + uploadlimit: to_string(Keyword.get(instance, :upload_limit)), + avatarlimit: to_string(Keyword.get(instance, :avatar_upload_limit)), + backgroundlimit: to_string(Keyword.get(instance, :background_upload_limit)), + bannerlimit: to_string(Keyword.get(instance, :banner_upload_limit)) + } - _ -> - vapid_public_key = Keyword.get(Pleroma.Web.Push.vapid_config(), :public_key) + data = %{ + name: Keyword.get(instance, :name), + description: Keyword.get(instance, :description), + server: Web.base_url(), + textlimit: to_string(Keyword.get(instance, :limit)), + uploadlimit: uploadlimit, + closed: bool_to_val(Keyword.get(instance, :registrations_open), "0", "1"), + private: bool_to_val(Keyword.get(instance, :public, true), "0", "1"), + vapidPublicKey: vapid_public_key, + accountActivationRequired: + bool_to_val(Keyword.get(instance, :account_activation_required, false)), + invitesEnabled: bool_to_val(Keyword.get(instance, :invites_enabled, false)), + safeDMMentionsEnabled: bool_to_val(Pleroma.Config.get([:instance, :safe_dm_mentions])) + } - uploadlimit = %{ - uploadlimit: to_string(Keyword.get(instance, :upload_limit)), - avatarlimit: to_string(Keyword.get(instance, :avatar_upload_limit)), - backgroundlimit: to_string(Keyword.get(instance, :background_upload_limit)), - bannerlimit: to_string(Keyword.get(instance, :banner_upload_limit)) - } - - data = %{ - name: Keyword.get(instance, :name), - description: Keyword.get(instance, :description), - server: Web.base_url(), - textlimit: to_string(Keyword.get(instance, :limit)), - uploadlimit: uploadlimit, - closed: if(Keyword.get(instance, :registrations_open), do: "0", else: "1"), - private: if(Keyword.get(instance, :public, true), do: "0", else: "1"), - vapidPublicKey: vapid_public_key, - accountActivationRequired: - if(Keyword.get(instance, :account_activation_required, false), do: "1", else: "0"), - invitesEnabled: if(Keyword.get(instance, :invites_enabled, false), do: "1", else: "0"), - safeDMMentionsEnabled: - if(Pleroma.Config.get([:instance, :safe_dm_mentions]), do: "1", else: "0") - } + managed_config = Keyword.get(instance, :managed_config) + data = + if managed_config do pleroma_fe = Pleroma.Config.get([:frontend_configurations, :pleroma_fe]) + Map.put(data, "pleromafe", pleroma_fe) + else + data + end - managed_config = Keyword.get(instance, :managed_config) - - data = - if managed_config do - data |> Map.put("pleromafe", pleroma_fe) - else - data - end - - json(conn, %{site: data}) - end + json(conn, %{site: data}) end + defp bool_to_val(true), do: "1" + defp bool_to_val(_), do: "0" + defp bool_to_val(true, val, _), do: val + defp bool_to_val(_, _, val), do: val + def frontend_configurations(conn, _params) do config = Pleroma.Config.get(:frontend_configurations, %{}) @@ -217,20 +225,16 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do json(conn, config) end - def version(conn, _params) do + def version(%{assigns: %{format: "xml"}} = conn, _params) do version = Pleroma.Application.named_version() - case get_format(conn) do - "xml" -> - response = "#{version}" + conn + |> put_resp_content_type("application/xml") + |> send_resp(200, "#{version}") + end - conn - |> put_resp_content_type("application/xml") - |> send_resp(200, response) - - _ -> - json(conn, version) - end + def version(conn, _params) do + json(conn, Pleroma.Application.named_version()) end def emoji(conn, _params) do diff --git a/mix.exs b/mix.exs index 2a8fe2e9d..dfff53d57 100644 --- a/mix.exs +++ b/mix.exs @@ -190,12 +190,13 @@ defmodule Pleroma.Mixfile do tag = String.trim(tag), {describe, 0} <- System.cmd("git", ["describe", "--tags", "--abbrev=8"]), describe = String.trim(describe), - ahead <- String.replace(describe, tag, "") do + ahead <- String.replace(describe, tag, ""), + ahead <- String.trim_leading(ahead, "-") do {String.replace_prefix(tag, "v", ""), if(ahead != "", do: String.trim(ahead))} else _ -> {commit_hash, 0} = System.cmd("git", ["rev-parse", "--short", "HEAD"]) - {nil, "-0-g" <> String.trim(commit_hash)} + {nil, "0-g" <> String.trim(commit_hash)} end if git_tag && version != git_tag do @@ -207,14 +208,15 @@ defmodule Pleroma.Mixfile do # Branch name as pre-release version component, denoted with a dot branch_name = with {branch_name, 0} <- System.cmd("git", ["rev-parse", "--abbrev-ref", "HEAD"]), + branch_name <- String.trim(branch_name), branch_name <- System.get_env("PLEROMA_BUILD_BRANCH") || branch_name, - true <- branch_name != "master" do + true <- branch_name not in ["master", "HEAD"] do branch_name = branch_name |> String.trim() |> String.replace(identifier_filter, "-") - "." <> branch_name + branch_name end build_name = @@ -234,6 +236,17 @@ defmodule Pleroma.Mixfile do env_override -> env_override end + # Pre-release version, denoted by appending a hyphen + # and a series of dot separated identifiers + pre_release = + [git_pre_release, branch_name] + |> Enum.filter(fn string -> string && string != "" end) + |> Enum.join(".") + |> (fn + "" -> nil + string -> "-" <> String.replace(string, identifier_filter, "-") + end).() + # Build metadata, denoted with a plus sign build_metadata = [build_name, env_name] @@ -244,7 +257,7 @@ defmodule Pleroma.Mixfile do string -> "+" <> String.replace(string, identifier_filter, "-") end).() - [version, git_pre_release, branch_name, build_metadata] + [version, pre_release, build_metadata] |> Enum.filter(fn string -> string && string != "" end) |> Enum.join() end diff --git a/priv/static/emoji/f_00b.png b/priv/static/emoji/f_00b.png deleted file mode 100644 index 3d00b89b0..000000000 Binary files a/priv/static/emoji/f_00b.png and /dev/null differ diff --git a/priv/static/emoji/f_00b11b.png b/priv/static/emoji/f_00b11b.png deleted file mode 100644 index 3e99ce464..000000000 Binary files a/priv/static/emoji/f_00b11b.png and /dev/null differ diff --git a/priv/static/emoji/f_00b33b.png b/priv/static/emoji/f_00b33b.png deleted file mode 100644 index 8f4929297..000000000 Binary files a/priv/static/emoji/f_00b33b.png and /dev/null differ diff --git a/priv/static/emoji/f_00h.png b/priv/static/emoji/f_00h.png deleted file mode 100644 index ba3da57c6..000000000 Binary files a/priv/static/emoji/f_00h.png and /dev/null differ diff --git a/priv/static/emoji/f_00t.png b/priv/static/emoji/f_00t.png deleted file mode 100644 index 31d98b433..000000000 Binary files a/priv/static/emoji/f_00t.png and /dev/null differ diff --git a/priv/static/emoji/f_01b.png b/priv/static/emoji/f_01b.png deleted file mode 100644 index 7bd2582c5..000000000 Binary files a/priv/static/emoji/f_01b.png and /dev/null differ diff --git a/priv/static/emoji/f_03b.png b/priv/static/emoji/f_03b.png deleted file mode 100644 index 9e4ff1bf7..000000000 Binary files a/priv/static/emoji/f_03b.png and /dev/null differ diff --git a/priv/static/emoji/f_10b.png b/priv/static/emoji/f_10b.png deleted file mode 100644 index 67c6493fc..000000000 Binary files a/priv/static/emoji/f_10b.png and /dev/null differ diff --git a/priv/static/emoji/f_11b.png b/priv/static/emoji/f_11b.png deleted file mode 100644 index b53328ba9..000000000 Binary files a/priv/static/emoji/f_11b.png and /dev/null differ diff --git a/priv/static/emoji/f_11b00b.png b/priv/static/emoji/f_11b00b.png deleted file mode 100644 index c4c30e11f..000000000 Binary files a/priv/static/emoji/f_11b00b.png and /dev/null differ diff --git a/priv/static/emoji/f_11b22b.png b/priv/static/emoji/f_11b22b.png deleted file mode 100644 index 47425e06e..000000000 Binary files a/priv/static/emoji/f_11b22b.png and /dev/null differ diff --git a/priv/static/emoji/f_11h.png b/priv/static/emoji/f_11h.png deleted file mode 100644 index 28342363a..000000000 Binary files a/priv/static/emoji/f_11h.png and /dev/null differ diff --git a/priv/static/emoji/f_11t.png b/priv/static/emoji/f_11t.png deleted file mode 100644 index dca67dc70..000000000 Binary files a/priv/static/emoji/f_11t.png and /dev/null differ diff --git a/priv/static/emoji/f_12b.png b/priv/static/emoji/f_12b.png deleted file mode 100644 index 9925adb7c..000000000 Binary files a/priv/static/emoji/f_12b.png and /dev/null differ diff --git a/priv/static/emoji/f_21b.png b/priv/static/emoji/f_21b.png deleted file mode 100644 index aa56d2cb2..000000000 Binary files a/priv/static/emoji/f_21b.png and /dev/null differ diff --git a/priv/static/emoji/f_22b.png b/priv/static/emoji/f_22b.png deleted file mode 100644 index 426878986..000000000 Binary files a/priv/static/emoji/f_22b.png and /dev/null differ diff --git a/priv/static/emoji/f_22b11b.png b/priv/static/emoji/f_22b11b.png deleted file mode 100644 index 4bdfb3107..000000000 Binary files a/priv/static/emoji/f_22b11b.png and /dev/null differ diff --git a/priv/static/emoji/f_22b33b.png b/priv/static/emoji/f_22b33b.png deleted file mode 100644 index adf94f811..000000000 Binary files a/priv/static/emoji/f_22b33b.png and /dev/null differ diff --git a/priv/static/emoji/f_22h.png b/priv/static/emoji/f_22h.png deleted file mode 100644 index 3b27e2de8..000000000 Binary files a/priv/static/emoji/f_22h.png and /dev/null differ diff --git a/priv/static/emoji/f_22t.png b/priv/static/emoji/f_22t.png deleted file mode 100644 index addd9fec7..000000000 Binary files a/priv/static/emoji/f_22t.png and /dev/null differ diff --git a/priv/static/emoji/f_23b.png b/priv/static/emoji/f_23b.png deleted file mode 100644 index beb69ab36..000000000 Binary files a/priv/static/emoji/f_23b.png and /dev/null differ diff --git a/priv/static/emoji/f_30b.png b/priv/static/emoji/f_30b.png deleted file mode 100644 index 41dbb2a5d..000000000 Binary files a/priv/static/emoji/f_30b.png and /dev/null differ diff --git a/priv/static/emoji/f_32b.png b/priv/static/emoji/f_32b.png deleted file mode 100644 index d8261e8a8..000000000 Binary files a/priv/static/emoji/f_32b.png and /dev/null differ diff --git a/priv/static/emoji/f_33b.png b/priv/static/emoji/f_33b.png deleted file mode 100644 index 71b8b914a..000000000 Binary files a/priv/static/emoji/f_33b.png and /dev/null differ diff --git a/priv/static/emoji/f_33b00b.png b/priv/static/emoji/f_33b00b.png deleted file mode 100644 index 65b6e24b8..000000000 Binary files a/priv/static/emoji/f_33b00b.png and /dev/null differ diff --git a/priv/static/emoji/f_33b22b.png b/priv/static/emoji/f_33b22b.png deleted file mode 100644 index d71a8ddd4..000000000 Binary files a/priv/static/emoji/f_33b22b.png and /dev/null differ diff --git a/priv/static/emoji/f_33h.png b/priv/static/emoji/f_33h.png deleted file mode 100644 index e141c5184..000000000 Binary files a/priv/static/emoji/f_33h.png and /dev/null differ diff --git a/priv/static/emoji/f_33t.png b/priv/static/emoji/f_33t.png deleted file mode 100644 index d5a23073d..000000000 Binary files a/priv/static/emoji/f_33t.png and /dev/null differ diff --git a/test/emails/admin_email_test.exs b/test/emails/admin_email_test.exs index 4bf54b0c2..9e83c73c6 100644 --- a/test/emails/admin_email_test.exs +++ b/test/emails/admin_email_test.exs @@ -24,7 +24,6 @@ defmodule Pleroma.Emails.AdminEmailTest do assert res.to == [{to_user.name, to_user.email}] assert res.from == {config[:name], config[:notify_email]} - assert res.reply_to == {reporter.name, reporter.email} assert res.subject == "#{config[:name]} Report" assert res.html_body == @@ -34,4 +33,17 @@ defmodule Pleroma.Emails.AdminEmailTest do status_url }\">#{status_url}\n \n

\n\n" end + + test "it works when the reporter is a remote user without email" do + config = Pleroma.Config.get(:instance) + to_user = insert(:user) + reporter = insert(:user, email: nil, local: false) + account = insert(:user) + + res = + AdminEmail.report(to_user, reporter, account, [%{name: "Test", id: "12"}], "Test comment") + + assert res.to == [{to_user.name, to_user.email}] + assert res.from == {config[:name], config[:notify_email]} + end end diff --git a/test/fixtures/nypd-facial-recognition-children-teenagers.html b/test/fixtures/nypd-facial-recognition-children-teenagers.html new file mode 100644 index 000000000..5702c4484 --- /dev/null +++ b/test/fixtures/nypd-facial-recognition-children-teenagers.html @@ -0,0 +1,227 @@ + + + + She Was Arrested at 14. Then Her Photo Went to a Facial Recognition Database. - The New York Times + + + + + + + + + + + + + + + + + + + + + +

Advertisement

She Was Arrested at 14. Then Her Photo Went to a Facial Recognition Database.

With little oversight, the N.Y.P.D. has been using powerful surveillance technology on photos of children and teenagers.

Image
CreditCreditSarah Blesener for The New York Times
Joseph GoldsteinAli Watkins

[What you need to know to start the day: Get New York Today in your inbox.]

The New York Police Department has been loading thousands of arrest photos of children and teenagers into a facial recognition database despite evidence the technology has a higher risk of false matches in younger faces.

For about four years, internal records show, the department has used the technology to compare crime scene images with its collection of juvenile mug shots, the photos that are taken at an arrest. Most of the photos are of teenagers, largely 13 to 16 years old, but children as young as 11 have been included.

Elected officials and civil rights groups said the disclosure that the city was deploying a powerful surveillance tool on adolescents — whose privacy seems sacrosanct and whose status is protected in the criminal justice system — was a striking example of the Police Department’s ability to adopt advancing technology with little public scrutiny.

Several members of the City Council as well as a range of civil liberties groups said they were unaware of the policy until they were contacted by The New York Times.

Police Department officials defended the decision, saying it was just the latest evolution of a longstanding policing technique: using arrest photos to identify suspects.

“I don’t think this is any secret decision that’s made behind closed doors,” the city’s chief of detectives, Dermot F. Shea, said in an interview. “This is just process, and making sure we’re doing everything to fight crime.”

Other cities have begun to debate whether law enforcement should use facial recognition, which relies on an algorithm to quickly pore through images and suggest matches. In May, San Francisco blocked city agencies, including the police, from using the tool amid unease about potential government abuse. Detroit is facing public resistance to a technology that has been shown to have lower accuracy with people with darker skin.

In New York, the state Education Department recently told the Lockport, N.Y., school district to delay a plan to use facial recognition on students, citing privacy concerns.

“At the end of the day, it should be banned — no young people,” said Councilman Donovan Richards, a Queens Democrat who heads the Public Safety Committee, which oversees the Police Department.

The department said its legal bureau had approved using facial recognition on juveniles. The algorithm may suggest a lead, but detectives would not make an arrest based solely on that, Chief Shea said.

Image
CreditChang W. Lee/The New York Times

Still, facial recognition has not been widely tested on children. Most algorithms are taught to “think” based on adult faces, and there is growing evidence that they do not work as well on children.

The National Institute of Standards and Technology, which is part of the Commerce Department and evaluates facial recognition algorithms for accuracy, recently found the vast majority of more than 100 facial recognition algorithms had a higher rate of mistaken matches among children. The error rate was most pronounced in young children but was also seen in those aged 10 to 16.

Aging poses another problem: The appearance of children and adolescents can change drastically as bones stretch and shift, altering the underlying facial structure.

“I would use extreme caution in using those algorithms,” said Karl Ricanek Jr., a computer science professor and co-founder of the Face Aging Group at the University of North Carolina-Wilmington.

Technology that can match an image of a younger teenager to a recent arrest photo may be less effective at finding the same person even one or two years later, he said.

“The systems do not have the capacity to understand the dynamic changes that occur to a child’s face,” Dr. Ricanek said.

Idemia and DataWorks Plus, the two companies that provide facial recognition software to the Police Department, did not respond to requests for comment.

The New York Police Department can take arrest photos of minors as young as 11 who are charged with a felony, depending on the severity of the charge.

And in many cases, the department keeps the photos for years, making facial recognition comparisons to what may have effectively become outdated images. There are photos of 5,500 individuals in the juvenile database, 4,100 of whom are no longer 16 or under, the department said. Teenagers 17 and older are considered adults in the criminal justice system.

Police officials declined to provide statistics on how often their facial recognition systems provide false matches, or to explain how they evaluate the system’s effectiveness.

“We are comfortable with this technology because it has proved to be a valuable investigative method,” Chief Shea said. Facial recognition has helped lead to thousands of arrests of both adults and juveniles, the department has said.

Mayor Bill de Blasio had been aware the department was using the technology on minors, said Freddi Goldstein, a spokeswoman for the mayor.

She said the Police Department followed “strict guidelines” in applying the technology and City Hall monitored the agency’s compliance with the policies.

The Times learned details of the department’s use of facial recognition by reviewing documents posted online earlier this year by Clare Garvie, a senior associate at the Center on Privacy and Technology at Georgetown Law. Ms. Garvie received the documents as part of an open records lawsuit.

It could not be determined whether other large police departments used facial recognition with juveniles because very few have written policies governing the use of the technology, Ms. Garvie said.

New York detectives rely on a vast network of surveillance cameras throughout the city to provide images of people believed to have committed a crime. Since 2011, the department has had a dedicated unit of officers who use facial recognition to compare those images against millions of photos, usually mug shots. The software proposes matches, which have led to thousands of arrests, the department said.

By 2013, top police officials were meeting to discuss including juveniles in the program, the documents reviewed by The Times showed.

The documents showed that the juvenile database had been integrated into the system by 2015.

“We have these photos. It makes sense,” Chief Shea said in the interview.

State law requires that arrest photos be destroyed if the case is resolved in the juvenile’s favor, or if the child is found to have committed only a misdemeanor, rather than a felony. The photos also must be destroyed if a person reaches age 21 without a criminal record.

When children are charged with crimes, the court system usually takes some steps to prevent their acts from defining them in later years. Children who are 16 and under, for instance, are generally sent to Family Court, where records are not public.

Yet including their photos in a facial recognition database runs the risk that an imperfect algorithm identifies them as possible suspects in later crimes, civil rights advocates said. A mistaken match could lead investigators to focus on the wrong person from the outset, they said.

“It’s very disturbing to know that no matter what I’m doing at that moment, someone might be scanning my picture to try to find someone who committed a crime,” said Bailey, a 17-year-old Brooklyn girl who had admitted guilt in Family Court to a group attack that happened when she was 14. She said she was present at the attack but did not participate.

Bailey, who asked that she be identified only by her last name because she did not want her juvenile arrest to be public, has not been arrested again and is now a student at John Jay College of Criminal Justice.

Recent studies indicate that people of color, as well as children and women, have a greater risk of misidentification than their counterparts, said Joy Buolamwini, the founder of the Algorithmic Justice League and graduate researcher at the M.I.T. Media Lab, who has examined how human biases are built into artificial intelligence.

The racial disparities in the juvenile justice system are stark: In New York, black and Latino juveniles were charged with crimes at far higher rates than whites in 2017, the most recent year for which numbers were available. Black juveniles outnumbered white juveniles more than 15 to 1.

“If the facial recognition algorithm has a negative bias toward a black population, that will get magnified more toward children,” Dr. Ricanek said, adding that in terms of diminished accuracy, “you’re now putting yourself in unknown territory.”

Opinion | James O’Neill
How Facial Recognition Makes You Safer
Opinion | Jim Shultz
Spying on Children Won’t Keep Them Safe

Joseph Goldstein writes about policing and the criminal justice system. He has been a reporter at The Times since 2011, and is based in New York. He also worked for a year in the Kabul bureau, reporting on Afghanistan. @JoeKGoldstein

Ali Watkins is a reporter on the Metro Desk, covering courts and social services. Previously, she covered national security in Washington for The Times, BuzzFeed and McClatchy Newspapers. @AliWatkins

A version of this article appears in print on , Section A, Page 1 of the New York edition with the headline: In New York, Police Computers Scan Faces, Some as Young as 11. Order Reprints | Today’s Paper | Subscribe

Advertisement

+ + + + + + + + + + +
+ +
+ + + + \ No newline at end of file diff --git a/test/fixtures/nypd-facial-recognition-children-teenagers2.html b/test/fixtures/nypd-facial-recognition-children-teenagers2.html new file mode 100644 index 000000000..ae8b26aff --- /dev/null +++ b/test/fixtures/nypd-facial-recognition-children-teenagers2.html @@ -0,0 +1,226 @@ + + + + She Was Arrested at 14. Then Her Photo Went to a Facial Recognition Database. - The New York Times + + + + + + + + + + + + + + + + + + + + +

Advertisement

She Was Arrested at 14. Then Her Photo Went to a Facial Recognition Database.

With little oversight, the N.Y.P.D. has been using powerful surveillance technology on photos of children and teenagers.

Image
CreditCreditSarah Blesener for The New York Times
Joseph GoldsteinAli Watkins

[What you need to know to start the day: Get New York Today in your inbox.]

The New York Police Department has been loading thousands of arrest photos of children and teenagers into a facial recognition database despite evidence the technology has a higher risk of false matches in younger faces.

For about four years, internal records show, the department has used the technology to compare crime scene images with its collection of juvenile mug shots, the photos that are taken at an arrest. Most of the photos are of teenagers, largely 13 to 16 years old, but children as young as 11 have been included.

Elected officials and civil rights groups said the disclosure that the city was deploying a powerful surveillance tool on adolescents — whose privacy seems sacrosanct and whose status is protected in the criminal justice system — was a striking example of the Police Department’s ability to adopt advancing technology with little public scrutiny.

Several members of the City Council as well as a range of civil liberties groups said they were unaware of the policy until they were contacted by The New York Times.

Police Department officials defended the decision, saying it was just the latest evolution of a longstanding policing technique: using arrest photos to identify suspects.

“I don’t think this is any secret decision that’s made behind closed doors,” the city’s chief of detectives, Dermot F. Shea, said in an interview. “This is just process, and making sure we’re doing everything to fight crime.”

Other cities have begun to debate whether law enforcement should use facial recognition, which relies on an algorithm to quickly pore through images and suggest matches. In May, San Francisco blocked city agencies, including the police, from using the tool amid unease about potential government abuse. Detroit is facing public resistance to a technology that has been shown to have lower accuracy with people with darker skin.

In New York, the state Education Department recently told the Lockport, N.Y., school district to delay a plan to use facial recognition on students, citing privacy concerns.

“At the end of the day, it should be banned — no young people,” said Councilman Donovan Richards, a Queens Democrat who heads the Public Safety Committee, which oversees the Police Department.

The department said its legal bureau had approved using facial recognition on juveniles. The algorithm may suggest a lead, but detectives would not make an arrest based solely on that, Chief Shea said.

Image
CreditChang W. Lee/The New York Times

Still, facial recognition has not been widely tested on children. Most algorithms are taught to “think” based on adult faces, and there is growing evidence that they do not work as well on children.

The National Institute of Standards and Technology, which is part of the Commerce Department and evaluates facial recognition algorithms for accuracy, recently found the vast majority of more than 100 facial recognition algorithms had a higher rate of mistaken matches among children. The error rate was most pronounced in young children but was also seen in those aged 10 to 16.

Aging poses another problem: The appearance of children and adolescents can change drastically as bones stretch and shift, altering the underlying facial structure.

“I would use extreme caution in using those algorithms,” said Karl Ricanek Jr., a computer science professor and co-founder of the Face Aging Group at the University of North Carolina-Wilmington.

Technology that can match an image of a younger teenager to a recent arrest photo may be less effective at finding the same person even one or two years later, he said.

“The systems do not have the capacity to understand the dynamic changes that occur to a child’s face,” Dr. Ricanek said.

Idemia and DataWorks Plus, the two companies that provide facial recognition software to the Police Department, did not respond to requests for comment.

The New York Police Department can take arrest photos of minors as young as 11 who are charged with a felony, depending on the severity of the charge.

And in many cases, the department keeps the photos for years, making facial recognition comparisons to what may have effectively become outdated images. There are photos of 5,500 individuals in the juvenile database, 4,100 of whom are no longer 16 or under, the department said. Teenagers 17 and older are considered adults in the criminal justice system.

Police officials declined to provide statistics on how often their facial recognition systems provide false matches, or to explain how they evaluate the system’s effectiveness.

“We are comfortable with this technology because it has proved to be a valuable investigative method,” Chief Shea said. Facial recognition has helped lead to thousands of arrests of both adults and juveniles, the department has said.

Mayor Bill de Blasio had been aware the department was using the technology on minors, said Freddi Goldstein, a spokeswoman for the mayor.

She said the Police Department followed “strict guidelines” in applying the technology and City Hall monitored the agency’s compliance with the policies.

The Times learned details of the department’s use of facial recognition by reviewing documents posted online earlier this year by Clare Garvie, a senior associate at the Center on Privacy and Technology at Georgetown Law. Ms. Garvie received the documents as part of an open records lawsuit.

It could not be determined whether other large police departments used facial recognition with juveniles because very few have written policies governing the use of the technology, Ms. Garvie said.

New York detectives rely on a vast network of surveillance cameras throughout the city to provide images of people believed to have committed a crime. Since 2011, the department has had a dedicated unit of officers who use facial recognition to compare those images against millions of photos, usually mug shots. The software proposes matches, which have led to thousands of arrests, the department said.

By 2013, top police officials were meeting to discuss including juveniles in the program, the documents reviewed by The Times showed.

The documents showed that the juvenile database had been integrated into the system by 2015.

“We have these photos. It makes sense,” Chief Shea said in the interview.

State law requires that arrest photos be destroyed if the case is resolved in the juvenile’s favor, or if the child is found to have committed only a misdemeanor, rather than a felony. The photos also must be destroyed if a person reaches age 21 without a criminal record.

When children are charged with crimes, the court system usually takes some steps to prevent their acts from defining them in later years. Children who are 16 and under, for instance, are generally sent to Family Court, where records are not public.

Yet including their photos in a facial recognition database runs the risk that an imperfect algorithm identifies them as possible suspects in later crimes, civil rights advocates said. A mistaken match could lead investigators to focus on the wrong person from the outset, they said.

“It’s very disturbing to know that no matter what I’m doing at that moment, someone might be scanning my picture to try to find someone who committed a crime,” said Bailey, a 17-year-old Brooklyn girl who had admitted guilt in Family Court to a group attack that happened when she was 14. She said she was present at the attack but did not participate.

Bailey, who asked that she be identified only by her last name because she did not want her juvenile arrest to be public, has not been arrested again and is now a student at John Jay College of Criminal Justice.

Recent studies indicate that people of color, as well as children and women, have a greater risk of misidentification than their counterparts, said Joy Buolamwini, the founder of the Algorithmic Justice League and graduate researcher at the M.I.T. Media Lab, who has examined how human biases are built into artificial intelligence.

The racial disparities in the juvenile justice system are stark: In New York, black and Latino juveniles were charged with crimes at far higher rates than whites in 2017, the most recent year for which numbers were available. Black juveniles outnumbered white juveniles more than 15 to 1.

“If the facial recognition algorithm has a negative bias toward a black population, that will get magnified more toward children,” Dr. Ricanek said, adding that in terms of diminished accuracy, “you’re now putting yourself in unknown territory.”

Opinion | James O’Neill
How Facial Recognition Makes You Safer
Opinion | Jim Shultz
Spying on Children Won’t Keep Them Safe

Joseph Goldstein writes about policing and the criminal justice system. He has been a reporter at The Times since 2011, and is based in New York. He also worked for a year in the Kabul bureau, reporting on Afghanistan. @JoeKGoldstein

Ali Watkins is a reporter on the Metro Desk, covering courts and social services. Previously, she covered national security in Washington for The Times, BuzzFeed and McClatchy Newspapers. @AliWatkins

A version of this article appears in print on , Section A, Page 1 of the New York edition with the headline: In New York, Police Computers Scan Faces, Some as Young as 11. Order Reprints | Today’s Paper | Subscribe

Advertisement

+ + + + + + + + + + +
+ +
+ + + + diff --git a/test/fixtures/nypd-facial-recognition-children-teenagers3.html b/test/fixtures/nypd-facial-recognition-children-teenagers3.html new file mode 100644 index 000000000..53454d23e --- /dev/null +++ b/test/fixtures/nypd-facial-recognition-children-teenagers3.html @@ -0,0 +1,227 @@ + + + + She Was Arrested at 14. Then Her Photo Went to a Facial Recognition Database. - The New York Times + + + + + + + + + + + + + + + + + + + + + +

Advertisement

She Was Arrested at 14. Then Her Photo Went to a Facial Recognition Database.

With little oversight, the N.Y.P.D. has been using powerful surveillance technology on photos of children and teenagers.

Image
CreditCreditSarah Blesener for The New York Times
Joseph GoldsteinAli Watkins

[What you need to know to start the day: Get New York Today in your inbox.]

The New York Police Department has been loading thousands of arrest photos of children and teenagers into a facial recognition database despite evidence the technology has a higher risk of false matches in younger faces.

For about four years, internal records show, the department has used the technology to compare crime scene images with its collection of juvenile mug shots, the photos that are taken at an arrest. Most of the photos are of teenagers, largely 13 to 16 years old, but children as young as 11 have been included.

Elected officials and civil rights groups said the disclosure that the city was deploying a powerful surveillance tool on adolescents — whose privacy seems sacrosanct and whose status is protected in the criminal justice system — was a striking example of the Police Department’s ability to adopt advancing technology with little public scrutiny.

Several members of the City Council as well as a range of civil liberties groups said they were unaware of the policy until they were contacted by The New York Times.

Police Department officials defended the decision, saying it was just the latest evolution of a longstanding policing technique: using arrest photos to identify suspects.

“I don’t think this is any secret decision that’s made behind closed doors,” the city’s chief of detectives, Dermot F. Shea, said in an interview. “This is just process, and making sure we’re doing everything to fight crime.”

Other cities have begun to debate whether law enforcement should use facial recognition, which relies on an algorithm to quickly pore through images and suggest matches. In May, San Francisco blocked city agencies, including the police, from using the tool amid unease about potential government abuse. Detroit is facing public resistance to a technology that has been shown to have lower accuracy with people with darker skin.

In New York, the state Education Department recently told the Lockport, N.Y., school district to delay a plan to use facial recognition on students, citing privacy concerns.

“At the end of the day, it should be banned — no young people,” said Councilman Donovan Richards, a Queens Democrat who heads the Public Safety Committee, which oversees the Police Department.

The department said its legal bureau had approved using facial recognition on juveniles. The algorithm may suggest a lead, but detectives would not make an arrest based solely on that, Chief Shea said.

Image
CreditChang W. Lee/The New York Times

Still, facial recognition has not been widely tested on children. Most algorithms are taught to “think” based on adult faces, and there is growing evidence that they do not work as well on children.

The National Institute of Standards and Technology, which is part of the Commerce Department and evaluates facial recognition algorithms for accuracy, recently found the vast majority of more than 100 facial recognition algorithms had a higher rate of mistaken matches among children. The error rate was most pronounced in young children but was also seen in those aged 10 to 16.

Aging poses another problem: The appearance of children and adolescents can change drastically as bones stretch and shift, altering the underlying facial structure.

“I would use extreme caution in using those algorithms,” said Karl Ricanek Jr., a computer science professor and co-founder of the Face Aging Group at the University of North Carolina-Wilmington.

Technology that can match an image of a younger teenager to a recent arrest photo may be less effective at finding the same person even one or two years later, he said.

“The systems do not have the capacity to understand the dynamic changes that occur to a child’s face,” Dr. Ricanek said.

Idemia and DataWorks Plus, the two companies that provide facial recognition software to the Police Department, did not respond to requests for comment.

The New York Police Department can take arrest photos of minors as young as 11 who are charged with a felony, depending on the severity of the charge.

And in many cases, the department keeps the photos for years, making facial recognition comparisons to what may have effectively become outdated images. There are photos of 5,500 individuals in the juvenile database, 4,100 of whom are no longer 16 or under, the department said. Teenagers 17 and older are considered adults in the criminal justice system.

Police officials declined to provide statistics on how often their facial recognition systems provide false matches, or to explain how they evaluate the system’s effectiveness.

“We are comfortable with this technology because it has proved to be a valuable investigative method,” Chief Shea said. Facial recognition has helped lead to thousands of arrests of both adults and juveniles, the department has said.

Mayor Bill de Blasio had been aware the department was using the technology on minors, said Freddi Goldstein, a spokeswoman for the mayor.

She said the Police Department followed “strict guidelines” in applying the technology and City Hall monitored the agency’s compliance with the policies.

The Times learned details of the department’s use of facial recognition by reviewing documents posted online earlier this year by Clare Garvie, a senior associate at the Center on Privacy and Technology at Georgetown Law. Ms. Garvie received the documents as part of an open records lawsuit.

It could not be determined whether other large police departments used facial recognition with juveniles because very few have written policies governing the use of the technology, Ms. Garvie said.

New York detectives rely on a vast network of surveillance cameras throughout the city to provide images of people believed to have committed a crime. Since 2011, the department has had a dedicated unit of officers who use facial recognition to compare those images against millions of photos, usually mug shots. The software proposes matches, which have led to thousands of arrests, the department said.

By 2013, top police officials were meeting to discuss including juveniles in the program, the documents reviewed by The Times showed.

The documents showed that the juvenile database had been integrated into the system by 2015.

“We have these photos. It makes sense,” Chief Shea said in the interview.

State law requires that arrest photos be destroyed if the case is resolved in the juvenile’s favor, or if the child is found to have committed only a misdemeanor, rather than a felony. The photos also must be destroyed if a person reaches age 21 without a criminal record.

When children are charged with crimes, the court system usually takes some steps to prevent their acts from defining them in later years. Children who are 16 and under, for instance, are generally sent to Family Court, where records are not public.

Yet including their photos in a facial recognition database runs the risk that an imperfect algorithm identifies them as possible suspects in later crimes, civil rights advocates said. A mistaken match could lead investigators to focus on the wrong person from the outset, they said.

“It’s very disturbing to know that no matter what I’m doing at that moment, someone might be scanning my picture to try to find someone who committed a crime,” said Bailey, a 17-year-old Brooklyn girl who had admitted guilt in Family Court to a group attack that happened when she was 14. She said she was present at the attack but did not participate.

Bailey, who asked that she be identified only by her last name because she did not want her juvenile arrest to be public, has not been arrested again and is now a student at John Jay College of Criminal Justice.

Recent studies indicate that people of color, as well as children and women, have a greater risk of misidentification than their counterparts, said Joy Buolamwini, the founder of the Algorithmic Justice League and graduate researcher at the M.I.T. Media Lab, who has examined how human biases are built into artificial intelligence.

The racial disparities in the juvenile justice system are stark: In New York, black and Latino juveniles were charged with crimes at far higher rates than whites in 2017, the most recent year for which numbers were available. Black juveniles outnumbered white juveniles more than 15 to 1.

“If the facial recognition algorithm has a negative bias toward a black population, that will get magnified more toward children,” Dr. Ricanek said, adding that in terms of diminished accuracy, “you’re now putting yourself in unknown territory.”

Opinion | James O’Neill
How Facial Recognition Makes You Safer
Opinion | Jim Shultz
Spying on Children Won’t Keep Them Safe

Joseph Goldstein writes about policing and the criminal justice system. He has been a reporter at The Times since 2011, and is based in New York. He also worked for a year in the Kabul bureau, reporting on Afghanistan. @JoeKGoldstein

Ali Watkins is a reporter on the Metro Desk, covering courts and social services. Previously, she covered national security in Washington for The Times, BuzzFeed and McClatchy Newspapers. @AliWatkins

A version of this article appears in print on , Section A, Page 1 of the New York edition with the headline: In New York, Police Computers Scan Faces, Some as Young as 11. Order Reprints | Today’s Paper | Subscribe

Advertisement

+ + + + + + + + + + +
+ +
+ + + + diff --git a/test/fixtures/users_mock/masto_closed_followers_page.json b/test/fixtures/users_mock/masto_closed_followers_page.json new file mode 100644 index 000000000..04ab0c4d3 --- /dev/null +++ b/test/fixtures/users_mock/masto_closed_followers_page.json @@ -0,0 +1 @@ +{"@context":"https://www.w3.org/ns/activitystreams","id":"http://localhost:4001/users/masto_closed/followers?page=1","type":"OrderedCollectionPage","totalItems":437,"next":"http://localhost:4001/users/masto_closed/followers?page=2","partOf":"http://localhost:4001/users/masto_closed/followers","orderedItems":["https://testing.uguu.ltd/users/rin","https://patch.cx/users/rin","https://letsalllovela.in/users/xoxo","https://pleroma.site/users/crushv","https://aria.company/users/boris","https://kawen.space/users/crushv","https://freespeech.host/users/cvcvcv","https://pleroma.site/users/picpub","https://pixelfed.social/users/nosleep","https://boopsnoot.gq/users/5c1896d162f7d337f90492a3","https://pikachu.rocks/users/waifu","https://royal.crablettesare.life/users/crablettes"]} diff --git a/test/fixtures/users_mock/masto_closed_following_page.json b/test/fixtures/users_mock/masto_closed_following_page.json new file mode 100644 index 000000000..8d8324699 --- /dev/null +++ b/test/fixtures/users_mock/masto_closed_following_page.json @@ -0,0 +1 @@ +{"@context":"https://www.w3.org/ns/activitystreams","id":"http://localhost:4001/users/masto_closed/following?page=1","type":"OrderedCollectionPage","totalItems":152,"next":"http://localhost:4001/users/masto_closed/following?page=2","partOf":"http://localhost:4001/users/masto_closed/following","orderedItems":["https://testing.uguu.ltd/users/rin","https://patch.cx/users/rin","https://letsalllovela.in/users/xoxo","https://pleroma.site/users/crushv","https://aria.company/users/boris","https://kawen.space/users/crushv","https://freespeech.host/users/cvcvcv","https://pleroma.site/users/picpub","https://pixelfed.social/users/nosleep","https://boopsnoot.gq/users/5c1896d162f7d337f90492a3","https://pikachu.rocks/users/waifu","https://royal.crablettesare.life/users/crablettes"]} diff --git a/test/flake_id_test.exs b/test/flake_id_test.exs index ca2338041..85ed5bbdf 100644 --- a/test/flake_id_test.exs +++ b/test/flake_id_test.exs @@ -39,4 +39,9 @@ defmodule Pleroma.FlakeIdTest do assert dump(flake_s) == {:ok, flake} assert dump(flake) == {:ok, flake} end + + test "is_flake_id?/1" do + assert is_flake_id?("9eoozpwTul5mjSEDRI") + refute is_flake_id?("http://example.com/activities/3ebbadd1-eb14-4e20-8118-b6f79c0c7b0b") + end end diff --git a/test/support/factory.ex b/test/support/factory.ex index c751546ce..8f638b98f 100644 --- a/test/support/factory.ex +++ b/test/support/factory.ex @@ -182,8 +182,8 @@ defmodule Pleroma.Factory do } end - def like_activity_factory do - note_activity = insert(:note_activity) + def like_activity_factory(attrs \\ %{}) do + note_activity = attrs[:note_activity] || insert(:note_activity) object = Object.normalize(note_activity) user = insert(:user) diff --git a/test/support/http_request_mock.ex b/test/support/http_request_mock.ex index 2ed5f5042..3adb5ba3b 100644 --- a/test/support/http_request_mock.ex +++ b/test/support/http_request_mock.ex @@ -51,6 +51,10 @@ defmodule HttpRequestMock do }} end + def get("https://mastodon.social/users/not_found", _, _, _) do + {:ok, %Tesla.Env{status: 404}} + end + def get("https://mastodon.sdf.org/users/rinpatch", _, _, _) do {:ok, %Tesla.Env{ @@ -792,6 +796,14 @@ defmodule HttpRequestMock do }} end + def get("http://localhost:4001/users/masto_closed/followers?page=1", _, _, _) do + {:ok, + %Tesla.Env{ + status: 200, + body: File.read!("test/fixtures/users_mock/masto_closed_followers_page.json") + }} + end + def get("http://localhost:4001/users/masto_closed/following", _, _, _) do {:ok, %Tesla.Env{ @@ -800,6 +812,14 @@ defmodule HttpRequestMock do }} end + def get("http://localhost:4001/users/masto_closed/following?page=1", _, _, _) do + {:ok, + %Tesla.Env{ + status: 200, + body: File.read!("test/fixtures/users_mock/masto_closed_following_page.json") + }} + end + def get("http://localhost:4001/users/fuser2/followers", _, _, _) do {:ok, %Tesla.Env{ diff --git a/test/user_test.exs b/test/user_test.exs index 556df45fd..7ec241c25 100644 --- a/test/user_test.exs +++ b/test/user_test.exs @@ -1393,4 +1393,78 @@ defmodule Pleroma.UserTest do assert %User{bio: "test-bio"} = User.get_cached_by_ap_id(user.ap_id) end end + + describe "following/followers synchronization" do + setup do + sync = Pleroma.Config.get([:instance, :external_user_synchronization]) + on_exit(fn -> Pleroma.Config.put([:instance, :external_user_synchronization], sync) end) + end + + test "updates the counters normally on following/getting a follow when disabled" do + Pleroma.Config.put([:instance, :external_user_synchronization], false) + user = insert(:user) + + other_user = + insert(:user, + local: false, + follower_address: "http://localhost:4001/users/masto_closed/followers", + following_address: "http://localhost:4001/users/masto_closed/following", + info: %{ap_enabled: true} + ) + + assert User.user_info(other_user).following_count == 0 + assert User.user_info(other_user).follower_count == 0 + + {:ok, user} = Pleroma.User.follow(user, other_user) + other_user = Pleroma.User.get_by_id(other_user.id) + + assert User.user_info(user).following_count == 1 + assert User.user_info(other_user).follower_count == 1 + end + + test "syncronizes the counters with the remote instance for the followed when enabled" do + Pleroma.Config.put([:instance, :external_user_synchronization], false) + + user = insert(:user) + + other_user = + insert(:user, + local: false, + follower_address: "http://localhost:4001/users/masto_closed/followers", + following_address: "http://localhost:4001/users/masto_closed/following", + info: %{ap_enabled: true} + ) + + assert User.user_info(other_user).following_count == 0 + assert User.user_info(other_user).follower_count == 0 + + Pleroma.Config.put([:instance, :external_user_synchronization], true) + {:ok, _user} = User.follow(user, other_user) + other_user = User.get_by_id(other_user.id) + + assert User.user_info(other_user).follower_count == 437 + end + + test "syncronizes the counters with the remote instance for the follower when enabled" do + Pleroma.Config.put([:instance, :external_user_synchronization], false) + + user = insert(:user) + + other_user = + insert(:user, + local: false, + follower_address: "http://localhost:4001/users/masto_closed/followers", + following_address: "http://localhost:4001/users/masto_closed/following", + info: %{ap_enabled: true} + ) + + assert User.user_info(other_user).following_count == 0 + assert User.user_info(other_user).follower_count == 0 + + Pleroma.Config.put([:instance, :external_user_synchronization], true) + {:ok, other_user} = User.follow(other_user, user) + + assert User.user_info(other_user).following_count == 152 + end + end end diff --git a/test/web/activity_pub/activity_pub_controller_test.exs b/test/web/activity_pub/activity_pub_controller_test.exs index 40344f17e..251055ee1 100644 --- a/test/web/activity_pub/activity_pub_controller_test.exs +++ b/test/web/activity_pub/activity_pub_controller_test.exs @@ -180,18 +180,65 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do end describe "/object/:uuid/likes" do - test "it returns the like activities in a collection", %{conn: conn} do + setup do like = insert(:like_activity) like_object_ap_id = Object.normalize(like).data["id"] - uuid = String.split(like_object_ap_id, "/") |> List.last() + uuid = + like_object_ap_id + |> String.split("/") + |> List.last() + + [id: like.data["id"], uuid: uuid] + end + + test "it returns the like activities in a collection", %{conn: conn, id: id, uuid: uuid} do result = conn |> put_req_header("accept", "application/activity+json") |> get("/objects/#{uuid}/likes") |> json_response(200) - assert List.first(result["first"]["orderedItems"])["id"] == like.data["id"] + assert List.first(result["first"]["orderedItems"])["id"] == id + assert result["type"] == "OrderedCollection" + assert result["totalItems"] == 1 + refute result["first"]["next"] + end + + test "it does not crash when page number is exceeded total pages", %{conn: conn, uuid: uuid} do + result = + conn + |> put_req_header("accept", "application/activity+json") + |> get("/objects/#{uuid}/likes?page=2") + |> json_response(200) + + assert result["type"] == "OrderedCollectionPage" + assert result["totalItems"] == 1 + refute result["next"] + assert Enum.empty?(result["orderedItems"]) + end + + test "it contains the next key when likes count is more than 10", %{conn: conn} do + note = insert(:note_activity) + insert_list(11, :like_activity, note_activity: note) + + uuid = + note + |> Object.normalize() + |> Map.get(:data) + |> Map.get("id") + |> String.split("/") + |> List.last() + + result = + conn + |> put_req_header("accept", "application/activity+json") + |> get("/objects/#{uuid}/likes?page=1") + |> json_response(200) + + assert result["totalItems"] == 11 + assert length(result["orderedItems"]) == 10 + assert result["next"] end end diff --git a/test/web/activity_pub/activity_pub_test.exs b/test/web/activity_pub/activity_pub_test.exs index 1c0b274cb..d723f331f 100644 --- a/test/web/activity_pub/activity_pub_test.exs +++ b/test/web/activity_pub/activity_pub_test.exs @@ -677,14 +677,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do assert object.data["likes"] == [user.ap_id] assert object.data["like_count"] == 1 - [note_activity] = Activity.get_all_create_by_object_ap_id(object.data["id"]) - assert note_activity.data["object"]["like_count"] == 1 - {:ok, _like_activity, object} = ActivityPub.like(user_two, object) assert object.data["like_count"] == 2 - - [note_activity] = Activity.get_all_create_by_object_ap_id(object.data["id"]) - assert note_activity.data["object"]["like_count"] == 2 end end @@ -1128,4 +1122,65 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do assert result.id == activity.id end end + + describe "fetch_follow_information_for_user" do + test "syncronizes following/followers counters" do + user = + insert(:user, + local: false, + follower_address: "http://localhost:4001/users/fuser2/followers", + following_address: "http://localhost:4001/users/fuser2/following" + ) + + {:ok, info} = ActivityPub.fetch_follow_information_for_user(user) + assert info.follower_count == 527 + assert info.following_count == 267 + end + + test "detects hidden followers" do + mock(fn env -> + case env.url do + "http://localhost:4001/users/masto_closed/followers?page=1" -> + %Tesla.Env{status: 403, body: ""} + + _ -> + apply(HttpRequestMock, :request, [env]) + end + end) + + user = + insert(:user, + local: false, + follower_address: "http://localhost:4001/users/masto_closed/followers", + following_address: "http://localhost:4001/users/masto_closed/following" + ) + + {:ok, info} = ActivityPub.fetch_follow_information_for_user(user) + assert info.hide_followers == true + assert info.hide_follows == false + end + + test "detects hidden follows" do + mock(fn env -> + case env.url do + "http://localhost:4001/users/masto_closed/following?page=1" -> + %Tesla.Env{status: 403, body: ""} + + _ -> + apply(HttpRequestMock, :request, [env]) + end + end) + + user = + insert(:user, + local: false, + follower_address: "http://localhost:4001/users/masto_closed/followers", + following_address: "http://localhost:4001/users/masto_closed/following" + ) + + {:ok, info} = ActivityPub.fetch_follow_information_for_user(user) + assert info.hide_followers == false + assert info.hide_follows == true + end + end end diff --git a/test/web/activity_pub/transmogrifier_test.exs b/test/web/activity_pub/transmogrifier_test.exs index a1f5f6e36..e7498e005 100644 --- a/test/web/activity_pub/transmogrifier_test.exs +++ b/test/web/activity_pub/transmogrifier_test.exs @@ -1373,32 +1373,4 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do refute recipient.follower_address in fixed_object["to"] end end - - test "update_following_followers_counters/1" do - user1 = - insert(:user, - local: false, - follower_address: "http://localhost:4001/users/masto_closed/followers", - following_address: "http://localhost:4001/users/masto_closed/following" - ) - - user2 = - insert(:user, - local: false, - follower_address: "http://localhost:4001/users/fuser2/followers", - following_address: "http://localhost:4001/users/fuser2/following" - ) - - Transmogrifier.update_following_followers_counters(user1) - Transmogrifier.update_following_followers_counters(user2) - - %{follower_count: followers, following_count: following} = User.get_cached_user_info(user1) - assert followers == 437 - assert following == 152 - - %{follower_count: followers, following_count: following} = User.get_cached_user_info(user2) - - assert followers == 527 - assert following == 267 - end end diff --git a/test/web/admin_api/admin_api_controller_test.exs b/test/web/admin_api/admin_api_controller_test.exs index 6dda4ae51..bcbc18639 100644 --- a/test/web/admin_api/admin_api_controller_test.exs +++ b/test/web/admin_api/admin_api_controller_test.exs @@ -1914,6 +1914,78 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do ] } end + + test "delete part of settings by atom subkeys", %{conn: conn} do + config = + insert(:config, + key: "keyaa1", + value: :erlang.term_to_binary(subkey1: "val1", subkey2: "val2", subkey3: "val3") + ) + + conn = + post(conn, "/api/pleroma/admin/config", %{ + configs: [ + %{ + group: config.group, + key: config.key, + subkeys: [":subkey1", ":subkey3"], + delete: "true" + } + ] + }) + + assert( + json_response(conn, 200) == %{ + "configs" => [ + %{ + "group" => "pleroma", + "key" => "keyaa1", + "value" => [%{"tuple" => [":subkey2", "val2"]}] + } + ] + } + ) + end + end + + describe "config mix tasks run" do + setup %{conn: conn} do + admin = insert(:user, info: %{is_admin: true}) + + temp_file = "config/test.exported_from_db.secret.exs" + + Mix.shell(Mix.Shell.Quiet) + + on_exit(fn -> + Mix.shell(Mix.Shell.IO) + :ok = File.rm(temp_file) + end) + + dynamic = Pleroma.Config.get([:instance, :dynamic_configuration]) + + Pleroma.Config.put([:instance, :dynamic_configuration], true) + + on_exit(fn -> + Pleroma.Config.put([:instance, :dynamic_configuration], dynamic) + end) + + %{conn: assign(conn, :user, admin), admin: admin} + end + + test "transfer settings to DB and to file", %{conn: conn, admin: admin} do + assert Pleroma.Repo.all(Pleroma.Web.AdminAPI.Config) == [] + conn = get(conn, "/api/pleroma/admin/config/migrate_to_db") + assert json_response(conn, 200) == %{} + assert Pleroma.Repo.all(Pleroma.Web.AdminAPI.Config) > 0 + + conn = + build_conn() + |> assign(:user, admin) + |> get("/api/pleroma/admin/config/migrate_from_db") + + assert json_response(conn, 200) == %{} + assert Pleroma.Repo.all(Pleroma.Web.AdminAPI.Config) == [] + end end describe "GET /api/pleroma/admin/users/:nickname/statuses" do diff --git a/test/web/common_api/common_api_utils_test.exs b/test/web/common_api/common_api_utils_test.exs index 7510c8def..c281dd1f1 100644 --- a/test/web/common_api/common_api_utils_test.exs +++ b/test/web/common_api/common_api_utils_test.exs @@ -306,7 +306,6 @@ defmodule Pleroma.Web.CommonAPI.UtilsTest do mentions = [mentioned_user.ap_id] {to, cc} = Utils.get_to_and_cc(user, mentions, nil, "private", nil) - assert length(to) == 2 assert length(cc) == 0 @@ -360,4 +359,242 @@ defmodule Pleroma.Web.CommonAPI.UtilsTest do assert third_user.ap_id in to end end + + describe "get_by_id_or_ap_id/1" do + test "get activity by id" do + activity = insert(:note_activity) + %Pleroma.Activity{} = note = Utils.get_by_id_or_ap_id(activity.id) + assert note.id == activity.id + end + + test "get activity by ap_id" do + activity = insert(:note_activity) + %Pleroma.Activity{} = note = Utils.get_by_id_or_ap_id(activity.data["object"]) + assert note.id == activity.id + end + + test "get activity by object when type isn't `Create` " do + activity = insert(:like_activity) + %Pleroma.Activity{} = like = Utils.get_by_id_or_ap_id(activity.id) + assert like.data["object"] == activity.data["object"] + end + end + + describe "to_master_date/1" do + test "removes microseconds from date (NaiveDateTime)" do + assert Utils.to_masto_date(~N[2015-01-23 23:50:07.123]) == "2015-01-23T23:50:07.000Z" + end + + test "removes microseconds from date (String)" do + assert Utils.to_masto_date("2015-01-23T23:50:07.123Z") == "2015-01-23T23:50:07.000Z" + end + + test "returns empty string when date invalid" do + assert Utils.to_masto_date("2015-01?23T23:50:07.123Z") == "" + end + end + + describe "conversation_id_to_context/1" do + test "returns id" do + object = insert(:note) + assert Utils.conversation_id_to_context(object.id) == object.data["id"] + end + + test "returns error if object not found" do + assert Utils.conversation_id_to_context("123") == {:error, "No such conversation"} + end + end + + describe "maybe_notify_mentioned_recipients/2" do + test "returns recipients when activity is not `Create`" do + activity = insert(:like_activity) + assert Utils.maybe_notify_mentioned_recipients(["test"], activity) == ["test"] + end + + test "returns recipients from tag" do + user = insert(:user) + + object = + insert(:note, + user: user, + data: %{ + "tag" => [ + %{"type" => "Hashtag"}, + "", + %{"type" => "Mention", "href" => "https://testing.pleroma.lol/users/lain"}, + %{"type" => "Mention", "href" => "https://shitposter.club/user/5381"}, + %{"type" => "Mention", "href" => "https://shitposter.club/user/5381"} + ] + } + ) + + activity = insert(:note_activity, user: user, note: object) + + assert Utils.maybe_notify_mentioned_recipients(["test"], activity) == [ + "test", + "https://testing.pleroma.lol/users/lain", + "https://shitposter.club/user/5381" + ] + end + + test "returns recipients when object is map" do + user = insert(:user) + object = insert(:note, user: user) + + activity = + insert(:note_activity, + user: user, + note: object, + data_attrs: %{ + "object" => %{ + "tag" => [ + %{"type" => "Hashtag"}, + "", + %{"type" => "Mention", "href" => "https://testing.pleroma.lol/users/lain"}, + %{"type" => "Mention", "href" => "https://shitposter.club/user/5381"}, + %{"type" => "Mention", "href" => "https://shitposter.club/user/5381"} + ] + } + } + ) + + Pleroma.Repo.delete(object) + + assert Utils.maybe_notify_mentioned_recipients(["test"], activity) == [ + "test", + "https://testing.pleroma.lol/users/lain", + "https://shitposter.club/user/5381" + ] + end + + test "returns recipients when object not found" do + user = insert(:user) + object = insert(:note, user: user) + + activity = insert(:note_activity, user: user, note: object) + Pleroma.Repo.delete(object) + + assert Utils.maybe_notify_mentioned_recipients(["test-test"], activity) == [ + "test-test" + ] + end + end + + describe "attachments_from_ids_descs/2" do + test "returns [] when attachment ids is empty" do + assert Utils.attachments_from_ids_descs([], "{}") == [] + end + + test "returns list attachments with desc" do + object = insert(:note) + desc = Jason.encode!(%{object.id => "test-desc"}) + + assert Utils.attachments_from_ids_descs(["#{object.id}", "34"], desc) == [ + Map.merge(object.data, %{"name" => "test-desc"}) + ] + end + end + + describe "attachments_from_ids/1" do + test "returns attachments with descs" do + object = insert(:note) + desc = Jason.encode!(%{object.id => "test-desc"}) + + assert Utils.attachments_from_ids(%{ + "media_ids" => ["#{object.id}"], + "descriptions" => desc + }) == [ + Map.merge(object.data, %{"name" => "test-desc"}) + ] + end + + test "returns attachments without descs" do + object = insert(:note) + assert Utils.attachments_from_ids(%{"media_ids" => ["#{object.id}"]}) == [object.data] + end + + test "returns [] when not pass media_ids" do + assert Utils.attachments_from_ids(%{}) == [] + end + end + + describe "maybe_add_list_data/3" do + test "adds list params when found user list" do + user = insert(:user) + {:ok, %Pleroma.List{} = list} = Pleroma.List.create("title", user) + + assert Utils.maybe_add_list_data(%{additional: %{}, object: %{}}, user, {:list, list.id}) == + %{ + additional: %{"bcc" => [list.ap_id], "listMessage" => list.ap_id}, + object: %{"listMessage" => list.ap_id} + } + end + + test "returns original params when list not found" do + user = insert(:user) + {:ok, %Pleroma.List{} = list} = Pleroma.List.create("title", insert(:user)) + + assert Utils.maybe_add_list_data(%{additional: %{}, object: %{}}, user, {:list, list.id}) == + %{additional: %{}, object: %{}} + end + end + + describe "make_note_data/11" do + test "returns note data" do + user = insert(:user) + note = insert(:note) + user2 = insert(:user) + user3 = insert(:user) + + assert Utils.make_note_data( + user.ap_id, + [user2.ap_id], + "2hu", + "

This is :moominmamma: note

", + [], + note.id, + [name: "jimm"], + "test summary", + [user3.ap_id], + false, + %{"custom_tag" => "test"} + ) == %{ + "actor" => user.ap_id, + "attachment" => [], + "cc" => [user3.ap_id], + "content" => "

This is :moominmamma: note

", + "context" => "2hu", + "sensitive" => false, + "summary" => "test summary", + "tag" => ["jimm"], + "to" => [user2.ap_id], + "type" => "Note", + "custom_tag" => "test" + } + end + end + + describe "maybe_add_attachments/3" do + test "returns parsed results when no_links is true" do + assert Utils.maybe_add_attachments( + {"test", [], ["tags"]}, + [], + true + ) == {"test", [], ["tags"]} + end + + test "adds attachments to parsed results" do + attachment = %{"url" => [%{"href" => "SakuraPM.png"}]} + + assert Utils.maybe_add_attachments( + {"test", [], ["tags"]}, + [attachment], + false + ) == { + "test
SakuraPM.png", + [], + ["tags"] + } + end + end end diff --git a/test/web/federator_test.exs b/test/web/federator_test.exs index 6e143eee4..73cfaa8f1 100644 --- a/test/web/federator_test.exs +++ b/test/web/federator_test.exs @@ -229,5 +229,21 @@ defmodule Pleroma.Web.FederatorTest do :error = Federator.incoming_ap_doc(params) end + + test "it does not crash if MRF rejects the post" do + policies = Pleroma.Config.get([:instance, :rewrite_policy]) + mrf_keyword_policy = Pleroma.Config.get(:mrf_keyword) + Pleroma.Config.put([:mrf_keyword, :reject], ["lain"]) + Pleroma.Config.put([:instance, :rewrite_policy], Pleroma.Web.ActivityPub.MRF.KeywordPolicy) + + params = + File.read!("test/fixtures/mastodon-post-activity.json") + |> Poison.decode!() + + assert Federator.incoming_ap_doc(params) == :error + + Pleroma.Config.put([:instance, :rewrite_policy], policies) + Pleroma.Config.put(:mrf_keyword, mrf_keyword_policy) + end end end diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index 66016c886..e49c4cc22 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -1671,40 +1671,6 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do object = Repo.get(Object, media["id"]) assert object.data["actor"] == User.ap_id(conn.assigns[:user]) end - - test "returns proxied url when media proxy is enabled", %{conn: conn, image: image} do - Pleroma.Config.put([Pleroma.Upload, :base_url], "https://media.pleroma.social") - - proxy_url = "https://cache.pleroma.social" - Pleroma.Config.put([:media_proxy, :enabled], true) - Pleroma.Config.put([:media_proxy, :base_url], proxy_url) - - media = - conn - |> post("/api/v1/media", %{"file" => image}) - |> json_response(:ok) - - assert String.starts_with?(media["url"], proxy_url) - end - - test "returns media url when proxy is enabled but media url is whitelisted", %{ - conn: conn, - image: image - } do - media_url = "https://media.pleroma.social" - Pleroma.Config.put([Pleroma.Upload, :base_url], media_url) - - Pleroma.Config.put([:media_proxy, :enabled], true) - Pleroma.Config.put([:media_proxy, :base_url], "https://cache.pleroma.social") - Pleroma.Config.put([:media_proxy, :whitelist], ["media.pleroma.social"]) - - media = - conn - |> post("/api/v1/media", %{"file" => image}) - |> json_response(:ok) - - assert String.starts_with?(media["url"], media_url) - end end describe "locked accounts" do diff --git a/test/web/mastodon_api/search_controller_test.exs b/test/web/mastodon_api/search_controller_test.exs index 043b96c14..49c79ff0a 100644 --- a/test/web/mastodon_api/search_controller_test.exs +++ b/test/web/mastodon_api/search_controller_test.exs @@ -95,6 +95,18 @@ defmodule Pleroma.Web.MastodonAPI.SearchControllerTest do assert user_three.nickname in result_ids end + + test "returns account if query contains a space", %{conn: conn} do + user = insert(:user, %{nickname: "shp@shitposter.club"}) + + results = + conn + |> assign(:user, user) + |> get("/api/v1/accounts/search", %{"q" => "shp@shitposter.club xxx "}) + |> json_response(200) + + assert length(results) == 1 + end end describe ".search" do diff --git a/test/web/media_proxy/media_proxy_test.exs b/test/web/media_proxy/media_proxy_test.exs index edbbf9b66..0c94755df 100644 --- a/test/web/media_proxy/media_proxy_test.exs +++ b/test/web/media_proxy/media_proxy_test.exs @@ -171,21 +171,6 @@ defmodule Pleroma.Web.MediaProxyTest do encoded = url(url) assert decode_result(encoded) == url end - - test "does not change whitelisted urls" do - upload_config = Pleroma.Config.get([Pleroma.Upload]) - media_url = "https://media.pleroma.social" - Pleroma.Config.put([Pleroma.Upload, :base_url], media_url) - Pleroma.Config.put([:media_proxy, :whitelist], ["media.pleroma.social"]) - Pleroma.Config.put([:media_proxy, :base_url], "https://cache.pleroma.social") - - url = "#{media_url}/static/logo.png" - encoded = url(url) - - assert String.starts_with?(encoded, media_url) - - Pleroma.Config.put([Pleroma.Upload], upload_config) - end end describe "when disabled" do @@ -215,12 +200,43 @@ defmodule Pleroma.Web.MediaProxyTest do decoded end - test "mediaproxy whitelist" do - Pleroma.Config.put([:media_proxy, :enabled], true) - Pleroma.Config.put([:media_proxy, :whitelist], ["google.com", "feld.me"]) - url = "https://feld.me/foo.png" + describe "whitelist" do + setup do + Pleroma.Config.put([:media_proxy, :enabled], true) + :ok + end - unencoded = url(url) - assert unencoded == url + test "mediaproxy whitelist" do + Pleroma.Config.put([:media_proxy, :whitelist], ["google.com", "feld.me"]) + url = "https://feld.me/foo.png" + + unencoded = url(url) + assert unencoded == url + end + + test "does not change whitelisted urls" do + Pleroma.Config.put([:media_proxy, :whitelist], ["mycdn.akamai.com"]) + Pleroma.Config.put([:media_proxy, :base_url], "https://cache.pleroma.social") + + media_url = "https://mycdn.akamai.com" + + url = "#{media_url}/static/logo.png" + encoded = url(url) + + assert String.starts_with?(encoded, media_url) + end + + test "ensure Pleroma.Upload base_url is always whitelisted" do + upload_config = Pleroma.Config.get([Pleroma.Upload]) + media_url = "https://media.pleroma.social" + Pleroma.Config.put([Pleroma.Upload, :base_url], media_url) + + url = "#{media_url}/static/logo.png" + encoded = url(url) + + assert String.starts_with?(encoded, media_url) + + Pleroma.Config.put([Pleroma.Upload], upload_config) + end end end diff --git a/test/web/ostatus/ostatus_test.exs b/test/web/ostatus/ostatus_test.exs index 4e8f3a0fc..803a97695 100644 --- a/test/web/ostatus/ostatus_test.exs +++ b/test/web/ostatus/ostatus_test.exs @@ -199,7 +199,7 @@ defmodule Pleroma.Web.OStatusTest do assert retweeted_activity.data["type"] == "Create" assert retweeted_activity.data["actor"] == user.ap_id assert retweeted_activity.local - assert retweeted_activity.data["object"]["announcement_count"] == 1 + assert Object.normalize(retweeted_activity).data["announcement_count"] == 1 end test "handle incoming retweets - Mastodon, salmon" do @@ -326,6 +326,14 @@ defmodule Pleroma.Web.OStatusTest do assert User.following?(follower, followed) end + test "refuse following over OStatus if the followed's account is locked" do + incoming = File.read!("test/fixtures/follow.xml") + _user = insert(:user, info: %{locked: true}, ap_id: "https://pawoo.net/users/pekorino") + + {:ok, [{:error, "It's not possible to follow locked accounts over OStatus"}]} = + OStatus.handle_incoming(incoming) + end + test "handle incoming unfollows with existing follow" do incoming_follow = File.read!("test/fixtures/follow.xml") {:ok, [_activity]} = OStatus.handle_incoming(incoming_follow) @@ -426,7 +434,7 @@ defmodule Pleroma.Web.OStatusTest do } end - test "find_make_or_update_user takes an author element and returns an updated user" do + test "find_make_or_update_actor takes an author element and returns an updated user" do uri = "https://social.heldscal.la/user/23211" {:ok, user} = OStatus.find_or_make_user(uri) @@ -439,14 +447,56 @@ defmodule Pleroma.Web.OStatusTest do doc = XML.parse_document(File.read!("test/fixtures/23211.atom")) [author] = :xmerl_xpath.string('//author[1]', doc) - {:ok, user} = OStatus.find_make_or_update_user(author) + {:ok, user} = OStatus.find_make_or_update_actor(author) assert user.avatar["type"] == "Image" assert user.name == old_name assert user.bio == old_bio - {:ok, user_again} = OStatus.find_make_or_update_user(author) + {:ok, user_again} = OStatus.find_make_or_update_actor(author) assert user_again == user end + + test "find_or_make_user disallows protocol downgrade" do + user = insert(:user, %{local: true}) + {:ok, user} = OStatus.find_or_make_user(user.ap_id) + + assert User.ap_enabled?(user) + + user = + insert(:user, %{ + ap_id: "https://social.heldscal.la/user/23211", + info: %{ap_enabled: true}, + local: false + }) + + assert User.ap_enabled?(user) + + {:ok, user} = OStatus.find_or_make_user(user.ap_id) + assert User.ap_enabled?(user) + end + + test "find_make_or_update_actor disallows protocol downgrade" do + user = insert(:user, %{local: true}) + {:ok, user} = OStatus.find_or_make_user(user.ap_id) + + assert User.ap_enabled?(user) + + user = + insert(:user, %{ + ap_id: "https://social.heldscal.la/user/23211", + info: %{ap_enabled: true}, + local: false + }) + + assert User.ap_enabled?(user) + + {:ok, user} = OStatus.find_or_make_user(user.ap_id) + assert User.ap_enabled?(user) + + doc = XML.parse_document(File.read!("test/fixtures/23211.atom")) + [author] = :xmerl_xpath.string('//author[1]', doc) + {:error, :invalid_protocol} = OStatus.find_make_or_update_actor(author) + end end describe "gathering user info from a user id" do diff --git a/test/web/rich_media/parsers/twitter_card_test.exs b/test/web/rich_media/parsers/twitter_card_test.exs new file mode 100644 index 000000000..f8e1c9b40 --- /dev/null +++ b/test/web/rich_media/parsers/twitter_card_test.exs @@ -0,0 +1,69 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.RichMedia.Parsers.TwitterCardTest do + use ExUnit.Case, async: true + alias Pleroma.Web.RichMedia.Parsers.TwitterCard + + test "returns error when html not contains twitter card" do + assert TwitterCard.parse("", %{}) == {:error, "No twitter card metadata found"} + end + + test "parses twitter card with only name attributes" do + html = File.read!("test/fixtures/nypd-facial-recognition-children-teenagers3.html") + + assert TwitterCard.parse(html, %{}) == + {:ok, + %{ + "app:id:googleplay": "com.nytimes.android", + "app:name:googleplay": "NYTimes", + "app:url:googleplay": "nytimes://reader/id/100000006583622", + site: nil, + title: + "She Was Arrested at 14. Then Her Photo Went to a Facial Recognition Database. - The New York Times" + }} + end + + test "parses twitter card with only property attributes" do + html = File.read!("test/fixtures/nypd-facial-recognition-children-teenagers2.html") + + assert TwitterCard.parse(html, %{}) == + {:ok, + %{ + card: "summary_large_image", + description: + "With little oversight, the N.Y.P.D. has been using powerful surveillance technology on photos of children and teenagers.", + image: + "https://static01.nyt.com/images/2019/08/01/nyregion/01nypd-juveniles-promo/01nypd-juveniles-promo-videoSixteenByNineJumbo1600.jpg", + "image:alt": "", + title: + "She Was Arrested at 14. Then Her Photo Went to a Facial Recognition Database.", + url: + "https://www.nytimes.com/2019/08/01/nyregion/nypd-facial-recognition-children-teenagers.html" + }} + end + + test "parses twitter card with name & property attributes" do + html = File.read!("test/fixtures/nypd-facial-recognition-children-teenagers.html") + + assert TwitterCard.parse(html, %{}) == + {:ok, + %{ + "app:id:googleplay": "com.nytimes.android", + "app:name:googleplay": "NYTimes", + "app:url:googleplay": "nytimes://reader/id/100000006583622", + card: "summary_large_image", + description: + "With little oversight, the N.Y.P.D. has been using powerful surveillance technology on photos of children and teenagers.", + image: + "https://static01.nyt.com/images/2019/08/01/nyregion/01nypd-juveniles-promo/01nypd-juveniles-promo-videoSixteenByNineJumbo1600.jpg", + "image:alt": "", + site: nil, + title: + "She Was Arrested at 14. Then Her Photo Went to a Facial Recognition Database.", + url: + "https://www.nytimes.com/2019/08/01/nyregion/nypd-facial-recognition-children-teenagers.html" + }} + end +end diff --git a/test/web/twitter_api/util_controller_test.exs b/test/web/twitter_api/util_controller_test.exs index 3d699e1df..640579c09 100644 --- a/test/web/twitter_api/util_controller_test.exs +++ b/test/web/twitter_api/util_controller_test.exs @@ -14,6 +14,17 @@ defmodule Pleroma.Web.TwitterAPI.UtilControllerTest do setup do Tesla.Mock.mock(fn env -> apply(HttpRequestMock, :request, [env]) end) + + instance_config = Pleroma.Config.get([:instance]) + pleroma_fe = Pleroma.Config.get([:frontend_configurations, :pleroma_fe]) + deny_follow_blocked = Pleroma.Config.get([:user, :deny_follow_blocked]) + + on_exit(fn -> + Pleroma.Config.put([:instance], instance_config) + Pleroma.Config.put([:frontend_configurations, :pleroma_fe], pleroma_fe) + Pleroma.Config.put([:user, :deny_follow_blocked], deny_follow_blocked) + end) + :ok end @@ -31,6 +42,35 @@ defmodule Pleroma.Web.TwitterAPI.UtilControllerTest do assert response == "job started" end + test "it imports follow lists from file", %{conn: conn} do + user1 = insert(:user) + user2 = insert(:user) + + with_mocks([ + {File, [], + read!: fn "follow_list.txt" -> + "Account address,Show boosts\n#{user2.ap_id},true" + end}, + {PleromaJobQueue, [:passthrough], []} + ]) do + response = + conn + |> assign(:user, user1) + |> post("/api/pleroma/follow_import", %{"list" => %Plug.Upload{path: "follow_list.txt"}}) + |> json_response(:ok) + + assert called( + PleromaJobQueue.enqueue( + :background, + User, + [:follow_import, user1, [user2.ap_id]] + ) + ) + + assert response == "job started" + end + end + test "it imports new-style mastodon follow lists", %{conn: conn} do user1 = insert(:user) user2 = insert(:user) @@ -79,6 +119,33 @@ defmodule Pleroma.Web.TwitterAPI.UtilControllerTest do assert response == "job started" end + + test "it imports blocks users from file", %{conn: conn} do + user1 = insert(:user) + user2 = insert(:user) + user3 = insert(:user) + + with_mocks([ + {File, [], read!: fn "blocks_list.txt" -> "#{user2.ap_id} #{user3.ap_id}" end}, + {PleromaJobQueue, [:passthrough], []} + ]) do + response = + conn + |> assign(:user, user1) + |> post("/api/pleroma/blocks_import", %{"list" => %Plug.Upload{path: "blocks_list.txt"}}) + |> json_response(:ok) + + assert called( + PleromaJobQueue.enqueue( + :background, + User, + [:blocks_import, user1, [user2.ap_id, user3.ap_id]] + ) + ) + + assert response == "job started" + end + end end describe "POST /api/pleroma/notifications/read" do @@ -98,6 +165,18 @@ defmodule Pleroma.Web.TwitterAPI.UtilControllerTest do assert Repo.get(Notification, notification1.id).seen refute Repo.get(Notification, notification2.id).seen end + + test "it returns error when notification not found", %{conn: conn} do + user1 = insert(:user) + + response = + conn + |> assign(:user, user1) + |> post("/api/pleroma/notifications/read", %{"id" => "22222222222222"}) + |> json_response(403) + + assert response == %{"error" => "Cannot get notification"} + end end describe "PUT /api/pleroma/notification_settings" do @@ -123,7 +202,63 @@ defmodule Pleroma.Web.TwitterAPI.UtilControllerTest do end end - describe "GET /api/statusnet/config.json" do + describe "GET /api/statusnet/config" do + test "it returns config in xml format", %{conn: conn} do + instance = Pleroma.Config.get(:instance) + + response = + conn + |> put_req_header("accept", "application/xml") + |> get("/api/statusnet/config") + |> response(:ok) + + assert response == + "\n\n#{Keyword.get(instance, :name)}\n#{ + Pleroma.Web.base_url() + }\n#{Keyword.get(instance, :limit)}\n#{ + !Keyword.get(instance, :registrations_open) + }\n\n\n" + end + + test "it returns config in json format", %{conn: conn} do + instance = Pleroma.Config.get(:instance) + Pleroma.Config.put([:instance, :managed_config], true) + Pleroma.Config.put([:instance, :registrations_open], false) + Pleroma.Config.put([:instance, :invites_enabled], true) + Pleroma.Config.put([:instance, :public], false) + Pleroma.Config.put([:frontend_configurations, :pleroma_fe], %{theme: "asuka-hospital"}) + + response = + conn + |> put_req_header("accept", "application/json") + |> get("/api/statusnet/config") + |> json_response(:ok) + + expected_data = %{ + "site" => %{ + "accountActivationRequired" => "0", + "closed" => "1", + "description" => Keyword.get(instance, :description), + "invitesEnabled" => "1", + "name" => Keyword.get(instance, :name), + "pleromafe" => %{"theme" => "asuka-hospital"}, + "private" => "1", + "safeDMMentionsEnabled" => "0", + "server" => Pleroma.Web.base_url(), + "textlimit" => to_string(Keyword.get(instance, :limit)), + "uploadlimit" => %{ + "avatarlimit" => to_string(Keyword.get(instance, :avatar_upload_limit)), + "backgroundlimit" => to_string(Keyword.get(instance, :background_upload_limit)), + "bannerlimit" => to_string(Keyword.get(instance, :banner_upload_limit)), + "uploadlimit" => to_string(Keyword.get(instance, :upload_limit)) + }, + "vapidPublicKey" => Keyword.get(Pleroma.Web.Push.vapid_config(), :public_key) + } + } + + assert response == expected_data + end + test "returns the state of safe_dm_mentions flag", %{conn: conn} do option = Pleroma.Config.get([:instance, :safe_dm_mentions]) Pleroma.Config.put([:instance, :safe_dm_mentions], true) @@ -210,7 +345,7 @@ defmodule Pleroma.Web.TwitterAPI.UtilControllerTest do end end - describe "GET /ostatus_subscribe?acct=...." do + describe "GET /ostatus_subscribe - remote_follow/2" do test "adds status to pleroma instance if the `acct` is a status", %{conn: conn} do conn = get( @@ -230,6 +365,172 @@ defmodule Pleroma.Web.TwitterAPI.UtilControllerTest do assert html_response(response, 200) =~ "Log in to follow" end + + test "show follow page if the `acct` is a account link", %{conn: conn} do + user = insert(:user) + + response = + conn + |> assign(:user, user) + |> get("/ostatus_subscribe?acct=https://mastodon.social/users/emelie") + + assert html_response(response, 200) =~ "Remote follow" + end + + test "show follow page with error when user cannot fecth by `acct` link", %{conn: conn} do + user = insert(:user) + + response = + conn + |> assign(:user, user) + |> get("/ostatus_subscribe?acct=https://mastodon.social/users/not_found") + + assert html_response(response, 200) =~ "Error fetching user" + end + end + + describe "POST /ostatus_subscribe - do_remote_follow/2 with assigned user " do + test "follows user", %{conn: conn} do + user = insert(:user) + user2 = insert(:user) + + response = + conn + |> assign(:user, user) + |> post("/ostatus_subscribe", %{"user" => %{"id" => user2.id}}) + |> response(200) + + assert response =~ "Account followed!" + assert user2.follower_address in refresh_record(user).following + end + + test "returns error when user is deactivated", %{conn: conn} do + user = insert(:user, info: %{deactivated: true}) + user2 = insert(:user) + + response = + conn + |> assign(:user, user) + |> post("/ostatus_subscribe", %{"user" => %{"id" => user2.id}}) + |> response(200) + + assert response =~ "Error following account" + end + + test "returns error when user is blocked", %{conn: conn} do + Pleroma.Config.put([:user, :deny_follow_blocked], true) + user = insert(:user) + user2 = insert(:user) + + {:ok, _user} = Pleroma.User.block(user2, user) + + response = + conn + |> assign(:user, user) + |> post("/ostatus_subscribe", %{"user" => %{"id" => user2.id}}) + |> response(200) + + assert response =~ "Error following account" + end + + test "returns error when followee not found", %{conn: conn} do + user = insert(:user) + + response = + conn + |> assign(:user, user) + |> post("/ostatus_subscribe", %{"user" => %{"id" => "jimm"}}) + |> response(200) + + assert response =~ "Error following account" + end + + test "returns success result when user already in followers", %{conn: conn} do + user = insert(:user) + user2 = insert(:user) + {:ok, _, _, _} = CommonAPI.follow(user, user2) + + response = + conn + |> assign(:user, refresh_record(user)) + |> post("/ostatus_subscribe", %{"user" => %{"id" => user2.id}}) + |> response(200) + + assert response =~ "Account followed!" + end + end + + describe "POST /ostatus_subscribe - do_remote_follow/2 without assigned user " do + test "follows", %{conn: conn} do + user = insert(:user) + user2 = insert(:user) + + response = + conn + |> post("/ostatus_subscribe", %{ + "authorization" => %{"name" => user.nickname, "password" => "test", "id" => user2.id} + }) + |> response(200) + + assert response =~ "Account followed!" + assert user2.follower_address in refresh_record(user).following + end + + test "returns error when followee not found", %{conn: conn} do + user = insert(:user) + + response = + conn + |> post("/ostatus_subscribe", %{ + "authorization" => %{"name" => user.nickname, "password" => "test", "id" => "jimm"} + }) + |> response(200) + + assert response =~ "Error following account" + end + + test "returns error when login invalid", %{conn: conn} do + user = insert(:user) + + response = + conn + |> post("/ostatus_subscribe", %{ + "authorization" => %{"name" => "jimm", "password" => "test", "id" => user.id} + }) + |> response(200) + + assert response =~ "Wrong username or password" + end + + test "returns error when password invalid", %{conn: conn} do + user = insert(:user) + user2 = insert(:user) + + response = + conn + |> post("/ostatus_subscribe", %{ + "authorization" => %{"name" => user.nickname, "password" => "42", "id" => user2.id} + }) + |> response(200) + + assert response =~ "Wrong username or password" + end + + test "returns error when user is blocked", %{conn: conn} do + Pleroma.Config.put([:user, :deny_follow_blocked], true) + user = insert(:user) + user2 = insert(:user) + {:ok, _user} = Pleroma.User.block(user2, user) + + response = + conn + |> post("/ostatus_subscribe", %{ + "authorization" => %{"name" => user.nickname, "password" => "test", "id" => user2.id} + }) + |> response(200) + + assert response =~ "Error following account" + end end describe "GET /api/pleroma/healthcheck" do @@ -311,5 +612,104 @@ defmodule Pleroma.Web.TwitterAPI.UtilControllerTest do assert user.info.deactivated == true end + + test "it returns returns when password invalid", %{conn: conn} do + user = insert(:user) + + response = + conn + |> assign(:user, user) + |> post("/api/pleroma/disable_account", %{"password" => "test1"}) + |> json_response(:ok) + + assert response == %{"error" => "Invalid password."} + user = User.get_cached_by_id(user.id) + + refute user.info.deactivated + end + end + + describe "GET /api/statusnet/version" do + test "it returns version in xml format", %{conn: conn} do + response = + conn + |> put_req_header("accept", "application/xml") + |> get("/api/statusnet/version") + |> response(:ok) + + assert response == "#{Pleroma.Application.named_version()}" + end + + test "it returns version in json format", %{conn: conn} do + response = + conn + |> put_req_header("accept", "application/json") + |> get("/api/statusnet/version") + |> json_response(:ok) + + assert response == "#{Pleroma.Application.named_version()}" + end + end + + describe "POST /main/ostatus - remote_subscribe/2" do + test "renders subscribe form", %{conn: conn} do + user = insert(:user) + + response = + conn + |> post("/main/ostatus", %{"nickname" => user.nickname, "profile" => ""}) + |> response(:ok) + + refute response =~ "Could not find user" + assert response =~ "Remotely follow #{user.nickname}" + end + + test "renders subscribe form with error when user not found", %{conn: conn} do + response = + conn + |> post("/main/ostatus", %{"nickname" => "nickname", "profile" => ""}) + |> response(:ok) + + assert response =~ "Could not find user" + refute response =~ "Remotely follow" + end + + test "it redirect to webfinger url", %{conn: conn} do + user = insert(:user) + user2 = insert(:user, ap_id: "shp@social.heldscal.la") + + conn = + conn + |> post("/main/ostatus", %{ + "user" => %{"nickname" => user.nickname, "profile" => user2.ap_id} + }) + + assert redirected_to(conn) == + "https://social.heldscal.la/main/ostatussub?profile=#{user.ap_id}" + end + + test "it renders form with error when use not found", %{conn: conn} do + user2 = insert(:user, ap_id: "shp@social.heldscal.la") + + response = + conn + |> post("/main/ostatus", %{"user" => %{"nickname" => "jimm", "profile" => user2.ap_id}}) + |> response(:ok) + + assert response =~ "Something went wrong." + end + end + + test "it returns new captcha", %{conn: conn} do + with_mock Pleroma.Captcha, + new: fn -> "test_captcha" end do + resp = + conn + |> get("/api/pleroma/captcha") + |> response(200) + + assert resp == "\"test_captcha\"" + assert called(Pleroma.Captcha.new()) + end end end