Merge branch 'develop' into 'develop'

Dedupe uploads

See merge request pleroma/pleroma!115

lib/mix/tasks/sample_config.eex

@@ -8,7 +8,8 @@ config :pleroma, :instance,
   name: "<%= name %>",
   email: "<%= email %>",
   limit: 5000,
-  registrations_open: true
+  registrations_open: true,
+  dedupe_media: false
 
 config :pleroma, :media_proxy,
   enabled: false,

lib/pleroma/upload.ex

@@ -2,20 +2,21 @@ defmodule Pleroma.Upload do
   alias Ecto.UUID
   alias Pleroma.Web
 
-  def store(%Plug.Upload{} = file) do
+  def store(%Plug.Upload{} = file, should_dedupe) do
-    uuid = UUID.generate()
+    content_type = get_content_type(file.path)
-    upload_folder = Path.join(upload_path(), uuid)
+    uuid = get_uuid(file, should_dedupe)
-    File.mkdir_p!(upload_folder)
+    name = get_name(file, uuid, content_type, should_dedupe)
-    result_file = Path.join(upload_folder, file.filename)
+    upload_folder = get_upload_path(uuid, should_dedupe)
-    File.cp!(file.path, result_file)
+    url_path = get_url(name, uuid, should_dedupe)
 
-    # fix content type on some image uploads
+    File.mkdir_p!(upload_folder)
-    content_type =
+    result_file = Path.join(upload_folder, name)
-      if file.content_type in [nil, "application/octet-stream"] do
+
-        get_content_type(file.path)
+    if File.exists?(result_file) do
-      else
+      File.rm!(file.path)
-        file.content_type
+    else
-      end
+      File.cp!(file.path, result_file)
+    end
 
     %{
       "type" => "Image",
@@ -23,26 +24,48 @@ defmodule Pleroma.Upload do
         %{
           "type" => "Link",
           "mediaType" => content_type,
-          "href" => url_for(Path.join(uuid, :cow_uri.urlencode(file.filename)))
+          "href" => url_path
         }
       ],
-      "name" => file.filename,
+      "name" => name
-      "uuid" => uuid
     }
   end
 
-  def store(%{"img" => "data:image/" <> image_data}) do
+  def store(%{"img" => "data:image/" <> image_data}, should_dedupe) do
     parsed = Regex.named_captures(~r/(?<filetype>jpeg|png|gif);base64,(?<data>.*)/, image_data)
-    data = Base.decode64!(parsed["data"])
+    data = Base.decode64!(parsed["data"], ignore: :whitespace)
     uuid = UUID.generate()
-    upload_folder = Path.join(upload_path(), uuid)
+    uuidpath = Path.join(upload_path(), uuid)
+    uuid = UUID.generate()
+
+    File.mkdir_p!(upload_path())
+
+    File.write!(uuidpath, data)
+
+    content_type = get_content_type(uuidpath)
+
+    name =
+      create_name(
+        String.downcase(Base.encode16(:crypto.hash(:sha256, data))),
+        parsed["filetype"],
+        content_type
+      )
+
+    upload_folder = get_upload_path(uuid, should_dedupe)
+    url_path = get_url(name, uuid, should_dedupe)
+
     File.mkdir_p!(upload_folder)
-    filename = Base.encode16(:crypto.hash(:sha256, data)) <> ".#{parsed["filetype"]}"
+    result_file = Path.join(upload_folder, name)
-    result_file = Path.join(upload_folder, filename)
 
-    File.write!(result_file, data)
+    if should_dedupe do
-
+      if !File.exists?(result_file) do
-    content_type = "image/#{parsed["filetype"]}"
+        File.rename(uuidpath, result_file)
+      else
+        File.rm!(uuidpath)
+      end
+    else
+      File.rename(uuidpath, result_file)
+    end
 
     %{
       "type" => "Image",
@@ -50,11 +73,10 @@ defmodule Pleroma.Upload do
         %{
           "type" => "Link",
           "mediaType" => content_type,
-          "href" => url_for(Path.join(uuid, :cow_uri.urlencode(filename)))
+          "href" => url_path
         }
       ],
-      "name" => filename,
+      "name" => name
-      "uuid" => uuid
     }
   end
 
@@ -63,6 +85,46 @@ defmodule Pleroma.Upload do
     Keyword.fetch!(settings, :uploads)
   end
 
+  defp create_name(uuid, ext, type) do
+    if type == "application/octet-stream" do
+      String.downcase(Enum.join([uuid, ext], "."))
+    else
+      String.downcase(Enum.join([uuid, List.last(String.split(type, "/"))], "."))
+    end
+  end
+
+  defp get_uuid(file, should_dedupe) do
+    if should_dedupe do
+      Base.encode16(:crypto.hash(:sha256, File.read!(file.path)))
+    else
+      UUID.generate()
+    end
+  end
+
+  defp get_name(file, uuid, type, should_dedupe) do
+    if should_dedupe do
+      create_name(uuid, List.last(String.split(file.filename, ".")), type)
+    else
+      file.filename
+    end
+  end
+
+  defp get_upload_path(uuid, should_dedupe) do
+    if should_dedupe do
+      upload_path()
+    else
+      Path.join(upload_path(), uuid)
+    end
+  end
+
+  defp get_url(name, uuid, should_dedupe) do
+    if should_dedupe do
+      url_for(:cow_uri.urlencode(name))
+    else
+      url_for(Path.join(uuid, :cow_uri.urlencode(name)))
+    end
+  end
+
   defp url_for(file) do
     "#{Web.base_url()}/media/#{file}"
   end

lib/pleroma/web/activity_pub/activity_pub.ex

@@ -496,7 +496,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
   end
 
   def upload(file) do
-    data = Upload.store(file)
+    data = Upload.store(file, Application.get_env(:pleroma, :instance)[:dedupe_media])
     Repo.insert(%Object{data: data})
   end
 

test/upload_test.exs

@@ -3,40 +3,45 @@ defmodule Pleroma.UploadTest do
   use Pleroma.DataCase
 
   describe "Storing a file" do
-    test "copies the file to the configured folder" do
+    test "copies the file to the configured folder with deduping" do
+      File.cp!("test/fixtures/image.jpg", "test/fixtures/image_tmp.jpg")
+
       file = %Plug.Upload{
         content_type: "image/jpg",
-        path: Path.absname("test/fixtures/image.jpg"),
+        path: Path.absname("test/fixtures/image_tmp.jpg"),
         filename: "an [image.jpg"
       }
 
-      data = Upload.store(file)
+      data = Upload.store(file, true)
-      assert data["name"] == "an [image.jpg"
 
-      assert List.first(data["url"])["href"] ==
+      assert data["name"] ==
-               "http://localhost:4001/media/#{data["uuid"]}/an%20%5Bimage.jpg"
+               "e7a6d0cf595bff76f14c9a98b6c199539559e8b844e02e51e5efcfd1f614a2df.jpeg"
     end
 
-    test "fixes an incorrect content type" do
+    test "copies the file to the configured folder without deduping" do
+      File.cp!("test/fixtures/image.jpg", "test/fixtures/image_tmp.jpg")
+
+      file = %Plug.Upload{
+        content_type: "image/jpg",
+        path: Path.absname("test/fixtures/image_tmp.jpg"),
+        filename: "an [image.jpg"
+      }
+
+      data = Upload.store(file, false)
+      assert data["name"] == "an [image.jpg"
+    end
+
+    test "fixes incorrect content type" do
+      File.cp!("test/fixtures/image.jpg", "test/fixtures/image_tmp.jpg")
+
       file = %Plug.Upload{
         content_type: "application/octet-stream",
-        path: Path.absname("test/fixtures/image.jpg"),
+        path: Path.absname("test/fixtures/image_tmp.jpg"),
         filename: "an [image.jpg"
       }
 
-      data = Upload.store(file)
+      data = Upload.store(file, true)
       assert hd(data["url"])["mediaType"] == "image/jpeg"
     end
-
-    test "does not modify a valid content type" do
-      file = %Plug.Upload{
-        content_type: "image/png",
-        path: Path.absname("test/fixtures/image.jpg"),
-        filename: "an [image.jpg"
-      }
-
-      data = Upload.store(file)
-      assert hd(data["url"])["mediaType"] == "image/png"
-    end
   end
 end