diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a34c4ec..c454b17 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -55,6 +55,7 @@ jobs: - clean-s3-cache - python-instagram - moa + - nix-s3-dedup runs-on: ubuntu-latest steps: - name: Checkout repository @@ -91,4 +92,4 @@ jobs: nix-store -r $DRV_PATH fi env: - NIXPKGS_ALLOW_UNFREE: 1 \ No newline at end of file + NIXPKGS_ALLOW_UNFREE: 1 diff --git a/default.nix b/default.nix index 6ff7a2e..19806ec 100644 --- a/default.nix +++ b/default.nix @@ -55,4 +55,5 @@ clean-s3-cache = pkgs.python3Packages.callPackage ./scripts/clean-s3-cache.nix {}; python-instagram = pkgs.python3Packages.callPackage ./python/instagram.nix {}; moa = pkgs.python3Packages.callPackage ./moa {}; + nix-s3-dedup = pkgs.callPackage ./nix {}; } diff --git a/nix/dedupNixCache.patch b/nix/dedupNixCache.patch new file mode 100644 index 0000000..3994ed0 --- /dev/null +++ b/nix/dedupNixCache.patch @@ -0,0 +1,54 @@ +diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc +index 844553ad3..a7a9cd702 100644 +--- a/src/libstore/s3-binary-cache-store.cc ++++ b/src/libstore/s3-binary-cache-store.cc +@@ -214,6 +214,14 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual + Stats stats; + + S3Helper s3Helper; ++ ref mNixosCache; ++ ++ BinaryCacheStore * nixosCache() { ++ auto * casted = dynamic_cast(&*this->mNixosCache); ++ if (!casted) ++ throw UsageError("cache.nixos.org is not a BinaryCacheStore"); ++ return casted; ++ } + + S3BinaryCacheStoreImpl( + const std::string & uriScheme, +@@ -227,6 +235,7 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual + , S3BinaryCacheStore(params) + , bucketName(bucketName) + , s3Helper(profile, region, scheme, endpoint) ++ , mNixosCache(openStore("https://cache.nixos.org/")) + { + diskCache = getNarInfoDiskCache(); + } +@@ -268,6 +277,8 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual + + bool fileExists(const std::string & path) override + { ++ if(this->nixosCache()->fileExists(path)) ++ return true; + stats.head++; + + auto res = s3Helper.client->HeadObject( +@@ -389,6 +400,8 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual + std::shared_ptr> istream, + const std::string & mimeType) override + { ++ if(this->nixosCache()->fileExists(path)) ++ return; + auto compress = [&](std::string compression) + { + auto compressed = nix::compress(compression, StreamToSourceAdapter(istream).drain()); +@@ -407,6 +420,8 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual + + void getFile(const std::string & path, Sink & sink) override + { ++ if(this->nixosCache()->fileExists(path)) ++ return this->nixosCache()->getFile(path, sink); + stats.get++; + + // FIXME: stream output to sink. diff --git a/nix/default.nix b/nix/default.nix new file mode 100644 index 0000000..7ee0351 --- /dev/null +++ b/nix/default.nix @@ -0,0 +1,8 @@ +{nix}: +nix.overrideAttrs (super: { + patches = + (super.patches or []) + ++ [ + ./dedupNixCache.patch + ]; +})