Store gem name, version, platform, and surrogate-key in S3 metadata (#5568)

This way, we can reliably determine what gem was downloaded from only the
request path + response headers.

I added the `surrogate-key` so yanks/restores will properly purge the
cache.

This also adds a new maintenance task to backfill the metadata for all
existing gems in S3, so going forward rubytogether/kirby can assume
the headers are present.

As a follow-up, the Fastly logging will need to be updated to include
these three new headers.

Signed-off-by: Samuel Giddins <segiddins@segiddins.me>

Author: segiddins

app/models/pusher.rb

@@ -308,7 +308,11 @@ def write_gem(body, spec_contents)
     spec_path = "quick/Marshal.4.8/#{@version.full_name}.gemspec.rz"
 
     # do all processing _before_ we upload anything to S3, so we lower the chances of orphaned files
-    RubygemFs.instance.store(gem_path, gem_contents, checksum_sha256: version.sha256)
+    RubygemFs.instance.store(gem_path, gem_contents, checksum_sha256: version.sha256,
+                             metadata: {
+                               "gem" => version.rubygem.name, "version" => version.number, "platform" => version.platform,
+                               "surrogate-key" => "gem/#{version.rubygem.name}", "sha256" => version.sha256
+                             })
     RubygemFs.instance.store(spec_path, spec_contents, checksum_sha256: version.spec_sha256)
 
     Fastly.purge(path: gem_path)

app/tasks/maintenance/backfill_gem_s3_metadata_task.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+class Maintenance::BackfillGemS3MetadataTask < MaintenanceTasks::Task
+  include SemanticLogger::Loggable
+
+  def collection
+    Version.indexed.includes(:rubygem)
+  end
+
+  def process(version)
+    sha256 = version.sha256
+
+    gem_path = "gems/#{version.gem_file_name}"
+    gem_contents, response = RubygemFs.instance.get_object(gem_path)
+
+    if gem_contents.nil?
+      logger.error("Version #{version.full_name} has no gem contents")
+      return
+    end
+
+    actual_sha256 = Digest::SHA256.base64digest(gem_contents)
+    # Validate the stored content matches the expected checksum
+    if actual_sha256 != sha256
+      logger.error("Version #{version.full_name} has sha256 mismatch", expected: sha256, actual: actual_sha256)
+      return
+    end
+
+    existing_metadata = response[:metadata]
+    new_metadata = {
+      "gem" => version.rubygem.name, "version" => version.number, "platform" => version.platform,
+      "surrogate-key" => "gem/#{version.rubygem.name}", "sha256" => sha256
+    }
+
+    if existing_metadata == new_metadata
+      # No changes needed
+    elsif existing_metadata <= new_metadata
+      logger.info("Updating metadata for #{version.full_name}", existing_metadata: existing_metadata, new_metadata: new_metadata)
+      RubygemFs.instance.store(gem_path, gem_contents, checksum_sha256: sha256, metadata: new_metadata)
+    else
+      logger.error("Version #{version.full_name} has unexpected metadata", existing_metadata:, new_metadata:)
+    end
+  end
+end

lib/rubygem_fs.rb

@@ -82,6 +82,12 @@ def get(key)
       nil
     end
 
+    def get_object(key)
+      body = get(key)
+      return unless body
+      [body, head(key)]
+    end
+
     def each_key(prefix: nil, &)
       return enum_for(__method__, prefix:) unless block_given?
       base = dir_for(prefix)
@@ -186,6 +192,13 @@ def get(key)
       nil
     end
 
+    def get_object(key)
+      response = s3.get_object(key: key, bucket: bucket)
+      [response.body.read, response.to_h]
+    rescue Aws::S3::Errors::NoSuchKey
+      nil
+    end
+
     def each_key(prefix: nil, &)
       return enum_for(__method__, prefix: prefix) unless block_given?
       s3.list_objects_v2(bucket: bucket, prefix: prefix).each do |response|

test/integration/push_test.rb

@@ -106,7 +106,11 @@ class PushTest < ActionDispatch::IntegrationTest
 
     refute_nil RubygemFs.instance.get("gems/sandworm-2.0.0.gem")
     refute_nil RubygemFs.instance.get("quick/Marshal.4.8/sandworm-2.0.0.gemspec.rz")
-    assert_equal({ checksum_sha256: rubygem.versions.find_by!(full_name: "sandworm-2.0.0").sha256, key: "gems/sandworm-2.0.0.gem" },
+    checksum_sha256 = rubygem.versions.find_by!(full_name: "sandworm-2.0.0").sha256
+
+    assert_equal({ checksum_sha256:, key: "gems/sandworm-2.0.0.gem",
+                   metadata: { "gem" => "sandworm", "version" => "2.0.0", "platform" => "ruby",
+                               "surrogate-key" => "gem/sandworm", "sha256" => checksum_sha256 } },
                  RubygemFs.instance.head("gems/sandworm-2.0.0.gem"))
 
     spec = Gem::Package.new("sandworm-2.0.0.gem").spec

test/tasks/maintenance/backfill_gem_s3_metadata_task_test.rb

@@ -0,0 +1,161 @@
+# frozen_string_literal: true
+
+require "test_helper"
+
+# test "#process performs a task iteration" do
+#   Maintenance::BackfillGemS3MetadataTask.process(element)
+# end
+class Maintenance::BackfillGemS3MetadataTaskTest < ActiveSupport::TestCase
+  include SemanticLogger::Test::Minitest
+
+  make_my_diffs_pretty!
+
+  test "#collection returns a collection of indexed versions" do
+    create(:version, indexed: false)
+    v = create(:version, indexed: true)
+
+    assert_equal [v], Maintenance::BackfillGemS3MetadataTask.collection.to_a
+  end
+
+  test "#process makes no changes for newly pushed gems" do
+    @gem = gem_file("bin_and_img-0.1.0.gem")
+    @user = create(:user)
+    pusher = Pusher.new(create(:api_key, owner: @user), @gem)
+
+    assert pusher.process, "gem should be pushed successfully: #{pusher.code} #{pusher.message}"
+
+    assert_no_changes -> { [pusher.version.reload.updated_at, RubygemFs.instance.get_object("gems/bin_and_img-0.1.0.gem")] } do
+      Maintenance::BackfillGemS3MetadataTask.process(pusher.version)
+    end
+  end
+
+  test "#process logs missing gem contents" do
+    logger = SemanticLogger::Test::CaptureLogEvents.new
+    Maintenance::BackfillGemS3MetadataTask.stubs(:logger).returns(logger)
+
+    @gem = gem_file("bin_and_img-0.1.0.gem")
+    @user = create(:user)
+    pusher = Pusher.new(create(:api_key, owner: @user), @gem)
+
+    assert pusher.process, "gem should be pushed successfully: #{pusher.code} #{pusher.message}"
+
+    # remove metadata from the fs
+    RubygemFs.instance.remove("gems/bin_and_img-0.1.0.gem")
+
+    assert_no_changes -> { [pusher.version.reload.updated_at, RubygemFs.instance.get_object("gems/bin_and_img-0.1.0.gem")] } do
+      Maintenance::BackfillGemS3MetadataTask.process(pusher.version)
+    end
+
+    assert_semantic_logger_event(
+      logger.events[0],
+        level:            :error,
+        message_includes: "Version bin_and_img-0.1.0 has no gem contents"
+    )
+    assert_equal 1, logger.events.size
+  end
+
+  test "#process logs sha256 mismatch" do
+    logger = SemanticLogger::Test::CaptureLogEvents.new
+    Maintenance::BackfillGemS3MetadataTask.stubs(:logger).returns(logger)
+
+    @gem = gem_file("bin_and_img-0.1.0.gem")
+    @user = create(:user)
+    pusher = Pusher.new(create(:api_key, owner: @user), @gem)
+
+    assert pusher.process, "gem should be pushed successfully: #{pusher.code} #{pusher.message}"
+
+    # remove metadata from the fs
+    RubygemFs.instance.remove("gems/bin_and_img-0.1.0.gem")
+    RubygemFs.instance.store("gems/bin_and_img-0.1.0.gem", "contents")
+
+    assert_no_changes -> { [pusher.version.reload.updated_at, RubygemFs.instance.get_object("gems/bin_and_img-0.1.0.gem")] } do
+      Maintenance::BackfillGemS3MetadataTask.process(pusher.version)
+    end
+
+    assert_semantic_logger_event(
+      logger.events[0],
+        level:            :error,
+        message_includes: "Version bin_and_img-0.1.0 has sha256 mismatch"
+    )
+    assert_equal 1, logger.events.size
+  end
+
+  test "#process logs unexpected metadata" do
+    logger = SemanticLogger::Test::CaptureLogEvents.new
+    Maintenance::BackfillGemS3MetadataTask.stubs(:logger).returns(logger)
+
+    @gem = gem_file("bin_and_img-0.1.0.gem")
+    @user = create(:user)
+    pusher = Pusher.new(create(:api_key, owner: @user), @gem)
+
+    assert pusher.process, "gem should be pushed successfully: #{pusher.code} #{pusher.message}"
+
+    # remove metadata from the fs
+    body, response = RubygemFs.instance.get_object("gems/bin_and_img-0.1.0.gem")
+    RubygemFs.instance.store("gems/bin_and_img-0.1.0.gem", body, metadata: response[:metadata].merge("unexpected" => "value"))
+
+    assert_no_changes -> { [pusher.version.reload.updated_at, RubygemFs.instance.get_object("gems/bin_and_img-0.1.0.gem")] } do
+      Maintenance::BackfillGemS3MetadataTask.process(pusher.version)
+    end
+
+    assert_semantic_logger_event(
+      logger.events[0],
+        level:            :error,
+        message_includes: "Version bin_and_img-0.1.0 has unexpected metadata"
+    )
+    assert_equal 1, logger.events.size
+  end
+
+  test "#process logs conflicting metadata" do
+    logger = SemanticLogger::Test::CaptureLogEvents.new
+    Maintenance::BackfillGemS3MetadataTask.stubs(:logger).returns(logger)
+
+    @gem = gem_file("bin_and_img-0.1.0.gem")
+    @user = create(:user)
+    pusher = Pusher.new(create(:api_key, owner: @user), @gem)
+
+    assert pusher.process, "gem should be pushed successfully: #{pusher.code} #{pusher.message}"
+
+    # remove metadata from the fs
+    body, response = RubygemFs.instance.get_object("gems/bin_and_img-0.1.0.gem")
+    RubygemFs.instance.store("gems/bin_and_img-0.1.0.gem", body, metadata: response[:metadata].merge("gem" => "not_bin_and_img"))
+
+    assert_no_changes -> { [pusher.version.reload.updated_at, RubygemFs.instance.get_object("gems/bin_and_img-0.1.0.gem")] } do
+      Maintenance::BackfillGemS3MetadataTask.process(pusher.version)
+    end
+
+    assert_semantic_logger_event(
+      logger.events[0],
+        level:            :error,
+        message_includes: "Version bin_and_img-0.1.0 has unexpected metadata"
+    )
+    assert_equal 1, logger.events.size
+  end
+
+  test "#process updates metadata" do
+    @gem = gem_file("bin_and_img-0.1.0.gem")
+    @user = create(:user)
+    pusher = Pusher.new(create(:api_key, owner: @user), @gem)
+
+    assert pusher.process, "gem should be pushed successfully: #{pusher.code} #{pusher.message}"
+
+    # remove metadata from the fs
+    body, = RubygemFs.instance.get_object("gems/bin_and_img-0.1.0.gem")
+    RubygemFs.instance.store("gems/bin_and_img-0.1.0.gem", body, metadata: {})
+
+    assert_no_changes -> { [pusher.version.reload.updated_at] } do
+      Maintenance::BackfillGemS3MetadataTask.process(pusher.version)
+    end
+
+    assert_equal [
+      @gem.tap(&:rewind).read,
+      checksum_sha256: pusher.version.sha256,
+      metadata: { "gem" => "bin_and_img",
+                  "version" => "0.1.0",
+                  "platform" => "ruby",
+                  "surrogate-key" => "gem/bin_and_img",
+                  "sha256" => pusher.version.sha256 },
+      key: "gems/bin_and_img-0.1.0.gem"
+    ], RubygemFs.instance.get_object("gems/bin_and_img-0.1.0.gem")
+  end
+end