Vendor compact index into rubygems.org

Vendor the compact index API into rubygems.org so we don't need to have
a dependency on releasing it to update the compact index API. It doesn't
make sense to have this when rubygems.org is the consumer and is where
the spec is published.

Author: eileencodes

.rubocop.yml

@@ -160,6 +160,7 @@ Minitest/MultipleAssertions:
 Style/ClassAndModuleChildren:
   EnforcedStyle: compact
   Exclude:
+    - lib/compact_index/*.rb
     - lib/gemcutter/middleware/hostess.rb
     - lib/gemcutter/middleware/redirector.rb
 

Gemfile

@@ -49,7 +49,6 @@ gem "opensearch-ruby", "~> 3.4"
 gem "searchkick", "~> 6.1"
 gem "faraday_middleware-aws-sigv4", "~> 1.0"
 gem "xml-simple", "~> 1.1"
-gem "compact_index", "~> 0.15.0"
 gem "rack-attack", "~> 6.8"
 gem "rqrcode", "~> 3.2"
 gem "rotp", "~> 6.2"

Gemfile.lock

@@ -200,7 +200,6 @@ GEM
       bcrypt (>= 3.1.1)
       email_validator (~> 2.0)
       railties (>= 5.0)
-    compact_index (0.15.0)
     concurrent-ruby (1.3.6)
     connection_pool (3.0.2)
     cose (1.3.1)
@@ -941,7 +940,6 @@ DEPENDENCIES
   capybara (~> 3.40)
   chartkick (~> 5.2)
   clearance (~> 2.11)
-  compact_index (~> 0.15.0)
   csv (~> 3.3)
   dalli (~> 3.2)
   datadog (~> 2.30)
@@ -1104,7 +1102,6 @@ CHECKSUMS
   choice (0.2.0) sha256=a19617f7dfd4921b38a85d0616446620de685a113ec6d1ecc85bdb67bf38c974
   chunky_png (1.4.0) sha256=89d5b31b55c0cf4da3cf89a2b4ebc3178d8abe8cbaf116a1dba95668502fdcfe
   clearance (2.11.0) sha256=c198711033c095cf127ea076f6afd4b52723548992b5d40bc1d2e3a28e9b0c79
-  compact_index (0.15.0) sha256=5c6c404afca8928a7d9f4dde9524f6e1610db17e675330803055db282da84a8b
   concurrent-ruby (1.3.6) sha256=6b56837e1e7e5292f9864f34b69c5a2cbc75c0cf5338f1ce9903d10fa762d5ab
   connection_pool (3.0.2) sha256=33fff5ba71a12d2aa26cb72b1db8bba2a1a01823559fb01d29eb74c286e62e0a
   cose (1.3.1) sha256=d5d4dbcd6b035d513edc4e1ab9bc10e9ce13b4011c96e3d1b8fe5e6413fd6de5

config/application.rb

@@ -33,7 +33,7 @@ class Application < Rails::Application
     # Please, add to the `ignore` list any other `lib` subdirectories that do
     # not contain `.rb` files, or that should not be reloaded or eager loaded.
     # Common ones are `templates`, `generators`, or `middleware`, for example.
-    config.autoload_lib(ignore: %w[assets tasks cops shoryuken])
+    config.autoload_lib(ignore: %w[assets tasks cops shoryuken compact_index])
 
     # Settings in config/environments/* take precedence over those specified here.
     # Application configuration can go into files in config/initializers

lib/compact_index.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+# Vendored from https://github.com/rubygems/compact_index v0.15.0
+
+require_relative "compact_index/gem"
+require_relative "compact_index/gem_version"
+require_relative "compact_index/dependency"
+
+require_relative "compact_index/versions_file"
+
+module CompactIndex
+  def self.names(gem_names)
+    gem_names.join("\n").prepend("---\n") << "\n"
+  end
+
+  def self.versions(versions_file, gems = nil, args = {})
+    versions_file.contents(gems, args)
+  end
+
+  def self.info(versions)
+    versions.inject(+"---\n") do |output, version|
+      output << version.to_line << "\n"
+    end
+  end
+end

lib/compact_index/dependency.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module CompactIndex
+  Dependency = Struct.new(:gem, :version, :platform, :checksum) do
+    def version_and_platform
+      if platform.nil? || platform == "ruby"
+        version
+      else
+        "#{version}-#{platform}"
+      end
+    end
+  end
+end

lib/compact_index/gem.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module CompactIndex
+  Gem = Struct.new(:name, :versions) do
+    def <=>(other)
+      name <=> other.name
+    end
+  end
+end

lib/compact_index/gem_version.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module CompactIndex
+  GemVersion = Struct.new(:number, :platform, :checksum, :info_checksum,
+                          :dependencies, :ruby_version, :rubygems_version) do
+    def number_and_platform
+      if platform.nil? || platform == "ruby"
+        number
+      else
+        "#{number}-#{platform}"
+      end
+    end
+
+    def <=>(other)
+      number_comp = number <=> other.number
+
+      if number_comp.zero?
+        [number, platform].compact <=> [other.number, other.platform].compact
+      else
+        number_comp
+      end
+    end
+
+    def to_line
+      line = "#{number_and_platform} #{deps_line}|checksum:#{checksum}"
+      line << ",ruby:#{ruby_version_line}" if ruby_version && ruby_version != ">= 0"
+      line << ",rubygems:#{rubygems_version_line}" if rubygems_version && rubygems_version != ">= 0"
+      line
+    end
+
+    private
+
+    def ruby_version_line
+      join_multiple(ruby_version)
+    end
+
+    def rubygems_version_line
+      join_multiple(rubygems_version)
+    end
+
+    def deps_line
+      return "" if dependencies.nil?
+
+      dependencies.map do |d|
+        [d[:gem], join_multiple(d.version_and_platform)].join(":")
+      end.join(",")
+    end
+
+    def join_multiple(requirements)
+      requirements = requirements.split(", ")
+      requirements.sort!
+      requirements.join("&")
+    end
+  end
+end

lib/compact_index/versions_file.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+require "time"
+require "date"
+require "digest"
+
+module CompactIndex
+  class VersionsFile
+    def initialize(file = nil)
+      @path = file || "/versions.list"
+    end
+
+    def contents(gems = nil, args = {})
+      gems = calculate_info_checksums(gems) if args.delete(:calculate_info_checksums) { false }
+
+      raise ArgumentError, "Unknown options: #{args.keys.join(', ')}" unless args.empty?
+
+      File.read(@path).tap do |out|
+        out << gem_lines(gems) if gems
+      end
+    end
+
+    def updated_at
+      created_at_header(@path) || Time.at(0).utc.to_datetime
+    end
+
+    def create(gems, timestamp = Time.now.iso8601)
+      gems.sort!
+
+      File.open(@path, "w") do |io|
+        io.write "created_at: #{timestamp}\n---\n"
+        io.write gem_lines(gems)
+      end
+    end
+
+    private
+
+    def gem_lines(gems)
+      gems.reduce(+"") do |lines, gem|
+        version_numbers = gem.versions.map(&:number_and_platform).join(",")
+        lines << gem.name <<
+          " " << version_numbers <<
+          " #{gem.versions.last.info_checksum}\n"
+      end
+    end
+
+    def calculate_info_checksums(gems)
+      gems.each do |gem|
+        info_checksum = Digest::MD5.hexdigest(CompactIndex.info(gem[:versions]))
+        gem[:versions].last[:info_checksum] = info_checksum
+      end
+    end
+
+    def created_at_header(path)
+      return unless File.exist? path
+
+      File.open(path) do |file|
+        file.each_line do |line|
+          line.match(/created_at: (.*)\n|---\n/) do |match|
+            return match[1] && DateTime.parse(match[1])
+          end
+        end
+      end
+
+      nil
+    end
+  end
+end

test/unit/compact_index_test.rb

@@ -0,0 +1,138 @@
+# frozen_string_literal: true
+
+require "test_helper"
+require "compact_index"
+
+class CompactIndexTest < ActiveSupport::TestCase
+  class NamesTest < CompactIndexTest
+    test "returns the gem list for one gem name" do
+      assert_equal "---\ngem\n", CompactIndex.names(["gem"])
+    end
+
+    test "returns the gem list for multiple gem names" do
+      assert_equal "---\ngem-1\ngem_2\n", CompactIndex.names(%w[gem-1 gem_2])
+    end
+  end
+
+  class VersionsTest < CompactIndexTest
+    test "delegates to VersionsFile#contents" do
+      file = Tempfile.new("versions-endpoint")
+      versions_file = CompactIndex::VersionsFile.new(file.path)
+      gems = [CompactIndex::Gem.new("test", [build_version])]
+
+      assert_equal versions_file.contents(gems), CompactIndex.versions(versions_file, gems)
+    ensure
+      file&.unlink
+    end
+  end
+
+  class InfoTest < CompactIndexTest
+    test "without dependencies" do
+      param = [build_version(number: "1.0.1")]
+
+      assert_equal "---\n1.0.1 |checksum:sum+test_gem+1.0.1\n", CompactIndex.info(param)
+    end
+
+    test "multiple versions" do
+      param = [
+        build_version(number: "1.0.1", checksum: "abc1"),
+        build_version(number: "1.0.2", checksum: "abc2")
+      ]
+
+      assert_equal "---\n1.0.1 |checksum:abc1\n1.0.2 |checksum:abc2\n", CompactIndex.info(param)
+    end
+
+    test "one dependency" do
+      deps = [CompactIndex::Dependency.new("foo", "=1.0.1", "ruby", "abc123")]
+      param = [build_version(number: "1.0.1", dependencies: deps)]
+
+      assert_equal "---\n1.0.1 foo:=1.0.1|checksum:sum+test_gem+1.0.1\n", CompactIndex.info(param)
+    end
+
+    test "multiple dependencies" do
+      deps = [
+        CompactIndex::Dependency.new("foo1", "=1.0.1", "ruby", "abc123"),
+        CompactIndex::Dependency.new("foo2", "<2.0", "ruby", "abc123")
+      ]
+      param = [build_version(number: "1.0.1", dependencies: deps)]
+
+      assert_equal "---\n1.0.1 foo1:=1.0.1,foo2:<2.0|checksum:sum+test_gem+1.0.1\n", CompactIndex.info(param)
+    end
+
+    test "dependency with multiple versions" do
+      deps = [CompactIndex::Dependency.new("foo", "<2.0, >1.0", "ruby", "abc123")]
+      param = [build_version(number: "1.0.1", dependencies: deps)]
+
+      assert_equal "---\n1.0.1 foo:<2.0&>1.0|checksum:sum+test_gem+1.0.1\n", CompactIndex.info(param)
+    end
+
+    test "sorts the requirements" do
+      deps = [CompactIndex::Dependency.new("foo", ">1.0, <2.0", "ruby", "abc123")]
+      param = [build_version(number: "1.0.1", dependencies: deps)]
+
+      assert_equal "---\n1.0.1 foo:<2.0&>1.0|checksum:sum+test_gem+1.0.1\n", CompactIndex.info(param)
+    end
+
+    test "dependencies have platform" do
+      deps = [
+        CompactIndex::Dependency.new("a", "=1.1", "jruby", "abc123"),
+        CompactIndex::Dependency.new("b", "= 1.2", "darwin-13", "abc123")
+      ]
+      param = [build_version(number: "1.0.1", dependencies: deps)]
+
+      assert_equal "---\n1.0.1 a:=1.1-jruby,b:= 1.2-darwin-13|checksum:sum+test_gem+1.0.1\n", CompactIndex.info(param)
+    end
+
+    test "show ruby required version" do
+      param = [build_version(number: "1.0.1", ruby_version: ">1.8")]
+
+      assert_equal "---\n1.0.1 |checksum:sum+test_gem+1.0.1,ruby:>1.8\n", CompactIndex.info(param)
+    end
+
+    test "show ruby required version with multiple requirements" do
+      param = [build_version(number: "1.0.1", ruby_version: "< 2.5, >=2.2")]
+
+      assert_equal "---\n1.0.1 |checksum:sum+test_gem+1.0.1,ruby:< 2.5&>=2.2\n", CompactIndex.info(param)
+    end
+
+    test "show rubygems required version" do
+      param = [build_version(number: "1.0.1", rubygems_version: "=2.0")]
+
+      assert_equal "---\n1.0.1 |checksum:sum+test_gem+1.0.1,rubygems:=2.0\n", CompactIndex.info(param)
+    end
+
+    test "show rubygems required version with multiple requirements" do
+      param = [build_version(number: "1.0.1", rubygems_version: ">2.0, <3.1")]
+
+      assert_equal "---\n1.0.1 |checksum:sum+test_gem+1.0.1,rubygems:<3.1&>2.0\n", CompactIndex.info(param)
+    end
+
+    test "show both rubygems and ruby required versions" do
+      param = [build_version(number: "1.0.1", ruby_version: ">1.9", rubygems_version: ">2.0")]
+
+      assert_equal "---\n1.0.1 |checksum:sum+test_gem+1.0.1,ruby:>1.9,rubygems:>2.0\n", CompactIndex.info(param)
+    end
+
+    test "adds platform next to version number" do
+      param = [build_version(number: "1.0.1", platform: "jruby")]
+
+      assert_equal "---\n1.0.1-jruby |checksum:sum+test_gem+1.0.1\n", CompactIndex.info(param)
+    end
+  end
+
+  private
+
+  def build_version(**args)
+    name = args.fetch(:name, "test_gem")
+    number = args.fetch(:number, "1.0")
+    CompactIndex::GemVersion.new(
+      number,
+      args[:platform],
+      args.fetch(:checksum, "sum+#{name}+#{number}"),
+      args.fetch(:info_checksum, "info+#{name}+#{number}"),
+      args[:dependencies],
+      args[:ruby_version],
+      args[:rubygems_version]
+    )
+  end
+end