Add support for uploading buildpacks packaged as cnb (tar) files

- This is a follow on to cloudfoundry#3898 which only supports uploading CNBs as zip or tgz whereas they are released in cnb format

Signed-off-by: Tom Kennedy <tom.kennedy@broadcom.com>

Author: tomkennedy513

app/messages/buildpack_upload_message.rb

@@ -3,6 +3,7 @@
 module VCAP::CloudController
   GZIP_MIME = Regexp.new("\x1F\x8B\x08".force_encoding('binary'))
   ZIP_MIME = Regexp.new("PK\x03\x04".force_encoding('binary'))
+  CNB_MIME = Regexp.new("\x75\x73\x74\x61\x72".force_encoding('binary'))
 
   class BuildpackUploadMessage < BaseMessage
     class MissingFilePathError < StandardError; end
@@ -53,7 +54,10 @@ def is_archive
 
       return if mime_bits =~ /^#{VCAP::CloudController::GZIP_MIME}/ || mime_bits =~ /^#{VCAP::CloudController::ZIP_MIME}/
 
-      errors.add(:base, "#{bits_name} is not a zip or gzip archive")
+      mime_bits_at_offset = File.read(bits_path, 5, 257)
+      return if mime_bits_at_offset =~  /^#{VCAP::CloudController::CNB_MIME}/
+
+      errors.add(:base, "#{bits_name} is not a zip or gzip archive or cnb file")
     end
 
     def missing_file_path

spec/support/test_cnb.rb

@@ -0,0 +1,16 @@
+require 'zlib'
+require 'rubygems/package'
+
+module TestCnb
+  def self.create(name, file_count, file_size = 1024, &)
+    File.open(name, 'wb') do |file|
+      Gem::Package::TarWriter.new(file) do |tar|
+        file_count.times do |i|
+          tar.add_file_simple("test_#{i}", 0o644, file_size) do |f|
+            f.write('A' * file_size)
+          end
+        end
+      end
+    end
+  end
+end

spec/unit/lib/cloud_controller/upload_buildpack_spec.rb

@@ -10,10 +10,12 @@ module VCAP::CloudController
     let(:tmpdir) { Dir.mktmpdir }
     let(:filename) { 'file.zip' }
     let(:tgz_filename) { 'file.tgz' }
+    let(:cnb_filename) { 'file.cnb' }
 
     let(:sha_valid_zip) { Digester.new(algorithm: OpenSSL::Digest::SHA256).digest_file(valid_zip) }
     let(:sha_valid_zip2) { Digester.new(algorithm: OpenSSL::Digest::SHA256).digest_file(valid_zip2) }
     let(:sha_valid_tgz) { Digester.new(algorithm: OpenSSL::Digest::SHA256).digest_file(valid_tgz) }
+    let(:sha_valid_cnb) { Digester.new(algorithm: OpenSSL::Digest::SHA256).digest_file(valid_cnb) }
 
     let(:valid_zip_manifest_stack) { nil }
     let(:valid_zip) do
@@ -43,6 +45,13 @@ module VCAP::CloudController
       Rack::Test::UploadedFile.new(tgz_file)
     end
 
+    let(:valid_cnb) do
+      cnb_name = File.join(tmpdir, cnb_filename)
+      TestCnb.create(cnb_name, 3, 1024)
+      cnb_file = File.new(cnb_name)
+      Rack::Test::UploadedFile.new(cnb_file)
+    end
+
     let(:staging_timeout) { TestConfig.config_instance.get(:staging, :timeout_in_seconds) }
 
     let(:expected_sha_valid_zip) { "#{buildpack.guid}_#{sha_valid_zip}" }
@@ -152,7 +161,7 @@ module VCAP::CloudController
           end
         end
 
-        context 'lifecycle: cnb' do
+        context 'lifecycle: gzip cnb' do
           let!(:buildpack) { VCAP::CloudController::Buildpack.create_from_hash({ name: 'upload_cnb_buildpack', stack: 'cider', position: 0, lifecycle: 'cnb' }) }
           let(:expected_sha_valid_tgz) { "#{buildpack.guid}_#{sha_valid_tgz}" }
 
@@ -163,6 +172,17 @@ module VCAP::CloudController
           end
         end
 
+        context 'lifecycle: cnb' do
+          let!(:buildpack) { VCAP::CloudController::Buildpack.create_from_hash({ name: 'upload_cnb_buildpack', stack: 'cider', position: 0, lifecycle: 'cnb' }) }
+          let(:expected_sha_valid_cnb) { "#{buildpack.guid}_#{sha_valid_cnb}" }
+
+          it 'uploads' do
+            expect(buildpack_blobstore).to receive(:cp_to_blobstore).with(valid_cnb, expected_sha_valid_cnb)
+
+            upload_buildpack.upload_buildpack(buildpack, valid_cnb, cnb_filename)
+          end
+        end
+
         it 'updates the buildpack filename' do
           expect do
             upload_buildpack.upload_buildpack(buildpack, valid_zip, filename)

spec/unit/messages/buildpack_upload_message_spec.rb

@@ -102,20 +102,31 @@ module VCAP::CloudController
           allow(File).to receive(:read).and_return("PX\x03\x04".force_encoding('binary'))
           upload_message = BuildpackUploadMessage.new(opts)
           expect(upload_message).not_to be_valid
-          expect(upload_message.errors.full_messages[0]).to include('buildpack.abcd is not a zip or gzip')
+          expect(upload_message.errors.full_messages[0]).to include('buildpack.abcd is not a zip or gzip archive or cnb file')
         end
       end
 
       context 'when the file is a tgz' do
         let(:opts) { { bits_path: '/tmp/bar', bits_name: 'buildpack.tgz' } }
 
-        it 'is not valid' do
+        it 'is valid' do
           allow(File).to receive(:read).and_return("\x1F\x8B\x08".force_encoding('binary'))
           upload_message = BuildpackUploadMessage.new(opts)
           expect(upload_message).to be_valid
         end
       end
 
+      context 'when the file is a cnb/tar' do
+        let(:opts) { { bits_path: '/tmp/bar', bits_name: 'buildpack.cnb' } }
+
+        it 'is valid' do
+          values = ["\x0".force_encoding('binary'), "\x75\x73\x74\x61\x72\x00\x30\x30".force_encoding('binary')]
+          allow(File).to receive(:read).and_return(*values)
+          upload_message = BuildpackUploadMessage.new(opts)
+          expect(upload_message).to be_valid
+        end
+      end
+
       context 'when the file is empty' do
         let(:opts) { { bits_path: '/tmp/bar', bits_name: 'buildpack.zip' } }
         let(:stat_double) { instance_double(File::Stat, size: 0) }