diff --git a/.env.example b/.env.example index 6ff73b04c..bcf9830dc 100644 --- a/.env.example +++ b/.env.example @@ -62,3 +62,6 @@ SALESFORCE_CONNECT_PORT=5432 SALESFORCE_CONNECT_DB=salesforce_development SALESFORCE_CONNECT_PASSWORD=password SALESFORCE_CONNECT_USER=postgres + +SCRATCH_ASSET_CONFIG_BASE_URL=https://example.com/config/ +SCRATCH_ASSET_IMPORT_BASE_URL=https://example.com/assets/ \ No newline at end of file diff --git a/Gemfile b/Gemfile index ffc19fa7b..d2b344252 100644 --- a/Gemfile +++ b/Gemfile @@ -42,6 +42,7 @@ gem 'puma', '~> 7.2' gem 'rack_content_type_default', '~> 1.1' gem 'rack-cors' gem 'rails', '~> 7.1' +gem 'ruby-progressbar', '~> 1.13', require: false gem 'sentry-rails' gem 'statesman' diff --git a/Gemfile.lock b/Gemfile.lock index 89ac28885..462466768 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -635,6 +635,7 @@ DEPENDENCIES rubocop-rspec_rails ruby-lsp ruby-lsp-rspec (~> 0.1.29) + ruby-progressbar (~> 1.13) selenium-webdriver sentry-rails shoulda-matchers (~> 7.0) diff --git a/config/environments/development.rb b/config/environments/development.rb index bc0d9b4c8..bc4602d08 100644 --- a/config/environments/development.rb +++ b/config/environments/development.rb @@ -55,6 +55,8 @@ # Print deprecation notices to the Rails logger. config.active_support.deprecation = :log + config.log_level = ENV.fetch('RAILS_LOG_LEVEL', 'debug') + # Raise exceptions for disallowed deprecations. config.active_support.disallowed_deprecation = :raise diff --git a/lib/corp_middleware.rb b/lib/corp_middleware.rb index 8eaacd36e..e1b3e2db8 100644 --- a/lib/corp_middleware.rb +++ b/lib/corp_middleware.rb @@ -12,7 +12,7 @@ def call(env) request_origin = env['HTTP_HOST'] allowed_origins = OriginParser.parse_origins - if env['PATH_INFO'].start_with?('/rails/active_storage') && allowed_origins.any? do |origin| + if env['PATH_INFO'].start_with?('/rails/active_storage', '/api/scratch/assets/internalapi/asset/') && allowed_origins.any? do |origin| origin.is_a?(Regexp) ? origin =~ request_origin : origin == request_origin end headers['Cross-Origin-Resource-Policy'] = 'cross-origin' diff --git a/lib/scratch_asset_importer.rb b/lib/scratch_asset_importer.rb new file mode 100644 index 000000000..46568f7cd --- /dev/null +++ b/lib/scratch_asset_importer.rb @@ -0,0 +1,49 @@ +# frozen_string_literal: true + +require 'ruby-progressbar' + +class ScratchAssetImporter + def self.import(...) + new(...).import + end + + attr_reader :asset_base_url, :asset_names + + ASSET_FETCHING_DELAY = 0.2 + + def initialize(asset_names, asset_base_url) + @asset_names = asset_names + @asset_base_url = asset_base_url + end + + def import + bar = ProgressBar.create(format: '%t: |%B| %c of %C %E', total: asset_names.count) if show_progress? + + asset_names.each do |asset_name| + bar.increment if show_progress? + import_asset(asset_name) + end + end + + private + + def import_asset(asset_name) + return if ScratchAsset.exists?(filename: asset_name) + + sleep(ASSET_FETCHING_DELAY) + asset = connection.get("#{asset_name}/get/") + ScratchAsset.create!(filename: asset_name).file.attach(io: StringIO.new(asset.body), filename: asset_name) + rescue StandardError => e + Rails.logger.error("Failed to import asset #{asset_name}: #{e.message}") + end + + def connection + @connection ||= Faraday.new(url: asset_base_url) do |faraday| + faraday.response :raise_error + end + end + + def show_progress? + !Rails.env.test? + end +end diff --git a/lib/scratch_config_importer.rb b/lib/scratch_config_importer.rb new file mode 100644 index 000000000..ee005cd6e --- /dev/null +++ b/lib/scratch_config_importer.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +require 'ruby-progressbar' + +class ScratchConfigImporter + def self.import(...) + new(...).import + end + + attr_reader :asset_base_url, :asset_config_url + + def initialize(asset_config_url, asset_base_url) + @asset_config_url = asset_config_url + @asset_base_url = asset_base_url + end + + def import + config = Faraday.get(asset_config_url).body + asset_config = JSON.parse(config, symbolize_names: true) + asset_names = extract_asset_names(asset_config) + ScratchAssetImporter.import(asset_names, asset_base_url) + end + + private + + def extract_asset_names(config) + names = [] + config.each do |item| + names << item[:md5ext] if item[:md5ext] + names.concat(extract_asset_names(item.fetch(:costumes, []))) + names.concat(extract_asset_names(item.fetch(:sounds, []))) + end + names + end +end diff --git a/lib/tasks/scratch_assets.rake b/lib/tasks/scratch_assets.rake new file mode 100644 index 000000000..4e4d49c57 --- /dev/null +++ b/lib/tasks/scratch_assets.rake @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +require_relative 'seeds_helper' + +namespace :scratch_assets do + desc 'Import scratch assets' + task import_all: %i[import_backdrops import_costumes import_sounds import_sprites] + + task import_backdrops: :environment do + Rails.logger.info 'Importing backdrops...' + config_url = "#{config_base_url}backdrops.json" + ScratchConfigImporter.import(config_url, import_base_url) + end + + task import_costumes: :environment do + Rails.logger.info 'Importing costumes...' + config_url = "#{config_base_url}costumes.json" + ScratchConfigImporter.import(config_url, import_base_url) + end + + task import_sounds: :environment do + Rails.logger.info 'Importing sounds...' + config_url = "#{config_base_url}sounds.json" + ScratchConfigImporter.import(config_url, import_base_url) + end + + task import_sprites: :environment do + Rails.logger.info 'Importing sprites...' + config_url = "#{config_base_url}sprites.json" + ScratchConfigImporter.import(config_url, import_base_url) + end + + def config_base_url + ENV.fetch('SCRATCH_ASSET_CONFIG_BASE_URL') + end + + def import_base_url + ENV.fetch('SCRATCH_ASSET_IMPORT_BASE_URL') + end +end diff --git a/spec/lib/corp_middleware_spec.rb b/spec/lib/corp_middleware_spec.rb index bdd26c21a..aa94a2512 100644 --- a/spec/lib/corp_middleware_spec.rb +++ b/spec/lib/corp_middleware_spec.rb @@ -18,6 +18,12 @@ expect(headers['Cross-Origin-Resource-Policy']).to eq('cross-origin') end + it 'sets the Cross-Origin-Resource-Policy header for requests to scratch assets' do + _status, headers, _response = middleware.call(env.merge('PATH_INFO' => '/api/scratch/assets/internalapi/asset/123/get/')) + + expect(headers['Cross-Origin-Resource-Policy']).to eq('cross-origin') + end + it 'sets the Cross-Origin-Resource-Policy header for regex origin' do allow(ENV).to receive(:[]).with('ALLOWED_ORIGINS').and_return('/test\.com/') diff --git a/spec/lib/scratch_asset_importer_spec.rb b/spec/lib/scratch_asset_importer_spec.rb new file mode 100644 index 000000000..3058b3754 --- /dev/null +++ b/spec/lib/scratch_asset_importer_spec.rb @@ -0,0 +1,49 @@ +# frozen_string_literal: true + +require 'rails_helper' +require 'scratch_asset_importer' + +RSpec.describe ScratchAssetImporter do + describe '.import' do + it 'imports assets from the config' do + image = Rails.root.join('spec/fixtures/files/test_image_1.png').read + stub_request(:get, 'https://example.net/internalapi/asset/123abc.png/get/').to_return(status: 200, body: image) + + described_class.import(['123abc.png'], 'https://example.net/internalapi/asset/') + + scratch_asset = ScratchAsset.find_by(filename: '123abc.png') + expect(scratch_asset).to be_present + expect(scratch_asset.file.download).to eq(image) + end + + it 'does nothing if asset already exists' do + create(:scratch_asset, :with_file, filename: '123abc.png') + + expect do + described_class.import(['123abc.png'], 'https://example.net/internalapi/asset/') + end.not_to change(ScratchAsset, :count) + end + + it 'can import multiple assets' do + image = Rails.root.join('spec/fixtures/files/test_image_1.png').read + + stub_request(:get, 'https://example.net/internalapi/asset/123abc.png/get/').to_return(status: 200, body: image) + stub_request(:get, 'https://example.net/internalapi/asset/456xyz.png/get/').to_return(status: 200, body: image) + + described_class.import(['123abc.png', '456xyz.png'], 'https://example.net/internalapi/asset/') + expect(ScratchAsset.find_by(filename: '123abc.png')).to be_present + expect(ScratchAsset.find_by(filename: '456xyz.png')).to be_present + end + + it 'skips assets that fail to import' do + image = Rails.root.join('spec/fixtures/files/test_image_1.png').read + + stub_request(:get, 'https://example.net/internalapi/asset/123abc.png/get/').to_return(status: 500, body: 'error') + stub_request(:get, 'https://example.net/internalapi/asset/456xyz.png/get/').to_return(status: 200, body: image) + + described_class.import(['123abc.png', '456xyz.png'], 'https://example.net/internalapi/asset/') + expect(ScratchAsset.find_by(filename: '123abc.png')).not_to be_present + expect(ScratchAsset.find_by(filename: '456xyz.png')).to be_present + end + end +end diff --git a/spec/lib/scratch_config_importer_spec.rb b/spec/lib/scratch_config_importer_spec.rb new file mode 100644 index 000000000..3167a409a --- /dev/null +++ b/spec/lib/scratch_config_importer_spec.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +require 'rails_helper' +require 'scratch_asset_importer' + +RSpec.describe ScratchConfigImporter do + before do + allow(ScratchAssetImporter).to receive(:import) + end + + describe '.import' do + it 'imports assets from the config' do + config = [{ md5ext: '123abc.png' }].to_json + stub_request(:get, 'https://example.com/config/backdrops.json').to_return(status: 200, body: config) + + described_class.import('https://example.com/config/backdrops.json', 'https://example.net/internalapi/asset/') + + expect(ScratchAssetImporter).to have_received(:import).with(['123abc.png'], 'https://example.net/internalapi/asset/') + end + + it 'handles assets nested under sounds and costumes' do + config = [ + { + costumes: [{ + md5ext: '123abc.png' + }], + sounds: [{ + md5ext: '456xyz.png' + }] + } + ].to_json + stub_request(:get, 'https://example.com/config/sprites.json').to_return(status: 200, body: config) + + described_class.import('https://example.com/config/sprites.json', 'https://example.net/internalapi/asset/') + + expect(ScratchAssetImporter).to have_received(:import).with(['123abc.png', '456xyz.png'], 'https://example.net/internalapi/asset/') + end + end +end