From 4d07ae9007cddda34d56075bb0538c550b7f75fa Mon Sep 17 00:00:00 2001 From: Chris Zetter <253059100+zetter-rpf@users.noreply.github.com> Date: Fri, 27 Mar 2026 16:43:27 +0000 Subject: [PATCH 1/5] Task to import Scratch assets We have permission from the Scratch Foundation to use the library assets available with Scratch. These tasks imports them assets defined in the in the library configuration files. See [1] for instructions on how to run. https://github.com/RaspberryPiFoundation/digital-editor-issues/issues/1229 --- .env.example | 3 ++ lib/scratch_asset_importer.rb | 39 +++++++++++++++++++ lib/scratch_config_importer.rb | 35 +++++++++++++++++ lib/tasks/scratch_assets.rake | 40 +++++++++++++++++++ spec/lib/scratch_asset_importer_spec.rb | 49 ++++++++++++++++++++++++ spec/lib/scratch_config_importer_spec.rb | 39 +++++++++++++++++++ 6 files changed, 205 insertions(+) create mode 100644 lib/scratch_asset_importer.rb create mode 100644 lib/scratch_config_importer.rb create mode 100644 lib/tasks/scratch_assets.rake create mode 100644 spec/lib/scratch_asset_importer_spec.rb create mode 100644 spec/lib/scratch_config_importer_spec.rb diff --git a/.env.example b/.env.example index 6ff73b04c..bcf9830dc 100644 --- a/.env.example +++ b/.env.example @@ -62,3 +62,6 @@ SALESFORCE_CONNECT_PORT=5432 SALESFORCE_CONNECT_DB=salesforce_development SALESFORCE_CONNECT_PASSWORD=password SALESFORCE_CONNECT_USER=postgres + +SCRATCH_ASSET_CONFIG_BASE_URL=https://example.com/config/ +SCRATCH_ASSET_IMPORT_BASE_URL=https://example.com/assets/ \ No newline at end of file diff --git a/lib/scratch_asset_importer.rb b/lib/scratch_asset_importer.rb new file mode 100644 index 000000000..10dfdc7cc --- /dev/null +++ b/lib/scratch_asset_importer.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +require 'ruby-progressbar' + +class ScratchAssetImporter + def self.import(...) + new(...).import + end + + attr_reader :asset_base_url, :asset_names + + def initialize(asset_names, asset_base_url) + @asset_names = asset_names + @asset_base_url = asset_base_url + end + + def import + asset_names.each do |asset_name| + import_asset(asset_name) + end + end + + private + + def import_asset(asset_name) + return if ScratchAsset.exists?(filename: asset_name) + + asset = connection.get("#{asset_name}/get/") + ScratchAsset.create!(filename: asset_name).file.attach(io: StringIO.new(asset.body), filename: asset_name) + rescue StandardError => e + Rails.logger.error("Failed to import asset #{asset_name}: #{e.message}") + end + + def connection + @connection ||= Faraday.new(url: asset_base_url) do |faraday| + faraday.response :raise_error + end + end +end diff --git a/lib/scratch_config_importer.rb b/lib/scratch_config_importer.rb new file mode 100644 index 000000000..ee005cd6e --- /dev/null +++ b/lib/scratch_config_importer.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +require 'ruby-progressbar' + +class ScratchConfigImporter + def self.import(...) + new(...).import + end + + attr_reader :asset_base_url, :asset_config_url + + def initialize(asset_config_url, asset_base_url) + @asset_config_url = asset_config_url + @asset_base_url = asset_base_url + end + + def import + config = Faraday.get(asset_config_url).body + asset_config = JSON.parse(config, symbolize_names: true) + asset_names = extract_asset_names(asset_config) + ScratchAssetImporter.import(asset_names, asset_base_url) + end + + private + + def extract_asset_names(config) + names = [] + config.each do |item| + names << item[:md5ext] if item[:md5ext] + names.concat(extract_asset_names(item.fetch(:costumes, []))) + names.concat(extract_asset_names(item.fetch(:sounds, []))) + end + names + end +end diff --git a/lib/tasks/scratch_assets.rake b/lib/tasks/scratch_assets.rake new file mode 100644 index 000000000..4e4d49c57 --- /dev/null +++ b/lib/tasks/scratch_assets.rake @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +require_relative 'seeds_helper' + +namespace :scratch_assets do + desc 'Import scratch assets' + task import_all: %i[import_backdrops import_costumes import_sounds import_sprites] + + task import_backdrops: :environment do + Rails.logger.info 'Importing backdrops...' + config_url = "#{config_base_url}backdrops.json" + ScratchConfigImporter.import(config_url, import_base_url) + end + + task import_costumes: :environment do + Rails.logger.info 'Importing costumes...' + config_url = "#{config_base_url}costumes.json" + ScratchConfigImporter.import(config_url, import_base_url) + end + + task import_sounds: :environment do + Rails.logger.info 'Importing sounds...' + config_url = "#{config_base_url}sounds.json" + ScratchConfigImporter.import(config_url, import_base_url) + end + + task import_sprites: :environment do + Rails.logger.info 'Importing sprites...' + config_url = "#{config_base_url}sprites.json" + ScratchConfigImporter.import(config_url, import_base_url) + end + + def config_base_url + ENV.fetch('SCRATCH_ASSET_CONFIG_BASE_URL') + end + + def import_base_url + ENV.fetch('SCRATCH_ASSET_IMPORT_BASE_URL') + end +end diff --git a/spec/lib/scratch_asset_importer_spec.rb b/spec/lib/scratch_asset_importer_spec.rb new file mode 100644 index 000000000..3058b3754 --- /dev/null +++ b/spec/lib/scratch_asset_importer_spec.rb @@ -0,0 +1,49 @@ +# frozen_string_literal: true + +require 'rails_helper' +require 'scratch_asset_importer' + +RSpec.describe ScratchAssetImporter do + describe '.import' do + it 'imports assets from the config' do + image = Rails.root.join('spec/fixtures/files/test_image_1.png').read + stub_request(:get, 'https://example.net/internalapi/asset/123abc.png/get/').to_return(status: 200, body: image) + + described_class.import(['123abc.png'], 'https://example.net/internalapi/asset/') + + scratch_asset = ScratchAsset.find_by(filename: '123abc.png') + expect(scratch_asset).to be_present + expect(scratch_asset.file.download).to eq(image) + end + + it 'does nothing if asset already exists' do + create(:scratch_asset, :with_file, filename: '123abc.png') + + expect do + described_class.import(['123abc.png'], 'https://example.net/internalapi/asset/') + end.not_to change(ScratchAsset, :count) + end + + it 'can import multiple assets' do + image = Rails.root.join('spec/fixtures/files/test_image_1.png').read + + stub_request(:get, 'https://example.net/internalapi/asset/123abc.png/get/').to_return(status: 200, body: image) + stub_request(:get, 'https://example.net/internalapi/asset/456xyz.png/get/').to_return(status: 200, body: image) + + described_class.import(['123abc.png', '456xyz.png'], 'https://example.net/internalapi/asset/') + expect(ScratchAsset.find_by(filename: '123abc.png')).to be_present + expect(ScratchAsset.find_by(filename: '456xyz.png')).to be_present + end + + it 'skips assets that fail to import' do + image = Rails.root.join('spec/fixtures/files/test_image_1.png').read + + stub_request(:get, 'https://example.net/internalapi/asset/123abc.png/get/').to_return(status: 500, body: 'error') + stub_request(:get, 'https://example.net/internalapi/asset/456xyz.png/get/').to_return(status: 200, body: image) + + described_class.import(['123abc.png', '456xyz.png'], 'https://example.net/internalapi/asset/') + expect(ScratchAsset.find_by(filename: '123abc.png')).not_to be_present + expect(ScratchAsset.find_by(filename: '456xyz.png')).to be_present + end + end +end diff --git a/spec/lib/scratch_config_importer_spec.rb b/spec/lib/scratch_config_importer_spec.rb new file mode 100644 index 000000000..3167a409a --- /dev/null +++ b/spec/lib/scratch_config_importer_spec.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +require 'rails_helper' +require 'scratch_asset_importer' + +RSpec.describe ScratchConfigImporter do + before do + allow(ScratchAssetImporter).to receive(:import) + end + + describe '.import' do + it 'imports assets from the config' do + config = [{ md5ext: '123abc.png' }].to_json + stub_request(:get, 'https://example.com/config/backdrops.json').to_return(status: 200, body: config) + + described_class.import('https://example.com/config/backdrops.json', 'https://example.net/internalapi/asset/') + + expect(ScratchAssetImporter).to have_received(:import).with(['123abc.png'], 'https://example.net/internalapi/asset/') + end + + it 'handles assets nested under sounds and costumes' do + config = [ + { + costumes: [{ + md5ext: '123abc.png' + }], + sounds: [{ + md5ext: '456xyz.png' + }] + } + ].to_json + stub_request(:get, 'https://example.com/config/sprites.json').to_return(status: 200, body: config) + + described_class.import('https://example.com/config/sprites.json', 'https://example.net/internalapi/asset/') + + expect(ScratchAssetImporter).to have_received(:import).with(['123abc.png', '456xyz.png'], 'https://example.net/internalapi/asset/') + end + end +end From ea6a95eae912123ae239da873e194a7013b13c35 Mon Sep 17 00:00:00 2001 From: Chris Zetter <253059100+zetter-rpf@users.noreply.github.com> Date: Fri, 27 Mar 2026 16:47:32 +0000 Subject: [PATCH 2/5] Add a delay between fetching assets It's good practice when performing a lots of requests to add in time so not to overload any servers. --- lib/scratch_asset_importer.rb | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/lib/scratch_asset_importer.rb b/lib/scratch_asset_importer.rb index 10dfdc7cc..f67d98ec6 100644 --- a/lib/scratch_asset_importer.rb +++ b/lib/scratch_asset_importer.rb @@ -9,6 +9,8 @@ def self.import(...) attr_reader :asset_base_url, :asset_names + ASSET_FETCHING_DELAY = 0.2 + def initialize(asset_names, asset_base_url) @asset_names = asset_names @asset_base_url = asset_base_url @@ -25,6 +27,7 @@ def import def import_asset(asset_name) return if ScratchAsset.exists?(filename: asset_name) + sleep(ASSET_FETCHING_DELAY) asset = connection.get("#{asset_name}/get/") ScratchAsset.create!(filename: asset_name).file.attach(io: StringIO.new(asset.body), filename: asset_name) rescue StandardError => e @@ -36,4 +39,8 @@ def connection faraday.response :raise_error end end + + def show_progress? + !Rails.env.test? + end end From 607c38cbef61152060e138dc6edce52e24689681 Mon Sep 17 00:00:00 2001 From: Chris Zetter <253059100+zetter-rpf@users.noreply.github.com> Date: Fri, 27 Mar 2026 16:49:45 +0000 Subject: [PATCH 3/5] Make the rails log level configurable in development When running rake tasks locally there can be a lot of output from SQL queries and active storage, making it harder to see warnings. Make it possible to configure the Rails log level in development using environment variables. This is just like the code in production.rb except it uses the development default of debug. --- config/environments/development.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/config/environments/development.rb b/config/environments/development.rb index bc0d9b4c8..bc4602d08 100644 --- a/config/environments/development.rb +++ b/config/environments/development.rb @@ -55,6 +55,8 @@ # Print deprecation notices to the Rails logger. config.active_support.deprecation = :log + config.log_level = ENV.fetch('RAILS_LOG_LEVEL', 'debug') + # Raise exceptions for disallowed deprecations. config.active_support.disallowed_deprecation = :raise From 39dd62cf84eff1472cc0ea09be7d644966140759 Mon Sep 17 00:00:00 2001 From: Chris Zetter <253059100+zetter-rpf@users.noreply.github.com> Date: Fri, 27 Mar 2026 16:51:50 +0000 Subject: [PATCH 4/5] Display a progress bar Some of these tasks may take a while, the ruby progress bar library makes it easy to get a sense of progress. I've not shown the progress bar in tests as it messes the test output. --- Gemfile | 1 + Gemfile.lock | 1 + lib/scratch_asset_importer.rb | 3 +++ 3 files changed, 5 insertions(+) diff --git a/Gemfile b/Gemfile index ffc19fa7b..d2b344252 100644 --- a/Gemfile +++ b/Gemfile @@ -42,6 +42,7 @@ gem 'puma', '~> 7.2' gem 'rack_content_type_default', '~> 1.1' gem 'rack-cors' gem 'rails', '~> 7.1' +gem 'ruby-progressbar', '~> 1.13', require: false gem 'sentry-rails' gem 'statesman' diff --git a/Gemfile.lock b/Gemfile.lock index 89ac28885..462466768 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -635,6 +635,7 @@ DEPENDENCIES rubocop-rspec_rails ruby-lsp ruby-lsp-rspec (~> 0.1.29) + ruby-progressbar (~> 1.13) selenium-webdriver sentry-rails shoulda-matchers (~> 7.0) diff --git a/lib/scratch_asset_importer.rb b/lib/scratch_asset_importer.rb index f67d98ec6..46568f7cd 100644 --- a/lib/scratch_asset_importer.rb +++ b/lib/scratch_asset_importer.rb @@ -17,7 +17,10 @@ def initialize(asset_names, asset_base_url) end def import + bar = ProgressBar.create(format: '%t: |%B| %c of %C %E', total: asset_names.count) if show_progress? + asset_names.each do |asset_name| + bar.increment if show_progress? import_asset(asset_name) end end From 23387d70022a51ca5310e9a07eb3b95184b5b750 Mon Sep 17 00:00:00 2001 From: Chris Zetter <253059100+zetter-rpf@users.noreply.github.com> Date: Tue, 31 Mar 2026 14:25:36 +0100 Subject: [PATCH 5/5] Make sure Scratch assets can be displayed in library The library uses an tag to load and embed the assets, which is different to how assets are loaded in projects For this to work, the asset needs to have a Cross-Origin-Resource-Policy set. --- lib/corp_middleware.rb | 2 +- spec/lib/corp_middleware_spec.rb | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/lib/corp_middleware.rb b/lib/corp_middleware.rb index 8eaacd36e..e1b3e2db8 100644 --- a/lib/corp_middleware.rb +++ b/lib/corp_middleware.rb @@ -12,7 +12,7 @@ def call(env) request_origin = env['HTTP_HOST'] allowed_origins = OriginParser.parse_origins - if env['PATH_INFO'].start_with?('/rails/active_storage') && allowed_origins.any? do |origin| + if env['PATH_INFO'].start_with?('/rails/active_storage', '/api/scratch/assets/internalapi/asset/') && allowed_origins.any? do |origin| origin.is_a?(Regexp) ? origin =~ request_origin : origin == request_origin end headers['Cross-Origin-Resource-Policy'] = 'cross-origin' diff --git a/spec/lib/corp_middleware_spec.rb b/spec/lib/corp_middleware_spec.rb index bdd26c21a..aa94a2512 100644 --- a/spec/lib/corp_middleware_spec.rb +++ b/spec/lib/corp_middleware_spec.rb @@ -18,6 +18,12 @@ expect(headers['Cross-Origin-Resource-Policy']).to eq('cross-origin') end + it 'sets the Cross-Origin-Resource-Policy header for requests to scratch assets' do + _status, headers, _response = middleware.call(env.merge('PATH_INFO' => '/api/scratch/assets/internalapi/asset/123/get/')) + + expect(headers['Cross-Origin-Resource-Policy']).to eq('cross-origin') + end + it 'sets the Cross-Origin-Resource-Policy header for regex origin' do allow(ENV).to receive(:[]).with('ALLOWED_ORIGINS').and_return('/test\.com/')