diff options
Diffstat (limited to 'gems')
265 files changed, 10842 insertions, 0 deletions
diff --git a/gems/activerecord-gitlab/.gitignore b/gems/activerecord-gitlab/.gitignore new file mode 100644 index 00000000000..b04a8c840df --- /dev/null +++ b/gems/activerecord-gitlab/.gitignore @@ -0,0 +1,11 @@ +/.bundle/ +/.yardoc +/_yardoc/ +/coverage/ +/doc/ +/pkg/ +/spec/reports/ +/tmp/ + +# rspec failure tracking +.rspec_status diff --git a/gems/activerecord-gitlab/.gitlab-ci.yml b/gems/activerecord-gitlab/.gitlab-ci.yml new file mode 100644 index 00000000000..eb94904ce90 --- /dev/null +++ b/gems/activerecord-gitlab/.gitlab-ci.yml @@ -0,0 +1,4 @@ +include: + - local: gems/gem.gitlab-ci.yml + inputs: + gem_name: "activerecord-gitlab" diff --git a/gems/activerecord-gitlab/.rspec b/gems/activerecord-gitlab/.rspec new file mode 100644 index 00000000000..34c5164d9b5 --- /dev/null +++ b/gems/activerecord-gitlab/.rspec @@ -0,0 +1,3 @@ +--format documentation +--color +--require spec_helper diff --git a/gems/activerecord-gitlab/.rubocop.yml b/gems/activerecord-gitlab/.rubocop.yml new file mode 100644 index 00000000000..5fce096769a --- /dev/null +++ b/gems/activerecord-gitlab/.rubocop.yml @@ -0,0 +1,18 @@ +inherit_from: + - ../config/rubocop.yml + +# FIXME +Gitlab/RSpec/AvoidSetup: + Enabled: false + +Database/EstablishConnection: + Enabled: false + +Database/MultipleDatabases: + Enabled: false + +RSpec/EnvAssignment: + Enabled: false + +RSpec/MultipleMemoizedHelpers: + Enabled: false diff --git a/gems/activerecord-gitlab/Gemfile b/gems/activerecord-gitlab/Gemfile new file mode 100644 index 00000000000..be173b205f7 --- /dev/null +++ b/gems/activerecord-gitlab/Gemfile @@ -0,0 +1,5 @@ +# frozen_string_literal: true + +source "https://rubygems.org" + +gemspec diff --git a/gems/activerecord-gitlab/Gemfile.lock b/gems/activerecord-gitlab/Gemfile.lock new file mode 100644 index 00000000000..ba0118eb8e3 --- /dev/null +++ b/gems/activerecord-gitlab/Gemfile.lock @@ -0,0 +1,104 @@ +PATH + remote: . + specs: + activerecord-gitlab (0.2.0) + activerecord (>= 7) + +GEM + remote: https://rubygems.org/ + specs: + activemodel (7.0.6) + activesupport (= 7.0.6) + activerecord (7.0.6) + activemodel (= 7.0.6) + activesupport (= 7.0.6) + activesupport (7.0.6) + concurrent-ruby (~> 1.0, >= 1.0.2) + i18n (>= 1.6, < 2) + minitest (>= 5.1) + tzinfo (~> 2.0) + ast (2.4.2) + concurrent-ruby (1.2.2) + diff-lcs (1.5.0) + gitlab-styles (10.1.0) + rubocop (~> 1.50.2) + rubocop-graphql (~> 0.18) + rubocop-performance (~> 1.15) + rubocop-rails (~> 2.17) + rubocop-rspec (~> 2.22) + i18n (1.13.0) + concurrent-ruby (~> 1.0) + json (2.6.3) + mini_portile2 (2.8.2) + minitest (5.18.0) + parallel (1.23.0) + parser (3.2.2.3) + ast (~> 2.4.1) + racc + racc (1.7.1) + rack (3.0.8) + rainbow (3.1.1) + regexp_parser (2.8.1) + rexml (3.2.5) + rspec (3.12.0) + rspec-core (~> 3.12.0) + rspec-expectations (~> 3.12.0) + rspec-mocks (~> 3.12.0) + rspec-core (3.12.2) + rspec-support (~> 3.12.0) + rspec-expectations (3.12.3) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.12.0) + rspec-mocks (3.12.5) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.12.0) + rspec-support (3.12.0) + rubocop (1.50.2) + json (~> 2.3) + parallel (~> 1.10) + parser (>= 3.2.0.0) + rainbow (>= 2.2.2, < 4.0) + regexp_parser (>= 1.8, < 3.0) + rexml (>= 3.2.5, < 4.0) + rubocop-ast (>= 1.28.0, < 2.0) + ruby-progressbar (~> 1.7) + unicode-display_width (>= 2.4.0, < 3.0) + rubocop-ast (1.29.0) + parser (>= 3.2.1.0) + rubocop-capybara (2.18.0) + rubocop (~> 1.41) + rubocop-factory_bot (2.23.1) + rubocop (~> 1.33) + rubocop-graphql (0.19.0) + rubocop (>= 0.87, < 2) + rubocop-performance (1.18.0) + rubocop (>= 1.7.0, < 2.0) + rubocop-ast (>= 0.4.0) + rubocop-rails (2.20.2) + activesupport (>= 4.2.0) + rack (>= 1.1) + rubocop (>= 1.33.0, < 2.0) + rubocop-rspec (2.22.0) + rubocop (~> 1.33) + rubocop-capybara (~> 2.17) + rubocop-factory_bot (~> 2.22) + ruby-progressbar (1.13.0) + sqlite3 (1.6.3) + mini_portile2 (~> 2.8.0) + tzinfo (2.0.6) + concurrent-ruby (~> 1.0) + unicode-display_width (2.4.2) + +PLATFORMS + ruby + +DEPENDENCIES + activerecord-gitlab! + gitlab-styles (~> 10.1.0) + rspec (~> 3.12) + rubocop (~> 1.50) + rubocop-rspec (~> 2.22) + sqlite3 (~> 1.6) + +BUNDLED WITH + 2.4.14 diff --git a/gems/activerecord-gitlab/README.md b/gems/activerecord-gitlab/README.md new file mode 100644 index 00000000000..c5b56e367b9 --- /dev/null +++ b/gems/activerecord-gitlab/README.md @@ -0,0 +1,4 @@ +# ActiveRecord::GitlabPatches + +This gem adds GitLab specific Active Record patches. +We have patches as a separate gem to isolate complexity. diff --git a/gems/activerecord-gitlab/activerecord-gitlab.gemspec b/gems/activerecord-gitlab/activerecord-gitlab.gemspec new file mode 100644 index 00000000000..267938d0de3 --- /dev/null +++ b/gems/activerecord-gitlab/activerecord-gitlab.gemspec @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +require_relative "lib/active_record/gitlab_patches/version" + +Gem::Specification.new do |spec| + spec.name = "activerecord-gitlab" + spec.version = ActiveRecord::GitlabPatches::Version::VERSION + spec.authors = ["group::tenant-scale"] + spec.email = ["engineering@gitlab.com"] + + spec.summary = "GitLab ActiveRecord patches" + spec.description = "GitLab stores any patches relating to ActiveRecord here" + spec.homepage = "https://gitlab.com/gitlab-org/gitlab/-/tree/master/gems/activerecord-gitlab" + spec.license = "MIT" + spec.required_ruby_version = ">= 3.0" + spec.metadata["rubygems_mfa_required"] = "true" + + spec.files = Dir["lib/**/*.rb"] + spec.require_paths = ["lib"] + + spec.add_runtime_dependency "activerecord", ">= 7" + + spec.add_development_dependency "gitlab-styles", "~> 10.1.0" + spec.add_development_dependency "rspec", "~> 3.12" + spec.add_development_dependency "rubocop", "~> 1.50" + spec.add_development_dependency "rubocop-rspec", "~> 2.22" + spec.add_development_dependency "sqlite3", "~> 1.6" +end diff --git a/gems/activerecord-gitlab/lib/active_record/gitlab_patches.rb b/gems/activerecord-gitlab/lib/active_record/gitlab_patches.rb new file mode 100644 index 00000000000..257602497f0 --- /dev/null +++ b/gems/activerecord-gitlab/lib/active_record/gitlab_patches.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +require "active_record" +require_relative "gitlab_patches/version" +require_relative "gitlab_patches/rescue_from" +require_relative "gitlab_patches/partitioning" + +module ActiveRecord + module GitlabPatches + end +end diff --git a/gems/activerecord-gitlab/lib/active_record/gitlab_patches/partitioning.rb b/gems/activerecord-gitlab/lib/active_record/gitlab_patches/partitioning.rb new file mode 100644 index 00000000000..cf0bd4849e2 --- /dev/null +++ b/gems/activerecord-gitlab/lib/active_record/gitlab_patches/partitioning.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +require_relative "partitioning/associations/builder/association" +require_relative "partitioning/reflection/abstract_reflection" +require_relative "partitioning/reflection/association_reflection" +require_relative "partitioning/reflection/macro_reflection" +require_relative "partitioning/base" + +module ActiveRecord + module GitlabPatches + # This allows to filter data by a dedicated column for association and joins to ActiveRecord::Base. + # + # class ApplicationRecord < ActiveRecord::Base + # belongs_to :pipeline, + # -> (build) { where(partition_id: build.partition_id) }, + # partition_foreign_key: :partition_id + # + # To control the join filter + # def self.use_partition_id_filter? + # Feature.enabled?(...) + # end + # end + module Partitioning + ActiveSupport.on_load(:active_record) do + ::ActiveRecord::Associations::Builder::Association.prepend( + ActiveRecord::GitlabPatches::Partitioning::Associations::Builder::Association + ) + ::ActiveRecord::Reflection::AbstractReflection.prepend( + ActiveRecord::GitlabPatches::Partitioning::Reflection::AbstractReflection + ) + ::ActiveRecord::Reflection::AssociationReflection.prepend( + ActiveRecord::GitlabPatches::Partitioning::Reflection::AssociationReflection + ) + ::ActiveRecord::Reflection::MacroReflection.prepend( + ActiveRecord::GitlabPatches::Partitioning::Reflection::MacroReflection + ) + ::ActiveRecord::Base.prepend( + ActiveRecord::GitlabPatches::Partitioning::Base + ) + end + end + end +end diff --git a/gems/activerecord-gitlab/lib/active_record/gitlab_patches/partitioning/associations/builder/association.rb b/gems/activerecord-gitlab/lib/active_record/gitlab_patches/partitioning/associations/builder/association.rb new file mode 100644 index 00000000000..3c92ba91c31 --- /dev/null +++ b/gems/activerecord-gitlab/lib/active_record/gitlab_patches/partitioning/associations/builder/association.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module ActiveRecord + module GitlabPatches + module Partitioning + module Associations + module Builder + module Association + extend ActiveSupport::Concern + + class_methods do + def valid_options(options) + super + [:partition_foreign_key] + end + end + end + end + end + end + end +end diff --git a/gems/activerecord-gitlab/lib/active_record/gitlab_patches/partitioning/base.rb b/gems/activerecord-gitlab/lib/active_record/gitlab_patches/partitioning/base.rb new file mode 100644 index 00000000000..0c8a248b984 --- /dev/null +++ b/gems/activerecord-gitlab/lib/active_record/gitlab_patches/partitioning/base.rb @@ -0,0 +1,49 @@ +# frozen_string_literal: true + +if ::ActiveRecord::VERSION::STRING >= "7.1" + raise 'New version of active-record detected, please remove or update this patch' +end + +module ActiveRecord + module GitlabPatches + module Partitioning + module Base + extend ActiveSupport::Concern + + def _query_constraints_hash + constraints_hash = super + + return constraints_hash unless self.class.use_partition_id_filter? + + if self.class.query_constraints_list.nil? + { @primary_key => id_in_database } # rubocop:disable Gitlab/ModuleWithInstanceVariables + else + self.class.query_constraints_list.index_with do |column_name| + attribute_in_database(column_name) + end + end + end + + class_methods do + def use_partition_id_filter? + false + end + + def query_constraints(*columns_list) + raise ArgumentError, "You must specify at least one column to be used in querying" if columns_list.empty? + + @query_constraints_list = columns_list.map(&:to_s) + end + + def query_constraints_list # :nodoc: + @query_constraints_list ||= if base_class? || primary_key != base_class.primary_key + primary_key if primary_key.is_a?(Array) + else + base_class.query_constraints_list + end + end + end + end + end + end +end diff --git a/gems/activerecord-gitlab/lib/active_record/gitlab_patches/partitioning/reflection/abstract_reflection.rb b/gems/activerecord-gitlab/lib/active_record/gitlab_patches/partitioning/reflection/abstract_reflection.rb new file mode 100644 index 00000000000..7532cd120a5 --- /dev/null +++ b/gems/activerecord-gitlab/lib/active_record/gitlab_patches/partitioning/reflection/abstract_reflection.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module ActiveRecord + module GitlabPatches + module Partitioning + module Reflection + module AbstractReflection + extend ActiveSupport::Concern + + def join_scope(table, foreign_table, foreign_klass) + klass_scope = super + return klass_scope unless respond_to?(:options) + + partition_foreign_key = options[:partition_foreign_key] + if partition_foreign_key && klass.use_partition_id_filter? + klass_scope.where!(table[:partition_id].eq(foreign_table[partition_foreign_key])) + end + + klass_scope + end + end + end + end + end +end diff --git a/gems/activerecord-gitlab/lib/active_record/gitlab_patches/partitioning/reflection/association_reflection.rb b/gems/activerecord-gitlab/lib/active_record/gitlab_patches/partitioning/reflection/association_reflection.rb new file mode 100644 index 00000000000..299ceaab973 --- /dev/null +++ b/gems/activerecord-gitlab/lib/active_record/gitlab_patches/partitioning/reflection/association_reflection.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module ActiveRecord + module GitlabPatches + module Partitioning + module Reflection + module AssociationReflection + def check_eager_loadable! + return if scope && scope.arity == 1 && options.key?(:partition_foreign_key) + + super + end + end + end + end + end +end diff --git a/gems/activerecord-gitlab/lib/active_record/gitlab_patches/partitioning/reflection/macro_reflection.rb b/gems/activerecord-gitlab/lib/active_record/gitlab_patches/partitioning/reflection/macro_reflection.rb new file mode 100644 index 00000000000..7ec7da44253 --- /dev/null +++ b/gems/activerecord-gitlab/lib/active_record/gitlab_patches/partitioning/reflection/macro_reflection.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module ActiveRecord + module GitlabPatches + module Partitioning + module Reflection + module MacroReflection + def scope_for(relation, owner = nil) + if scope.arity == 1 && owner.nil? && options.key?(:partition_foreign_key) + relation + else + super + end + end + end + end + end + end +end diff --git a/gems/activerecord-gitlab/lib/active_record/gitlab_patches/rescue_from.rb b/gems/activerecord-gitlab/lib/active_record/gitlab_patches/rescue_from.rb new file mode 100644 index 00000000000..eaa42d1523d --- /dev/null +++ b/gems/activerecord-gitlab/lib/active_record/gitlab_patches/rescue_from.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +module ActiveRecord + module GitlabPatches + # This adds `rescue_from` to ActiveRecord::Base. + # Currently, only errors called from `ActiveRecord::Relation#exec_queries` + # will be handled by `rescue_from`. + # + # class ApplicationRecord < ActiveRecord::Base + # rescue_from MyException, with: :my_handler + # + # def my_handler(exception) + # Rails.logger.info exception.message + # + # raise exception + # end + # end + module RescueFrom + extend ActiveSupport::Concern + + prepended do |base| + base.include ActiveSupport::Rescuable + end + end + + module ExecQueriesRescueWithHandler + def exec_queries + super + rescue StandardError => e + klass.rescue_with_handler(e) || raise + end + end + end +end + +ActiveSupport.on_load(:active_record) do + ActiveRecord::Relation.prepend(ActiveRecord::GitlabPatches::ExecQueriesRescueWithHandler) + ActiveRecord::Base.prepend(ActiveRecord::GitlabPatches::RescueFrom) +end diff --git a/gems/activerecord-gitlab/lib/active_record/gitlab_patches/version.rb b/gems/activerecord-gitlab/lib/active_record/gitlab_patches/version.rb new file mode 100644 index 00000000000..00c5f254da8 --- /dev/null +++ b/gems/activerecord-gitlab/lib/active_record/gitlab_patches/version.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +module ActiveRecord + module GitlabPatches + module Version + VERSION = "0.2.0" + end + end +end diff --git a/gems/activerecord-gitlab/spec/active_record/gitlab_patches/partitioning/associations/belongs_to_spec.rb b/gems/activerecord-gitlab/spec/active_record/gitlab_patches/partitioning/associations/belongs_to_spec.rb new file mode 100644 index 00000000000..900a270c0a8 --- /dev/null +++ b/gems/activerecord-gitlab/spec/active_record/gitlab_patches/partitioning/associations/belongs_to_spec.rb @@ -0,0 +1,52 @@ +# frozen_string_literal: true + +RSpec.describe 'ActiveRecord::GitlabPatches::Partitioning::Associations::BelongsTo', :partitioning do + let(:pipeline) { Pipeline.create!(partition_id: 100) } + let(:job) { Job.create!(pipeline: pipeline, partition_id: pipeline.partition_id) } + + it 'finds associated record using partition_id' do + find_statement = <<~SQL.squish + SELECT \"pipelines\".* + FROM \"pipelines\" + WHERE \"pipelines\".\"id\" = #{pipeline.id} + AND \"pipelines\".\"partition_id\" = #{job.partition_id} + LIMIT 1 + SQL + + result = QueryRecorder.log do + job.reset.pipeline + end + + expect(result).to include(find_statement) + end + + it 'builds records using partition_id' do + pipeline = job.build_pipeline + + expect(pipeline.partition_id).to eq(job.partition_id) + end + + it 'saves records using partition_id' do + create_statement = <<~SQL.squish + INSERT INTO \"pipelines\" (\"partition_id\") VALUES (#{job.partition_id}) + SQL + + result = QueryRecorder.log do + job.build_pipeline.save! + end + + expect(result).to include(create_statement) + end + + it 'creates records using partition_id' do + create_statement = <<~SQL.squish + INSERT INTO \"pipelines\" (\"partition_id\") VALUES (#{job.partition_id}) + SQL + + result = QueryRecorder.log do + job.create_pipeline! + end + + expect(result).to include(create_statement) + end +end diff --git a/gems/activerecord-gitlab/spec/active_record/gitlab_patches/partitioning/associations/has_many_spec.rb b/gems/activerecord-gitlab/spec/active_record/gitlab_patches/partitioning/associations/has_many_spec.rb new file mode 100644 index 00000000000..3d6b24de998 --- /dev/null +++ b/gems/activerecord-gitlab/spec/active_record/gitlab_patches/partitioning/associations/has_many_spec.rb @@ -0,0 +1,115 @@ +# frozen_string_literal: true + +RSpec.describe 'ActiveRecord::GitlabPatches::Partitioning::Associations::HasMany', :partitioning do + let(:pipeline) { Pipeline.create!(partition_id: 100) } + let(:job) { Job.create!(pipeline: pipeline, partition_id: pipeline.partition_id) } + + it 'finds individual records using partition_id' do + find_statement = <<~SQL.squish + SELECT \"jobs\".* + FROM \"jobs\" + WHERE \"jobs\".\"pipeline_id\" = #{pipeline.id} + AND \"jobs\".\"partition_id\" = #{pipeline.partition_id} + AND \"jobs\".\"id\" = #{job.id} + LIMIT 1 + SQL + + result = QueryRecorder.log do + pipeline.jobs.find(job.id) + end + + expect(result).to include(find_statement) + end + + it 'finds all records using partition_id' do + find_statement = <<~SQL.squish + SELECT \"jobs\".* + FROM \"jobs\" + WHERE \"jobs\".\"pipeline_id\" = #{pipeline.id} + AND \"jobs\".\"partition_id\" = #{pipeline.partition_id} + SQL + + result = QueryRecorder.log do + pipeline.jobs.all.to_a + end + + expect(result).to include(find_statement) + end + + it 'jobs records using partition_id' do + build = pipeline.jobs.new(name: 'test job') + + expect(build.pipeline_id).to eq(pipeline.id) + expect(build.partition_id).to eq(pipeline.partition_id) + end + + it 'saves records using partition_id' do + create_statement = <<~SQL.squish + INSERT INTO \"jobs\" (\"pipeline_id\", \"partition_id\", \"name\") + VALUES (#{pipeline.id}, #{pipeline.partition_id}, 'test job') + SQL + + result = QueryRecorder.log do + build = pipeline.jobs.new(name: 'test job') + build.save! + end + + expect(result).to include(create_statement) + end + + it 'creates records using partition_id' do + create_statement = <<~SQL.squish + INSERT INTO \"jobs\" (\"pipeline_id\", \"partition_id\", \"name\") + VALUES (#{pipeline.id}, #{pipeline.partition_id}, 'test job') + SQL + + result = QueryRecorder.log do + pipeline.jobs.create!(name: 'test job') + end + + expect(result).to include(create_statement) + end + + it 'deletes_all records using partition_id' do + delete_statement = <<~SQL.squish + DELETE FROM \"jobs\" + WHERE \"jobs\".\"pipeline_id\" = #{pipeline.id} + AND \"jobs\".\"partition_id\" = #{pipeline.partition_id} + SQL + + result = QueryRecorder.log do + pipeline.jobs.delete_all + end + + expect(result).to include(delete_statement) + end + + it 'destroy_all records using partition_id' do + destroy_statement = <<~SQL.squish + DELETE FROM \"jobs\" + WHERE \"jobs\".\"id\" = #{job.id} + AND \"jobs\".\"partition_id\" = #{pipeline.partition_id} + SQL + + result = QueryRecorder.log do + pipeline.jobs.destroy_all # rubocop: disable Cop/DestroyAll + end + + expect(result).to include(destroy_statement) + end + + it 'counts records using partition_id' do + destroy_statement = <<~SQL.squish + SELECT COUNT(*) + FROM \"jobs\" + WHERE \"jobs\".\"pipeline_id\" = #{pipeline.id} + AND \"jobs\".\"partition_id\" = #{pipeline.partition_id} + SQL + + result = QueryRecorder.log do + pipeline.jobs.count + end + + expect(result).to include(destroy_statement) + end +end diff --git a/gems/activerecord-gitlab/spec/active_record/gitlab_patches/partitioning/associations/has_one_spec.rb b/gems/activerecord-gitlab/spec/active_record/gitlab_patches/partitioning/associations/has_one_spec.rb new file mode 100644 index 00000000000..aeb565c6dad --- /dev/null +++ b/gems/activerecord-gitlab/spec/active_record/gitlab_patches/partitioning/associations/has_one_spec.rb @@ -0,0 +1,101 @@ +# frozen_string_literal: true + +RSpec.describe 'ActiveRecord::GitlabPatches::Partitioning::Associations::HasOne', :partitioning do + let(:pipeline) { Pipeline.create!(partition_id: 100) } + let(:job) { Job.create!(pipeline: pipeline, partition_id: pipeline.partition_id) } + + it 'finds associated record using partition_id' do + find_statement = <<~SQL.squish + SELECT \"metadata\".* + FROM \"metadata\" + WHERE \"metadata\".\"job_id\" = #{job.id} + AND \"metadata\".\"partition_id\" = #{job.partition_id} + LIMIT 1 + SQL + + result = QueryRecorder.log do + job.reset.metadata + end + + expect(result).to include(find_statement) + end + + it 'builds records using partition_id' do + metadata = job.build_metadata + + expect(metadata.job_id).to eq(job.id) + expect(metadata.partition_id).to eq(job.partition_id) + end + + it 'saves records using partition_id' do + create_statement = <<~SQL.squish + INSERT INTO \"metadata\" (\"job_id\", \"partition_id\") VALUES (#{job.id}, #{job.partition_id}) + SQL + + result = QueryRecorder.log do + job.build_metadata.save! + end + + expect(result).to include(create_statement) + end + + it 'creates records using partition_id' do + create_statement = <<~SQL.squish + INSERT INTO \"metadata\" (\"job_id\", \"partition_id\") VALUES (#{job.id}, #{job.partition_id}) + SQL + + result = QueryRecorder.log do + job.create_metadata + end + + expect(result).to include(create_statement) + end + + it 'uses nested attributes on create' do + skip '`partitionable` will assign the `partition_id` value in this case.' + + statement1 = <<~SQL.squish + INSERT INTO \"jobs\" (\"pipeline_id\", \"partition_id\", \"name\") + VALUES (#{pipeline.id}, #{pipeline.partition_id}, 'test') + SQL + + statement2 = <<~SQL.squish + INSERT INTO \"metadata\" (\"job_id\", \"partition_id\", \"test_flag\") + VALUES (#{job.id}, #{job.partition_id}, 1) + SQL + + insert_statements = [statement1, statement2] + + result = QueryRecorder.log do + pipeline.jobs.create!(name: 'test', metadata_attributes: { test_flag: true }) + end + + insert_statements.each do |statement| + expect(result).to include(statement) + end + end + + it 'uses nested attributes on update' do + statement1 = <<~SQL.squish + UPDATE \"jobs\" SET \"name\" = 'other test' + WHERE \"jobs\".\"id\" = #{job.id} AND \"jobs\".\"partition_id\" = #{job.partition_id} + SQL + + statement2 = <<~SQL.squish + INSERT INTO \"metadata\" (\"job_id\", \"partition_id\", \"test_flag\") VALUES (#{job.id}, #{job.partition_id}, 1) + SQL + + update_statements = [statement1, statement2] + + job.name = 'other test' + job.metadata_attributes = { test_flag: true } + + result = QueryRecorder.log do + job.save! + end + + update_statements.each do |statement| + expect(result).to include(statement) + end + end +end diff --git a/gems/activerecord-gitlab/spec/active_record/gitlab_patches/partitioning/joins_spec.rb b/gems/activerecord-gitlab/spec/active_record/gitlab_patches/partitioning/joins_spec.rb new file mode 100644 index 00000000000..038fae43644 --- /dev/null +++ b/gems/activerecord-gitlab/spec/active_record/gitlab_patches/partitioning/joins_spec.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +RSpec.describe 'ActiveRecord::GitlabPatches::Partitioning::Associations::Joins', :partitioning do + let!(:pipeline) { Pipeline.create!(partition_id: 100) } + let!(:job) { Job.create!(pipeline: pipeline, partition_id: pipeline.partition_id) } + let!(:metadata) { Metadata.create!(job: job, partition_id: job.partition_id) } + + it 'joins using partition_id' do + join_statement = <<~SQL.squish + SELECT \"pipelines\".* + FROM \"pipelines\" + INNER JOIN \"jobs\" ON \"jobs\".\"pipeline_id\" = \"pipelines\".\"id\" + AND \"jobs\".\"partition_id\" = \"pipelines\".\"partition_id\" + WHERE \"pipelines\".\"partition_id\" = #{pipeline.partition_id} + SQL + + result = QueryRecorder.log do + Pipeline.where(partition_id: pipeline.partition_id).joins(:jobs).to_a + end + + expect(result).to include(join_statement) + end + + it 'joins other models using partition_id' do + join_statement = <<~SQL.squish + SELECT \"pipelines\".* + FROM \"pipelines\" + INNER JOIN \"jobs\" ON \"jobs\".\"pipeline_id\" = \"pipelines\".\"id\" + AND \"jobs\".\"partition_id\" = \"pipelines\".\"partition_id\" + INNER JOIN \"metadata\" ON \"metadata\".\"job_id\" = \"jobs\".\"id\" + AND \"metadata\".\"partition_id\" = \"jobs\".\"partition_id\" + WHERE \"pipelines\".\"partition_id\" = #{pipeline.partition_id} + SQL + + result = QueryRecorder.log do + Pipeline.where(partition_id: pipeline.partition_id).joins(jobs: :metadata).to_a + end + + expect(result).to include(join_statement) + end +end diff --git a/gems/activerecord-gitlab/spec/active_record/gitlab_patches/partitioning/preloads_spec.rb b/gems/activerecord-gitlab/spec/active_record/gitlab_patches/partitioning/preloads_spec.rb new file mode 100644 index 00000000000..f37a563fe9e --- /dev/null +++ b/gems/activerecord-gitlab/spec/active_record/gitlab_patches/partitioning/preloads_spec.rb @@ -0,0 +1,241 @@ +# frozen_string_literal: true + +RSpec.describe 'ActiveRecord::GitlabPatches::Partitioning::Associations::Preloads', :partitioning do + let(:project) { Project.create! } + + let!(:pipeline) { Pipeline.create!(project: project, partition_id: 100) } + let!(:other_pipeline) { Pipeline.create!(project: project, partition_id: 100) } + + let!(:job) { Job.create!(pipeline: pipeline, partition_id: pipeline.partition_id) } + let!(:other_job) { Job.create!(pipeline: pipeline, partition_id: pipeline.partition_id) } + + describe 'preload queries with single partition' do + it 'preloads metadata for jobs' do + statement1 = <<~SQL.squish + SELECT \"jobs\".* FROM \"jobs\" WHERE \"jobs\".\"partition_id\" = 100 + SQL + + statement2 = <<~SQL.squish + SELECT \"metadata\".* FROM \"metadata\" + WHERE \"metadata\".\"partition_id\" = 100 AND \"metadata\".\"job_id\" IN (#{job.id}, #{other_job.id}) + SQL + + preload_statements = [statement1, statement2] + + result = QueryRecorder.log do + Job.where(partition_id: 100).preload(:metadata).to_a + end + + preload_statements.each do |statement| + expect(result).to include(statement) + end + end + + it 'preloads jobs for pipelines' do + statement1 = <<~SQL.squish + SELECT \"pipelines\".* FROM \"pipelines\" WHERE \"pipelines\".\"partition_id\" = 100 + SQL + + statement2 = <<~SQL.squish + SELECT \"jobs\".* FROM \"jobs\" + WHERE \"jobs\".\"partition_id\" = 100 AND \"jobs\".\"pipeline_id\" IN (#{pipeline.id}, #{other_pipeline.id}) + SQL + + preload_statements = [statement1, statement2] + + result = QueryRecorder.log do + Pipeline.where(partition_id: 100).preload(:jobs).to_a + end + + preload_statements.each do |statement| + expect(result).to include(statement) + end + end + + it 'preloads jobs and metadata for pipelines' do + statement1 = <<~SQL.squish + SELECT \"pipelines\".* FROM \"pipelines\" WHERE \"pipelines\".\"partition_id\" = 100 + SQL + + statement2 = <<~SQL.squish + SELECT \"jobs\".* FROM \"jobs\" + WHERE \"jobs\".\"partition_id\" = 100 AND \"jobs\".\"pipeline_id\" IN (#{pipeline.id}, #{other_pipeline.id}) + SQL + + statement3 = <<~SQL.squish + SELECT \"metadata\".* FROM \"metadata\" + WHERE \"metadata\".\"partition_id\" = 100 AND \"metadata\".\"job_id\" IN (#{job.id}, #{other_job.id}) + SQL + + preload_statements = [statement1, statement2, statement3] + + result = QueryRecorder.log do + Pipeline.where(partition_id: 100).preload(jobs: :metadata).to_a + end + + preload_statements.each do |statement| + expect(result).to include(statement) + end + end + end + + describe 'preload queries with multiple partitions' do + let!(:recent_pipeline) { Pipeline.create!(project: project, partition_id: 200) } + let!(:test_job) { Job.create!(pipeline: recent_pipeline, partition_id: recent_pipeline.partition_id) } + let!(:deploy_job) { Job.create!(pipeline: recent_pipeline, partition_id: recent_pipeline.partition_id) } + + it 'preloads metadata for jobs' do + statement1 = <<~SQL.squish + SELECT \"jobs\".* FROM \"jobs\" WHERE \"jobs\".\"partition_id\" IN (100, 200) + SQL + + statement2 = <<~SQL.squish + SELECT \"metadata\".* FROM \"metadata\" + WHERE \"metadata\".\"partition_id\" = 100 AND \"metadata\".\"job_id\" IN (#{job.id}, #{other_job.id}) + SQL + + statement3 = <<~SQL.squish + SELECT \"jobs\".* FROM \"jobs\" WHERE \"jobs\".\"partition_id\" IN (100, 200) + SQL + + preload_statements = [statement1, statement2, statement3] + + result = QueryRecorder.log do + Job.where(partition_id: [100, 200]).preload(:metadata).to_a + end + + preload_statements.each do |statement| + expect(result).to include(statement) + end + end + + it 'preloads jobs for pipelines' do + statement1 = <<~SQL.squish + SELECT \"pipelines\".* FROM \"pipelines\" WHERE \"pipelines\".\"partition_id\" IN (100, 200) + SQL + + statement2 = <<~SQL.squish + SELECT \"jobs\".* FROM \"jobs\" + WHERE \"jobs\".\"partition_id\" = 100 AND \"jobs\".\"pipeline_id\" IN (#{pipeline.id}, #{other_pipeline.id}) + SQL + + statement3 = <<~SQL.squish + SELECT \"jobs\".* FROM \"jobs\" + WHERE \"jobs\".\"partition_id\" = 200 AND \"jobs\".\"pipeline_id\" = #{recent_pipeline.id} + SQL + + preload_statements = [statement1, statement2, statement3] + + result = QueryRecorder.log do + Pipeline.where(partition_id: [100, 200]).preload(:jobs).to_a + end + + preload_statements.each do |statement| + expect(result).to include(statement) + end + end + + it 'preloads jobs and metadata for pipelines' do + statement1 = <<~SQL.squish + SELECT \"pipelines\".* FROM \"pipelines\" WHERE \"pipelines\".\"partition_id\" IN (100, 200) + SQL + + statement2 = <<~SQL.squish + SELECT \"jobs\".* FROM \"jobs\" + WHERE \"jobs\".\"partition_id\" = 100 AND \"jobs\".\"pipeline_id\" IN (#{pipeline.id}, #{other_pipeline.id}) + SQL + + statement3 = <<~SQL.squish + SELECT \"jobs\".* FROM \"jobs\" + WHERE \"jobs\".\"partition_id\" = 200 AND \"jobs\".\"pipeline_id\" = #{recent_pipeline.id} + SQL + + statement4 = <<~SQL.squish + SELECT \"metadata\".* FROM \"metadata\" + WHERE \"metadata\".\"partition_id\" = 100 AND \"metadata\".\"job_id\" IN (#{job.id}, #{other_job.id}) + SQL + + statement5 = <<~SQL.squish + SELECT \"metadata\".* FROM \"metadata\" + WHERE \"metadata\".\"partition_id\" = 200 AND \"metadata\".\"job_id\" IN (#{test_job.id}, #{deploy_job.id}) + SQL + + preload_statements = [statement1, statement2, statement3, statement4, statement5] + + result = QueryRecorder.log do + Pipeline.where(partition_id: [100, 200]).preload(jobs: :metadata).to_a + end + + preload_statements.each do |statement| + expect(result).to include(statement) + end + end + end + + describe 'includes queries' do + it 'preloads data for pipeline with multiple queries' do + statement1 = <<~SQL.squish + SELECT \"pipelines\".* FROM \"pipelines\" + WHERE \"pipelines\".\"project_id\" = 1 AND \"pipelines\".\"id\" + IN (#{pipeline.id}, #{other_pipeline.id}) AND \"pipelines\".\"partition_id\" = 100 + SQL + + statement2 = <<~SQL.squish + SELECT \"jobs\".* FROM \"jobs\" + WHERE \"jobs\".\"partition_id\" = 100 AND \"jobs\".\"pipeline_id\" IN (#{pipeline.id}, #{other_pipeline.id}) + SQL + + statement3 = <<~SQL.squish + SELECT \"metadata\".* FROM \"metadata\" + WHERE \"metadata\".\"partition_id\" = 100 AND \"metadata\".\"job_id\" IN (#{job.id}, #{other_job.id}) + SQL + + preload_statements = [statement1, statement2, statement3] + + result = QueryRecorder.log do + project.pipelines.includes(jobs: :metadata).where(id: [pipeline.id, other_pipeline.id], partition_id: 100).to_a + end + + preload_statements.each do |statement| + expect(result).to include(statement) + end + end + + it 'preloads data for pipeline with join query' do + preload_statement = <<~SQL.squish + SELECT \"pipelines\".\"id\" + AS t0_r0, \"pipelines\".\"project_id\" + AS t0_r1, \"pipelines\".\"partition_id\" + AS t0_r2, \"jobs\".\"id\" + AS t1_r0, \"jobs\".\"pipeline_id\" + AS t1_r1, \"jobs\".\"partition_id\" + AS t1_r2, \"jobs\".\"name\" + AS t1_r3, \"metadata\".\"id\" + AS t2_r0, \"metadata\".\"job_id\" + AS t2_r1, \"metadata\".\"partition_id\" + AS t2_r2, \"metadata\".\"test_flag\" + AS t2_r3 + FROM \"pipelines\" + LEFT OUTER JOIN \"jobs\" ON \"jobs\".\"pipeline_id\" = \"pipelines\".\"id\" + AND \"jobs\".\"partition_id\" = \"pipelines\".\"partition_id\" + LEFT OUTER JOIN \"metadata\" ON \"metadata\".\"job_id\" = \"jobs\".\"id\" + AND \"metadata\".\"partition_id\" = \"jobs\".\"partition_id\" + WHERE \"pipelines\".\"project_id\" = 1 + AND \"pipelines\".\"id\" + IN (#{pipeline.id}, #{other_pipeline.id}) + AND \"pipelines\".\"partition_id\" = 100 + SQL + + result = QueryRecorder.log do + project + .pipelines + .includes(jobs: :metadata) + .references(:jobs, :metadata) + .where(id: [1, 2], partition_id: 100) + .to_a + end + + expect(result).to include(preload_statement) + end + end +end diff --git a/gems/activerecord-gitlab/spec/active_record/gitlab_patches/partitioning/single_model_queries_spec.rb b/gems/activerecord-gitlab/spec/active_record/gitlab_patches/partitioning/single_model_queries_spec.rb new file mode 100644 index 00000000000..b035d7a6277 --- /dev/null +++ b/gems/activerecord-gitlab/spec/active_record/gitlab_patches/partitioning/single_model_queries_spec.rb @@ -0,0 +1,110 @@ +# frozen_string_literal: true + +RSpec.describe 'ActiveRecord::GitlabPatches::Partitioning::Associations::SingleModelQueries', :partitioning do + let(:project) { Project.create! } + let(:pipeline) { Pipeline.create!(project: project, partition_id: 100) } + let(:job) { Job.create!(pipeline: pipeline, partition_id: pipeline.partition_id) } + + it 'creates using id and partition_id' do + create_statement = <<~SQL.squish + INSERT INTO \"jobs\" (\"pipeline_id\", \"partition_id\") + VALUES (#{pipeline.id}, #{pipeline.partition_id}) + SQL + + result = QueryRecorder.log do + Job.create!(pipeline_id: pipeline.id, partition_id: pipeline.partition_id) + end + + expect(result).to include(create_statement) + end + + it 'finds with id and partition_id' do + find_statement = <<~SQL.squish + SELECT \"jobs\".* + FROM \"jobs\" + WHERE \"jobs\".\"id\" = #{job.id} + AND \"jobs\".\"partition_id\" = #{job.partition_id} + LIMIT 1 + SQL + + result = QueryRecorder.log do + Job.find_by!(id: job.id, partition_id: job.partition_id) + end + + expect(result).to include(find_statement) + end + + it 'saves using id and partition_id' do + update_statement = <<~SQL.squish + UPDATE \"jobs\" + SET \"name\" = 'test' + WHERE \"jobs\".\"id\" = #{job.id} + AND \"jobs\".\"partition_id\" = #{job.partition_id} + SQL + + result = QueryRecorder.log do + job.name = 'test' + + job.save! + end + + expect(result).to include(update_statement) + end + + it 'updates using id and partition_id' do + update_statement = <<~SQL.squish + UPDATE \"jobs\" + SET \"name\" = 'test2' + WHERE \"jobs\".\"id\" = #{job.id} + AND \"jobs\".\"partition_id\" = #{job.partition_id} + SQL + + result = QueryRecorder.log do + job.update!(name: 'test2') + end + + expect(result).to include(update_statement) + end + + it 'deletes using id and partition_id' do + delete_statement = <<~SQL.squish + DELETE FROM \"jobs\" + WHERE \"jobs\".\"id\" = #{job.id} + AND \"jobs\".\"partition_id\" = #{job.partition_id} + SQL + + result = QueryRecorder.log do + job.delete + end + + expect(result).to include(delete_statement) + end + + it 'destroys using id and partition_id' do + destroy_statement = <<~SQL.squish + DELETE FROM \"jobs\" + WHERE \"jobs\".\"id\" = #{job.id} + AND \"jobs\".\"partition_id\" = #{job.partition_id} + SQL + + result = QueryRecorder.log do + job.destroy + end + + expect(result).to include(destroy_statement) + end + + it 'destroy_all using partition_id' do + destroy_statement = <<~SQL.squish + DELETE FROM \"jobs\" + WHERE \"jobs\".\"id\" = #{job.id} + AND \"jobs\".\"partition_id\" = #{job.partition_id} + SQL + + result = QueryRecorder.log do + Job.where(id: job.id).destroy_all # rubocop: disable Cop/DestroyAll + end + + expect(result).to include(destroy_statement) + end +end diff --git a/gems/activerecord-gitlab/spec/active_record/gitlab_patches/rescue_from_spec.rb b/gems/activerecord-gitlab/spec/active_record/gitlab_patches/rescue_from_spec.rb new file mode 100644 index 00000000000..c1537c3bd90 --- /dev/null +++ b/gems/activerecord-gitlab/spec/active_record/gitlab_patches/rescue_from_spec.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +RSpec.describe ActiveRecord::GitlabPatches::RescueFrom, :without_sqlite3 do + let(:model_with_rescue_from) do + Class.new(ActiveRecord::Base) do + rescue_from ActiveRecord::ConnectionNotEstablished, with: :handle_exception + + class << self + def handle_exception(exception); end + end + end + end + + let(:model_without_rescue_from) do + Class.new(ActiveRecord::Base) + end + + it 'triggers rescue_from' do + stub_const('ModelWithRescueFrom', model_with_rescue_from) + + expect(model_with_rescue_from).to receive(:handle_exception) + + expect { model_with_rescue_from.all.load }.not_to raise_error + end + + it 'does not trigger rescue_from' do + stub_const('ModelWithoutRescueFrom', model_without_rescue_from) + + expect { model_without_rescue_from.all.load }.to raise_error(ActiveRecord::ConnectionNotEstablished) + end +end diff --git a/gems/activerecord-gitlab/spec/active_record/gitlab_patches_spec.rb b/gems/activerecord-gitlab/spec/active_record/gitlab_patches_spec.rb new file mode 100644 index 00000000000..2e029a6adbf --- /dev/null +++ b/gems/activerecord-gitlab/spec/active_record/gitlab_patches_spec.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +RSpec.describe ActiveRecord::GitlabPatches do + it "has a version number" do + expect(described_class::Version::VERSION).not_to be nil + end +end diff --git a/gems/activerecord-gitlab/spec/spec_helper.rb b/gems/activerecord-gitlab/spec/spec_helper.rb new file mode 100644 index 00000000000..548295b3f2c --- /dev/null +++ b/gems/activerecord-gitlab/spec/spec_helper.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +ENV["RAILS_ENV"] = "test" + +require "active_record/gitlab_patches" + +RSpec.configure do |config| + # Enable flags like --only-failures and --next-failure + config.example_status_persistence_file_path = ".rspec_status" + + # Disable RSpec exposing methods globally on `Module` and `main` + config.disable_monkey_patching! + + config.expect_with :rspec do |c| + c.syntax = :expect + end + + Dir[File.expand_path("spec/support/**/*.rb")].each { |f| require f } + + config.around(:all, :partitioning) do |example| + ActiveRecord::Base.transaction do + example.run + + raise ActiveRecord::Rollback + end + end + + config.before(:all, :without_sqlite3) do + ActiveRecord::Base.remove_connection + end +end diff --git a/gems/activerecord-gitlab/spec/support/database.rb b/gems/activerecord-gitlab/spec/support/database.rb new file mode 100644 index 00000000000..998d945c311 --- /dev/null +++ b/gems/activerecord-gitlab/spec/support/database.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +RSpec.configure do |config| + config.before(:all) do + ActiveRecord::Base.establish_connection(adapter: "sqlite3", database: ":memory:") + ActiveRecord::Base.logger = Logger.new('/dev/null') + + ActiveRecord::Schema.define do + create_table :projects, force: true + + create_table :pipelines, force: true do |t| + t.integer :project_id + t.integer :partition_id + end + + create_table :jobs, force: true do |t| + t.integer :pipeline_id + t.integer :partition_id + t.string :name + end + + create_table :metadata, force: true do |t| + t.integer :job_id + t.integer :partition_id + t.boolean :test_flag, default: false + end + end + end +end diff --git a/gems/activerecord-gitlab/spec/support/models.rb b/gems/activerecord-gitlab/spec/support/models.rb new file mode 100644 index 00000000000..c0017656ea8 --- /dev/null +++ b/gems/activerecord-gitlab/spec/support/models.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +class PartitionedRecord < ActiveRecord::Base + self.abstract_class = true + + def self.use_partition_id_filter? + true + end + + alias_method :reset, :reload +end + +class Project < ActiveRecord::Base + has_many :pipelines +end + +class Pipeline < PartitionedRecord + belongs_to :project + query_constraints :id, :partition_id + + has_many :jobs, + ->(pipeline) { where(partition_id: pipeline.partition_id) }, + partition_foreign_key: :partition_id, + dependent: :destroy +end + +class Job < PartitionedRecord + query_constraints :id, :partition_id + + belongs_to :pipeline, + ->(build) { where(partition_id: build.partition_id) }, + partition_foreign_key: :partition_id + + has_one :metadata, + ->(build) { where(partition_id: build.partition_id) }, + foreign_key: :job_id, + partition_foreign_key: :partition_id, + inverse_of: :job, + autosave: true + + accepts_nested_attributes_for :metadata +end + +class Metadata < PartitionedRecord + self.table_name = :metadata + query_constraints :id, :partition_id + + belongs_to :job, + ->(metadata) { where(partition_id: metadata.partition_id) } +end diff --git a/gems/activerecord-gitlab/spec/support/query_recorder.rb b/gems/activerecord-gitlab/spec/support/query_recorder.rb new file mode 100644 index 00000000000..5129bae9240 --- /dev/null +++ b/gems/activerecord-gitlab/spec/support/query_recorder.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +class QueryRecorder + attr_reader :log + + def initialize(&block) + @log = [] + + ActiveRecord::Base.connection.unprepared_statement do + ActiveSupport::Notifications.subscribed(method(:callback), 'sql.active_record', &block) + end + end + + def callback(_name, _start, _finish, _message_id, values) + @log << values[:sql] + end + + def self.log(&block) + new(&block).log + end +end diff --git a/gems/click_house-client/.gitlab-ci.yml b/gems/click_house-client/.gitlab-ci.yml new file mode 100644 index 00000000000..0384da8a893 --- /dev/null +++ b/gems/click_house-client/.gitlab-ci.yml @@ -0,0 +1,4 @@ +include: + - local: gems/gem.gitlab-ci.yml + inputs: + gem_name: "click_house-client" diff --git a/gems/click_house-client/.rspec b/gems/click_house-client/.rspec new file mode 100644 index 00000000000..34c5164d9b5 --- /dev/null +++ b/gems/click_house-client/.rspec @@ -0,0 +1,3 @@ +--format documentation +--color +--require spec_helper diff --git a/gems/click_house-client/.rubocop.yml b/gems/click_house-client/.rubocop.yml new file mode 100644 index 00000000000..8c670b439d3 --- /dev/null +++ b/gems/click_house-client/.rubocop.yml @@ -0,0 +1,2 @@ +inherit_from: + - ../config/rubocop.yml diff --git a/gems/click_house-client/Gemfile b/gems/click_house-client/Gemfile new file mode 100644 index 00000000000..be173b205f7 --- /dev/null +++ b/gems/click_house-client/Gemfile @@ -0,0 +1,5 @@ +# frozen_string_literal: true + +source "https://rubygems.org" + +gemspec diff --git a/gems/click_house-client/Gemfile.lock b/gems/click_house-client/Gemfile.lock new file mode 100644 index 00000000000..a8e2b7ec0c6 --- /dev/null +++ b/gems/click_house-client/Gemfile.lock @@ -0,0 +1,102 @@ +PATH + remote: . + specs: + click_house-client (0.1.0) + activesupport (< 8) + addressable (~> 2.8) + json (~> 2.6.3) + +GEM + remote: https://rubygems.org/ + specs: + activesupport (7.0.6) + concurrent-ruby (~> 1.0, >= 1.0.2) + i18n (>= 1.6, < 2) + minitest (>= 5.1) + tzinfo (~> 2.0) + addressable (2.8.4) + public_suffix (>= 2.0.2, < 6.0) + ast (2.4.2) + concurrent-ruby (1.2.2) + diff-lcs (1.5.0) + gitlab-styles (10.1.0) + rubocop (~> 1.50.2) + rubocop-graphql (~> 0.18) + rubocop-performance (~> 1.15) + rubocop-rails (~> 2.17) + rubocop-rspec (~> 2.22) + i18n (1.14.1) + concurrent-ruby (~> 1.0) + json (2.6.3) + minitest (5.18.1) + parallel (1.23.0) + parser (3.2.2.3) + ast (~> 2.4.1) + racc + public_suffix (5.0.3) + racc (1.7.1) + rack (3.0.8) + rainbow (3.1.1) + rake (13.0.6) + regexp_parser (2.8.1) + rexml (3.2.5) + rspec (3.12.0) + rspec-core (~> 3.12.0) + rspec-expectations (~> 3.12.0) + rspec-mocks (~> 3.12.0) + rspec-core (3.12.2) + rspec-support (~> 3.12.0) + rspec-expectations (3.12.3) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.12.0) + rspec-mocks (3.12.5) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.12.0) + rspec-support (3.12.1) + rubocop (1.50.2) + json (~> 2.3) + parallel (~> 1.10) + parser (>= 3.2.0.0) + rainbow (>= 2.2.2, < 4.0) + regexp_parser (>= 1.8, < 3.0) + rexml (>= 3.2.5, < 4.0) + rubocop-ast (>= 1.28.0, < 2.0) + ruby-progressbar (~> 1.7) + unicode-display_width (>= 2.4.0, < 3.0) + rubocop-ast (1.29.0) + parser (>= 3.2.1.0) + rubocop-capybara (2.18.0) + rubocop (~> 1.41) + rubocop-factory_bot (2.23.1) + rubocop (~> 1.33) + rubocop-graphql (0.19.0) + rubocop (>= 0.87, < 2) + rubocop-performance (1.18.0) + rubocop (>= 1.7.0, < 2.0) + rubocop-ast (>= 0.4.0) + rubocop-rails (2.20.2) + activesupport (>= 4.2.0) + rack (>= 1.1) + rubocop (>= 1.33.0, < 2.0) + rubocop-rspec (2.22.0) + rubocop (~> 1.33) + rubocop-capybara (~> 2.17) + rubocop-factory_bot (~> 2.22) + ruby-progressbar (1.13.0) + tzinfo (2.0.6) + concurrent-ruby (~> 1.0) + unicode-display_width (2.4.2) + +PLATFORMS + ruby + +DEPENDENCIES + click_house-client! + gitlab-styles (~> 10.1.0) + rake (~> 13.0) + rspec (~> 3.0) + rubocop + rubocop-rspec + +BUNDLED WITH + 2.4.16 diff --git a/gems/click_house-client/README.md b/gems/click_house-client/README.md new file mode 100644 index 00000000000..6afabcad1a0 --- /dev/null +++ b/gems/click_house-client/README.md @@ -0,0 +1,3 @@ +ClickHouse::Client + +This Gem provides a simple way to query ClickHouse databases using the HTTP interface. diff --git a/gems/click_house-client/click_house-client.gemspec b/gems/click_house-client/click_house-client.gemspec new file mode 100644 index 00000000000..5544065ef17 --- /dev/null +++ b/gems/click_house-client/click_house-client.gemspec @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +Gem::Specification.new do |spec| + spec.name = "click_house-client" + spec.version = "0.1.0" + spec.authors = ["group::optimize"] + spec.email = ["engineering@gitlab.com"] + + spec.summary = "GitLab's client to interact with ClickHouse" + spec.description = "This Gem provides a simple way to query ClickHouse databases using the HTTP interface." + spec.homepage = "https://gitlab.com/gitlab-org/gitlab/-/tree/master/gems/click_house-client" + spec.license = "MIT" + spec.required_ruby_version = ">= 3.0" + + spec.add_runtime_dependency "activesupport", "< 8" + spec.add_runtime_dependency "addressable", "~> 2.8" + spec.add_runtime_dependency 'json', '~> 2.6.3' + + spec.add_development_dependency 'gitlab-styles', '~> 10.1.0' + spec.add_development_dependency "rake", "~> 13.0" + spec.add_development_dependency "rspec", "~> 3.0" + spec.add_development_dependency 'rubocop' + spec.add_development_dependency 'rubocop-rspec' +end diff --git a/gems/click_house-client/lib/click_house/client.rb b/gems/click_house-client/lib/click_house/client.rb new file mode 100644 index 00000000000..22c42d7be6e --- /dev/null +++ b/gems/click_house-client/lib/click_house/client.rb @@ -0,0 +1,64 @@ +# frozen_string_literal: true + +require 'addressable' +require 'json' +require 'active_support/time' +require_relative "client/database" +require_relative "client/configuration" +require_relative "client/formatter" +require_relative "client/response" + +module ClickHouse + module Client + class << self + def configuration + @configuration ||= Configuration.new + end + + def configure + yield(configuration) + configuration.validate! + end + end + + Error = Class.new(StandardError) + ConfigurationError = Class.new(Error) + DatabaseError = Class.new(Error) + + # Executes a SELECT database query + def self.select(query, database, configuration = self.configuration) + db = lookup_database(configuration, database) + + response = configuration.http_post_proc.call( + db.uri.to_s, + db.headers, + "#{query} FORMAT JSON" # always return JSON + ) + + raise DatabaseError, response.body unless response.success? + + Formatter.format(configuration.json_parser.parse(response.body)) + end + + # Executes any kinds of database query without returning any data (INSERT, DELETE) + def self.execute(query, database, configuration = self.configuration) + db = lookup_database(configuration, database) + + response = configuration.http_post_proc.call( + db.uri.to_s, + db.headers, + query + ) + + raise DatabaseError, response.body unless response.success? + + true + end + + private_class_method def self.lookup_database(configuration, database) + configuration.databases[database].tap do |db| + raise ConfigurationError, "The database '#{database}' is not configured" unless db + end + end + end +end diff --git a/gems/click_house-client/lib/click_house/client/configuration.rb b/gems/click_house-client/lib/click_house/client/configuration.rb new file mode 100644 index 00000000000..882b37993dc --- /dev/null +++ b/gems/click_house-client/lib/click_house/client/configuration.rb @@ -0,0 +1,68 @@ +# frozen_string_literal: true + +module ClickHouse + module Client + class Configuration + # Configuration options: + # + # *register_database* (method): registers a database, the following arguments are required: + # - database: database name + # - url: URL and port to the HTTP interface + # - username + # - password + # - variables (optional): configuration for the client + # + # *http_post_proc*: A callable object for invoking the HTTP request. + # The object must handle the following parameters: url, headers, body + # and return a Gitlab::ClickHouse::Client::Response object. + # + # *json_parser*: object for parsing JSON strings, it should respond to the "parse" method + # + # Example: + # + # Gitlab::ClickHouse::Client.configure do |c| + # c.register_database(:main, + # database: 'gitlab_clickhouse_test', + # url: 'http://localhost:8123', + # username: 'default', + # password: 'clickhouse', + # variables: { + # join_use_nulls: 1 # treat JOINs as per SQL standard + # } + # ) + # + # c.http_post_proc = lambda do |url, headers, body| + # options = { + # headers: headers, + # body: body, + # allow_local_requests: false + # } + # + # response = Gitlab::HTTP.post(url, options) + # Gitlab::ClickHouse::Client::Response.new(response.body, response.code) + # end + # + # c.json_parser = JSON + # end + attr_accessor :http_post_proc, :json_parser + attr_reader :databases + + def initialize + @databases = {} + @http_post_proc = nil + @json_parser = JSON + end + + def register_database(name, **args) + raise ConfigurationError, "The database '#{name}' is already registered" if @databases.key?(name) + + @databases[name] = Database.new(**args) + end + + def validate! + raise ConfigurationError, "The 'http_post_proc' option is not configured" unless @http_post_proc + raise ConfigurationError, "The 'json_parser' option is not configured" unless @json_parser + end + end + end +end diff --git a/gems/click_house-client/lib/click_house/client/database.rb b/gems/click_house-client/lib/click_house/client/database.rb new file mode 100644 index 00000000000..beeb2a8cbd6 --- /dev/null +++ b/gems/click_house-client/lib/click_house/client/database.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module ClickHouse + module Client + class Database + attr_reader :database + + def initialize(database:, url:, username:, password:, variables: {}) + @database = database + @url = url + @username = username + @password = password + @variables = variables.merge(database: database).freeze + end + + def uri + @uri ||= begin + parsed = Addressable::URI.parse(@url) + parsed.query_values = @variables + parsed + end + end + + def headers + @headers ||= { + 'X-ClickHouse-User' => @username, + 'X-ClickHouse-Key' => @password + }.freeze + end + end + end +end diff --git a/gems/click_house-client/lib/click_house/client/formatter.rb b/gems/click_house-client/lib/click_house/client/formatter.rb new file mode 100644 index 00000000000..bb60d8db7f7 --- /dev/null +++ b/gems/click_house-client/lib/click_house/client/formatter.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module ClickHouse + module Client + class Formatter + DEFAULT = ->(value) { value } + + TYPE_CASTERS = { + 'UInt64' => ->(value) { Integer(value) }, + "DateTime64(6, 'UTC')" => ->(value) { ActiveSupport::TimeZone["UTC"].parse(value) } + }.freeze + + def self.format(result) + name_type_mapping = result['meta'].each_with_object({}) do |column, hash| + hash[column['name']] = column['type'] + end + + result['data'].map do |row| + row.each_with_object({}) do |(column, value), casted_row| + caster = TYPE_CASTERS.fetch(name_type_mapping[column], DEFAULT) + + casted_row[column] = caster.call(value) + end + end + end + end + end +end diff --git a/gems/click_house-client/lib/click_house/client/response.rb b/gems/click_house-client/lib/click_house/client/response.rb new file mode 100644 index 00000000000..898f0b0e024 --- /dev/null +++ b/gems/click_house-client/lib/click_house/client/response.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module ClickHouse + module Client + class Response + attr_reader :body + + def initialize(body, http_status_code) + @body = body + @http_status_code = http_status_code + end + + def success? + @http_status_code == 200 + end + end + end +end diff --git a/gems/click_house-client/spec/click_house/client/configuration_spec.rb b/gems/click_house-client/spec/click_house/client/configuration_spec.rb new file mode 100644 index 00000000000..8cbd64ca650 --- /dev/null +++ b/gems/click_house-client/spec/click_house/client/configuration_spec.rb @@ -0,0 +1,66 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe ClickHouse::Client::Configuration do + subject(:configuration) do + config = described_class.new + config.http_post_proc = -> {} + config.json_parser = Object + config + end + + describe '#register_database' do + let(:database_options) do + { + database: 'test_db', + url: 'http://localhost:3333', + username: 'user', + password: 'pass', + variables: { + join_use_nulls: 1 + } + } + end + + it 'registers a database' do + configuration.register_database(:my_db, **database_options) + + expect(configuration.databases.size).to eq(1) + database = configuration.databases[:my_db] + + expect(database.uri.to_s).to eq('http://localhost:3333?database=test_db&join_use_nulls=1') + end + + context 'when adding the same DB multiple times' do + it 'raises error' do + configuration.register_database(:my_db, **database_options) + expect do + configuration.register_database(:my_db, **database_options) + end.to raise_error(ClickHouse::Client::ConfigurationError, /database 'my_db' is already registered/) + end + end + end + + describe '#validate!' do + context 'when `http_post_proc` option is not configured' do + it 'raises error' do + configuration.http_post_proc = nil + + expect do + configuration.validate! + end.to raise_error(ClickHouse::Client::ConfigurationError, /'http_post_proc' option is not configured/) + end + end + + context 'when `json_parser` option is not configured' do + it 'raises error' do + configuration.json_parser = nil + + expect do + configuration.validate! + end.to raise_error(ClickHouse::Client::ConfigurationError, /'json_parser' option is not configured/) + end + end + end +end diff --git a/gems/click_house-client/spec/click_house/client/database_spec.rb b/gems/click_house-client/spec/click_house/client/database_spec.rb new file mode 100644 index 00000000000..112b2ee12b1 --- /dev/null +++ b/gems/click_house-client/spec/click_house/client/database_spec.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe ClickHouse::Client::Database do + subject(:database) do + described_class.new( + database: 'test_db', + url: 'http://localhost:3333', + username: 'user', + password: 'pass', + variables: { + join_use_nulls: 1 + } + ) + end + + describe '#uri' do + it 'builds the correct URL' do + expect(database.uri.to_s).to eq('http://localhost:3333?database=test_db&join_use_nulls=1') + end + end + + describe '#headers' do + it 'returns the correct headers' do + expect(database.headers).to eq({ + 'X-ClickHouse-User' => 'user', + 'X-ClickHouse-Key' => 'pass' + }) + end + end +end diff --git a/gems/click_house-client/spec/click_house/client_spec.rb b/gems/click_house-client/spec/click_house/client_spec.rb new file mode 100644 index 00000000000..883199198ba --- /dev/null +++ b/gems/click_house-client/spec/click_house/client_spec.rb @@ -0,0 +1,98 @@ +# frozen_string_literal: true + +RSpec.describe ClickHouse::Client do + describe '#select' do + # Assuming we have a DB table with the following schema + # + # CREATE TABLE issues ( + # `id` UInt64, + # `title` String DEFAULT '', + # `description` Nullable(String), + # `created_at` DateTime64(6, 'UTC') DEFAULT now(), + # `updated_at` DateTime64(6, 'UTC') DEFAULT now() + # ) + # ENGINE = ReplacingMergeTree(updated_at) + # ORDER BY (id) + + let(:query_result_fixture) { File.expand_path('../fixtures/query_result.json', __dir__) } + + let(:database_config) do + { + database: 'test_db', + url: 'http://localhost:3333', + username: 'user', + password: 'pass', + variables: { + join_use_nulls: 1 + } + } + end + + let(:configuration) do + ClickHouse::Client::Configuration.new.tap do |config| + config.register_database(:test_db, **database_config) + config.http_post_proc = ->(_url, _headers, _query) { + body = File.read(query_result_fixture) + ClickHouse::Client::Response.new(body, 200) + } + end + end + + it 'parses the results and returns the data as array of hashes' do + result = described_class.select('SELECT * FROM issues', :test_db, configuration) + + timestamp1 = ActiveSupport::TimeZone["UTC"].parse('2023-06-21 13:33:44') + timestamp2 = ActiveSupport::TimeZone["UTC"].parse('2023-06-21 13:33:50') + timestamp3 = ActiveSupport::TimeZone["UTC"].parse('2023-06-21 13:33:40') + + expect(result).to eq([ + { + 'id' => 2, + 'title' => 'Title 2', + 'description' => 'description', + 'created_at' => timestamp1, + 'updated_at' => timestamp1 + }, + { + 'id' => 3, + 'title' => 'Title 3', + 'description' => nil, + 'created_at' => timestamp2, + 'updated_at' => timestamp2 + }, + { + 'id' => 1, + 'title' => 'Title 1', + 'description' => 'description', + 'created_at' => timestamp3, + 'updated_at' => timestamp3 + } + ]) + end + + context 'when the DB is not configured' do + it 'raises erro' do + expect do + described_class.select('SELECT * FROM issues', :different_db, configuration) + end.to raise_error(ClickHouse::Client::ConfigurationError, /not configured/) + end + end + + context 'when error response is returned' do + let(:configuration) do + ClickHouse::Client::Configuration.new.tap do |config| + config.register_database(:test_db, **database_config) + config.http_post_proc = ->(_url, _headers, _query) { + ClickHouse::Client::Response.new('some error', 404) + } + end + end + + it 'raises error' do + expect do + described_class.select('SELECT * FROM issues', :test_db, configuration) + end.to raise_error(ClickHouse::Client::DatabaseError, 'some error') + end + end + end +end diff --git a/gems/click_house-client/spec/fixtures/query_result.json b/gems/click_house-client/spec/fixtures/query_result.json new file mode 100644 index 00000000000..872b0e5b6ed --- /dev/null +++ b/gems/click_house-client/spec/fixtures/query_result.json @@ -0,0 +1,53 @@ +{ + "meta": [ + { + "name": "id", + "type": "UInt64" + }, + { + "name": "title", + "type:": "String" + }, + { + "name": "description", + "type": "Nullable(String)" + }, + { + "name": "created_at", + "type": "DateTime64(6, 'UTC')" + }, + { + "name": "updated_at", + "type": "DateTime64(6, 'UTC')" + } + ], + "data": [ + { + "id": "2", + "title": "Title 2", + "description": "description", + "created_at": "2023-06-21 13:33:44.000000", + "updated_at": "2023-06-21 13:33:44.000000" + }, + { + "id": "3", + "title": "Title 3", + "description": null, + "created_at": "2023-06-21 13:33:50.000000", + "updated_at": "2023-06-21 13:33:50.000000" + }, + { + "id": "1", + "title": "Title 1", + "description": "description", + "created_at": "2023-06-21 13:33:40.000000", + "updated_at": "2023-06-21 13:33:40.000000" + } + ], + "rows": 3, + "statistics": { + "elapsed": 0.001581789, + "rows_read": 3, + "bytes_read": 172 + } +} diff --git a/gems/click_house-client/spec/spec_helper.rb b/gems/click_house-client/spec/spec_helper.rb new file mode 100644 index 00000000000..d2b5e59c9a3 --- /dev/null +++ b/gems/click_house-client/spec/spec_helper.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +require "click_house/client" + +RSpec.configure do |config| + # Enable flags like --only-failures and --next-failure + config.example_status_persistence_file_path = ".rspec_status" + + # Disable RSpec exposing methods globally on `Module` and `main` + config.disable_monkey_patching! + + config.expect_with :rspec do |c| + c.syntax = :expect + end +end diff --git a/gems/config/rubocop.yml b/gems/config/rubocop.yml new file mode 100644 index 00000000000..a97d759b471 --- /dev/null +++ b/gems/config/rubocop.yml @@ -0,0 +1,98 @@ +inherit_gem: + gitlab-styles: + - rubocop-default.yml + +require: + - ../../rubocop/rubocop + - rubocop-rspec + +inherit_mode: + merge: + - Include + - Exclude + - AllowedPatterns + +AllCops: + # Target the current Ruby version. For example, "3.0" or "3.1". + TargetRubyVersion: <%= RUBY_VERSION[/^\d+\.\d+/, 0] %> + SuggestExtensions: false + NewCops: disable + +# This cop doesn't make sense in the context of gems +CodeReuse/ActiveRecord: + Enabled: false + +# This cop doesn't make sense in the context of gems +Cop/PutGroupRoutesUnderScope: + Enabled: false + +# This cop doesn't make sense in the context of gems +Cop/PutProjectRoutesUnderScope: + Enabled: false + +Gemspec/AvoidExecutingGit: + Enabled: true + +# We disable this since we support multiple Ruby versions +Gemspec/RequiredRubyVersion: + Enabled: false + +# This cop doesn't make sense in the context of gems +Gitlab/DocUrl: + Enabled: false + +# This cop doesn't make sense in the context of gems +Gitlab/NamespacedClass: + Enabled: false + +# This cop doesn't make sense in the context of gems +Gitlab/RSpec/AvoidSetup: + Enabled: false + +# This cop doesn't make sense in the context of gems +Graphql/AuthorizeTypes: + Enabled: false + +# This cop doesn't make sense in the context of gems +Graphql/Descriptions: + Enabled: false + +Naming/FileName: + Exclude: + - spec/**/*.rb + +# This cop doesn't make sense in the context of gems +RSpec/AvoidConditionalStatements: + Enabled: false + +RSpec/ContextWording: + Prefixes: + - 'when' + - 'with' + - 'without' + - 'for' + - 'and' + - 'on' + - 'in' + - 'as' + - 'if' + +# This cop doesn't make sense in the context of gems +RSpec/MissingFeatureCategory: + Enabled: false + +# Enable once we drop 3.0 support +Style/HashSyntax: + Enabled: false + +Style/Lambda: + EnforcedStyle: literal + +Style/RegexpLiteralMixedPreserve: + Enabled: true + SupportedStyles: + - slashes + - percent_r + - mixed + - mixed_preserve + EnforcedStyle: mixed_preserve diff --git a/gems/gem.gitlab-ci.yml b/gems/gem.gitlab-ci.yml new file mode 100644 index 00000000000..10905d8c243 --- /dev/null +++ b/gems/gem.gitlab-ci.yml @@ -0,0 +1,66 @@ +# The template generates jobs for gems vendored in the main GitLab project +# under `gem_path_prefix` (defaults to `gems/`). +# +# Inputs +# - `gem_name`: The name of the gem, i.e. if the gem is located at `gems/gitlab-rspec`, `gem_name` should be set to `gitlab-rspec`. +# - `gem_path_prefix`: The prefix of the gem path, i.e. if the gem is located at `vendor/gems/gitlab-rspec`, `gem_path_prefix` should be set to `vendor/gems/`. Defaults to `gems/`. +spec: + inputs: + gem_name: + gem_path_prefix: + default: "gems/" +--- +workflow: + name: '$PIPELINE_NAME' + rules: + - if: $CI_MERGE_REQUEST_ID + variables: + PIPELINE_NAME: '[$[[inputs.gem_name]] gem] Ruby $RUBY_VERSION $CI_MERGE_REQUEST_EVENT_TYPE MR pipeline' + +variables: + BUNDLE_PATH: "vendor" + BUNDLE_FROZEN: "true" + GIT_DEPTH: "20" + # 'GIT_STRATEGY: clone' optimizes the pack-objects cache hit ratio + GIT_STRATEGY: "clone" + GIT_SUBMODULE_STRATEGY: "none" + GET_SOURCES_ATTEMPTS: "3" + # Default Ruby version for jobs that don't use .ruby_matrix + RUBY_VERSION: "3.0" + +default: + image: "ruby:${RUBY_VERSION}" + cache: + key: "$[[inputs.gem_name]]-${RUBY_VERSION}" + paths: + - "$[[inputs.gem_path_prefix]]$[[inputs.gem_name]]/vendor/ruby" + before_script: + - cd $[[inputs.gem_path_prefix]]$[[inputs.gem_name]] + - ruby -v # Print out ruby version for debugging + - bundle_version=$(grep -A 1 "BUNDLED WITH" Gemfile.lock | tail -n 1 | sed -e 's/[[:space:]]//') + - gem install bundler --version "$bundle_version" --no-document # Bundler is not installed with the image + - bundle config # Show bundler configuration + - bundle install --jobs=$(nproc) --retry=3 + +.ruby_matrix: + parallel: + matrix: + - RUBY_VERSION: ["3.0", "3.1", "3.2"] + +rubocop: + extends: .ruby_matrix + rules: + - exists: ["$[[inputs.gem_path_prefix]]$[[inputs.gem_name]]/.rubocop.yml"] + script: + - bundle exec rubocop --config .rubocop.yml + +rspec: + extends: .ruby_matrix + script: + - bundle exec rspec + coverage: '/LOC \((\d+\.\d+%)\) covered.$/' + artifacts: + expire_in: 31d + when: always + paths: + - coverage/ diff --git a/gems/gitlab-rspec/.gitignore b/gems/gitlab-rspec/.gitignore new file mode 100644 index 00000000000..b04a8c840df --- /dev/null +++ b/gems/gitlab-rspec/.gitignore @@ -0,0 +1,11 @@ +/.bundle/ +/.yardoc +/_yardoc/ +/coverage/ +/doc/ +/pkg/ +/spec/reports/ +/tmp/ + +# rspec failure tracking +.rspec_status diff --git a/gems/gitlab-rspec/.gitlab-ci.yml b/gems/gitlab-rspec/.gitlab-ci.yml new file mode 100644 index 00000000000..910b391a0e9 --- /dev/null +++ b/gems/gitlab-rspec/.gitlab-ci.yml @@ -0,0 +1,4 @@ +include: + - local: gems/gem.gitlab-ci.yml + inputs: + gem_name: "gitlab-rspec" diff --git a/gems/gitlab-rspec/.rspec b/gems/gitlab-rspec/.rspec new file mode 100644 index 00000000000..34c5164d9b5 --- /dev/null +++ b/gems/gitlab-rspec/.rspec @@ -0,0 +1,3 @@ +--format documentation +--color +--require spec_helper diff --git a/gems/gitlab-rspec/.rubocop.yml b/gems/gitlab-rspec/.rubocop.yml new file mode 100644 index 00000000000..38c0c592dad --- /dev/null +++ b/gems/gitlab-rspec/.rubocop.yml @@ -0,0 +1,10 @@ +inherit_from: + - ../config/rubocop.yml + +RSpec/InstanceVariable: + Exclude: + - spec/**/*.rb + +Gitlab/ChangeTimezone: + Exclude: + - spec/gitlab/rspec/time_travel_spec.rb diff --git a/gems/gitlab-rspec/Gemfile b/gems/gitlab-rspec/Gemfile new file mode 100644 index 00000000000..be173b205f7 --- /dev/null +++ b/gems/gitlab-rspec/Gemfile @@ -0,0 +1,5 @@ +# frozen_string_literal: true + +source "https://rubygems.org" + +gemspec diff --git a/gems/gitlab-rspec/Gemfile.lock b/gems/gitlab-rspec/Gemfile.lock new file mode 100644 index 00000000000..dcdb4dd009e --- /dev/null +++ b/gems/gitlab-rspec/Gemfile.lock @@ -0,0 +1,182 @@ +PATH + remote: . + specs: + gitlab-rspec (0.1.0) + activesupport (>= 6.1, < 7.1) + rspec (~> 3.0) + +GEM + remote: https://rubygems.org/ + specs: + actionpack (7.0.4.3) + actionview (= 7.0.4.3) + activesupport (= 7.0.4.3) + rack (~> 2.0, >= 2.2.0) + rack-test (>= 0.6.3) + rails-dom-testing (~> 2.0) + rails-html-sanitizer (~> 1.0, >= 1.2.0) + actionview (7.0.4.3) + activesupport (= 7.0.4.3) + builder (~> 3.1) + erubi (~> 1.4) + rails-dom-testing (~> 2.0) + rails-html-sanitizer (~> 1.1, >= 1.2.0) + activesupport (7.0.4.3) + concurrent-ruby (~> 1.0, >= 1.0.2) + i18n (>= 1.6, < 2) + minitest (>= 5.1) + tzinfo (~> 2.0) + ast (2.4.2) + benchmark-malloc (0.2.0) + benchmark-perf (0.6.0) + benchmark-trend (0.4.0) + binding_of_caller (1.0.0) + debug_inspector (>= 0.0.1) + builder (3.2.4) + coderay (1.1.3) + concurrent-ruby (1.2.2) + crass (1.0.6) + debug_inspector (1.1.0) + diff-lcs (1.5.0) + erubi (1.12.0) + factory_bot (6.2.0) + activesupport (>= 5.0.0) + factory_bot_rails (6.2.0) + factory_bot (~> 6.2.0) + railties (>= 5.0.0) + gitlab-styles (10.1.0) + rubocop (~> 1.50.2) + rubocop-graphql (~> 0.18) + rubocop-performance (~> 1.15) + rubocop-rails (~> 2.17) + rubocop-rspec (~> 2.22) + i18n (1.13.0) + concurrent-ruby (~> 1.0) + json (2.6.3) + loofah (2.21.3) + crass (~> 1.0.2) + nokogiri (>= 1.12.0) + method_source (1.0.0) + mini_portile2 (2.8.2) + minitest (5.18.0) + nokogiri (1.15.2) + mini_portile2 (~> 2.8.2) + racc (~> 1.4) + parallel (1.23.0) + parser (3.2.2.1) + ast (~> 2.4.1) + proc_to_ast (0.1.0) + coderay + parser + unparser + racc (1.6.2) + rack (2.2.7) + rack-test (2.1.0) + rack (>= 1.3) + rails-dom-testing (2.0.3) + activesupport (>= 4.2.0) + nokogiri (>= 1.6) + rails-html-sanitizer (1.5.0) + loofah (~> 2.19, >= 2.19.1) + railties (7.0.4.3) + actionpack (= 7.0.4.3) + activesupport (= 7.0.4.3) + method_source + rake (>= 12.2) + thor (~> 1.0) + zeitwerk (~> 2.5) + rainbow (3.1.1) + rake (13.0.6) + regexp_parser (2.8.1) + rexml (3.2.5) + rspec (3.12.0) + rspec-core (~> 3.12.0) + rspec-expectations (~> 3.12.0) + rspec-mocks (~> 3.12.0) + rspec-benchmark (0.6.0) + benchmark-malloc (~> 0.2) + benchmark-perf (~> 0.6) + benchmark-trend (~> 0.4) + rspec (>= 3.0) + rspec-core (3.12.0) + rspec-support (~> 3.12.0) + rspec-expectations (3.12.2) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.12.0) + rspec-mocks (3.12.3) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.12.0) + rspec-parameterized (1.0.0) + rspec-parameterized-core (< 2) + rspec-parameterized-table_syntax (< 2) + rspec-parameterized-core (1.0.0) + parser + proc_to_ast + rspec (>= 2.13, < 4) + unparser + rspec-parameterized-table_syntax (1.0.0) + binding_of_caller + rspec-parameterized-core (< 2) + rspec-rails (6.0.2) + actionpack (>= 6.1) + activesupport (>= 6.1) + railties (>= 6.1) + rspec-core (~> 3.12) + rspec-expectations (~> 3.12) + rspec-mocks (~> 3.12) + rspec-support (~> 3.12) + rspec-support (3.12.0) + rubocop (1.50.2) + json (~> 2.3) + parallel (~> 1.10) + parser (>= 3.2.0.0) + rainbow (>= 2.2.2, < 4.0) + regexp_parser (>= 1.8, < 3.0) + rexml (>= 3.2.5, < 4.0) + rubocop-ast (>= 1.28.0, < 2.0) + ruby-progressbar (~> 1.7) + unicode-display_width (>= 2.4.0, < 3.0) + rubocop-ast (1.29.0) + parser (>= 3.2.1.0) + rubocop-capybara (2.18.0) + rubocop (~> 1.41) + rubocop-factory_bot (2.23.1) + rubocop (~> 1.33) + rubocop-graphql (0.19.0) + rubocop (>= 0.87, < 2) + rubocop-performance (1.18.0) + rubocop (>= 1.7.0, < 2.0) + rubocop-ast (>= 0.4.0) + rubocop-rails (2.20.2) + activesupport (>= 4.2.0) + rack (>= 1.1) + rubocop (>= 1.33.0, < 2.0) + rubocop-rspec (2.22.0) + rubocop (~> 1.33) + rubocop-capybara (~> 2.17) + rubocop-factory_bot (~> 2.22) + ruby-progressbar (1.13.0) + thor (1.2.2) + tzinfo (2.0.6) + concurrent-ruby (~> 1.0) + unicode-display_width (2.4.2) + unparser (0.6.7) + diff-lcs (~> 1.3) + parser (>= 3.2.0) + zeitwerk (2.6.8) + +PLATFORMS + ruby + +DEPENDENCIES + factory_bot_rails (~> 6.2.0) + gitlab-rspec! + gitlab-styles (~> 10.1.0) + rspec-benchmark (~> 0.6.0) + rspec-parameterized (~> 1.0) + rspec-rails (~> 6.0.1) + rubocop (~> 1.50) + rubocop-rspec (~> 2.22) + +BUNDLED WITH + 2.4.4 diff --git a/gems/gitlab-rspec/gitlab-rspec.gemspec b/gems/gitlab-rspec/gitlab-rspec.gemspec new file mode 100644 index 00000000000..c2c5b6c60b7 --- /dev/null +++ b/gems/gitlab-rspec/gitlab-rspec.gemspec @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +require_relative "lib/gitlab/rspec/version" + +Gem::Specification.new do |spec| + spec.name = "gitlab-rspec" + spec.version = Gitlab::Rspec::Version::VERSION + spec.authors = ["group::tenant-scale"] + spec.email = ["engineering@gitlab.com"] + + spec.summary = "GitLab RSpec extensions" + spec.description = "A set of useful helpers to configure RSpec with various stubs and CI configs." + spec.homepage = "https://gitlab.com/gitlab-org/gitlab/-/tree/master/gems/gitlab-rspec" + spec.license = "MIT" + spec.required_ruby_version = ">= 3.0" + spec.metadata["rubygems_mfa_required"] = "true" + + spec.files = Dir["lib/**/*.rb"] + spec.require_paths = ["lib"] + + spec.add_runtime_dependency "activesupport", ">= 6.1", "< 7.1" + spec.add_runtime_dependency "rspec", "~> 3.0" + + spec.add_development_dependency "factory_bot_rails", "~> 6.2.0" + spec.add_development_dependency "gitlab-styles", "~> 10.1.0" + spec.add_development_dependency "rspec-benchmark", "~> 0.6.0" + spec.add_development_dependency "rspec-parameterized", "~> 1.0" + spec.add_development_dependency "rspec-rails", "~> 6.0.1" + spec.add_development_dependency "rubocop", "~> 1.50" + spec.add_development_dependency "rubocop-rspec", "~> 2.22" +end diff --git a/gems/gitlab-rspec/lib/gitlab/rspec.rb b/gems/gitlab-rspec/lib/gitlab/rspec.rb new file mode 100644 index 00000000000..2d076e99614 --- /dev/null +++ b/gems/gitlab-rspec/lib/gitlab/rspec.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +require "rspec" +require_relative "rspec/version" + +module Gitlab + module Rspec + end +end diff --git a/gems/gitlab-rspec/lib/gitlab/rspec/all.rb b/gems/gitlab-rspec/lib/gitlab/rspec/all.rb new file mode 100644 index 00000000000..091d2ba0287 --- /dev/null +++ b/gems/gitlab-rspec/lib/gitlab/rspec/all.rb @@ -0,0 +1,8 @@ +# frozen_string_literal: true + +require_relative "../rspec" +require_relative "stub_env" + +require_relative "configurations/time_travel" + +Gitlab::Rspec::Configurations::TimeTravel.configure! diff --git a/gems/gitlab-rspec/lib/gitlab/rspec/configurations/time_travel.rb b/gems/gitlab-rspec/lib/gitlab/rspec/configurations/time_travel.rb new file mode 100644 index 00000000000..b30aa1cde0d --- /dev/null +++ b/gems/gitlab-rspec/lib/gitlab/rspec/configurations/time_travel.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +require 'active_support/all' +require 'active_support/testing/time_helpers' + +module Gitlab + module Rspec + module Configurations + class TimeTravel + def self.configure! + RSpec.configure do |config| + config.include ActiveSupport::Testing::TimeHelpers + + config.around(:example, :freeze_time) do |example| + freeze_time { example.run } + end + + config.around(:example, :time_travel_to) do |example| + date_or_time = example.metadata[:time_travel_to] + + unless date_or_time.respond_to?(:to_time) && date_or_time.to_time.present? + raise 'The time_travel_to RSpec metadata must have a Date or Time value.' + end + + travel_to(date_or_time) { example.run } + end + end + end + end + end + end +end diff --git a/gems/gitlab-rspec/lib/gitlab/rspec/stub_env.rb b/gems/gitlab-rspec/lib/gitlab/rspec/stub_env.rb new file mode 100644 index 00000000000..f8775d9f7b5 --- /dev/null +++ b/gems/gitlab-rspec/lib/gitlab/rspec/stub_env.rb @@ -0,0 +1,52 @@ +# frozen_string_literal: true + +# Inspired by https://github.com/ljkbennett/stub_env/blob/master/lib/stub_env/helpers.rb +module StubENV + # Stub ENV variables + # + # You can provide either a key and value as separate params or both in a Hash format + # + # Keys and values will always be converted to String, to comply with how ENV behaves + # + # @param key_or_hash [String, Hash<String,String>] + # @param value [String] + def stub_env(key_or_hash, value = nil) + init_stub unless env_stubbed? + + if key_or_hash.is_a? Hash + key_or_hash.each do |key, value| + add_stubbed_value(key, ensure_env_type(value)) + end + else + add_stubbed_value key_or_hash, ensure_env_type(value) + end + end + + private + + STUBBED_KEY = '__STUBBED__' + + def add_stubbed_value(key, value) + allow(ENV).to receive(:[]).with(key).and_return(value) + allow(ENV).to receive(:key?).with(key).and_return(true) + allow(ENV).to receive(:fetch).with(key).and_return(value) + allow(ENV).to receive(:fetch).with(key, anything) do |_, default_val| + value || default_val + end + end + + def env_stubbed? + ENV.fetch(STUBBED_KEY, false) + end + + def init_stub + allow(ENV).to receive(:[]).and_call_original + allow(ENV).to receive(:key?).and_call_original + allow(ENV).to receive(:fetch).and_call_original + add_stubbed_value(STUBBED_KEY, true) + end + + def ensure_env_type(value) + value.nil? ? value : value.to_s + end +end diff --git a/gems/gitlab-rspec/lib/gitlab/rspec/version.rb b/gems/gitlab-rspec/lib/gitlab/rspec/version.rb new file mode 100644 index 00000000000..34c51245012 --- /dev/null +++ b/gems/gitlab-rspec/lib/gitlab/rspec/version.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +module Gitlab + module Rspec + module Version + VERSION = "0.1.0" + end + end +end diff --git a/gems/gitlab-rspec/spec/gitlab/rspec/time_travel_spec.rb b/gems/gitlab-rspec/spec/gitlab/rspec/time_travel_spec.rb new file mode 100644 index 00000000000..79804a99f70 --- /dev/null +++ b/gems/gitlab-rspec/spec/gitlab/rspec/time_travel_spec.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +RSpec.describe 'time travel' do + before(:all) do + @original_time_zone = Time.zone + Time.zone = 'Eastern Time (US & Canada)' + end + + after(:all) do + Time.zone = @original_time_zone + end + + describe ':freeze_time' do + it 'freezes time around a spec example', :freeze_time do + expect { sleep 0.1 }.not_to change { Time.now.to_f } + end + end + + describe ':time_travel_to' do + it 'time-travels to the specified date', time_travel_to: '2020-01-01' do + expect(Date.current).to eq(Date.new(2020, 1, 1)) + end + + it 'time-travels to the specified date & time', time_travel_to: '2020-02-02 10:30:45 -0700' do + expect(Time.current).to eq(Time.new(2020, 2, 2, 17, 30, 45, '+00:00')) + end + end +end diff --git a/gems/gitlab-rspec/spec/spec_helper.rb b/gems/gitlab-rspec/spec/spec_helper.rb new file mode 100644 index 00000000000..c694747e094 --- /dev/null +++ b/gems/gitlab-rspec/spec/spec_helper.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +require 'gitlab/rspec/all' + +RSpec.configure do |config| + # Enable flags like --only-failures and --next-failure + config.example_status_persistence_file_path = ".rspec_status" + + # Disable RSpec exposing methods globally on `Module` and `main` + config.disable_monkey_patching! + + config.expect_with :rspec do |c| + c.syntax = :expect + end +end diff --git a/gems/gitlab-schema-validation/.gitignore b/gems/gitlab-schema-validation/.gitignore new file mode 100644 index 00000000000..b04a8c840df --- /dev/null +++ b/gems/gitlab-schema-validation/.gitignore @@ -0,0 +1,11 @@ +/.bundle/ +/.yardoc +/_yardoc/ +/coverage/ +/doc/ +/pkg/ +/spec/reports/ +/tmp/ + +# rspec failure tracking +.rspec_status diff --git a/gems/gitlab-schema-validation/.gitlab-ci.yml b/gems/gitlab-schema-validation/.gitlab-ci.yml new file mode 100644 index 00000000000..03db9e02b30 --- /dev/null +++ b/gems/gitlab-schema-validation/.gitlab-ci.yml @@ -0,0 +1,4 @@ +include: + - local: gems/gem.gitlab-ci.yml + inputs: + gem_name: "gitlab-schema-validation" diff --git a/gems/gitlab-schema-validation/.rspec b/gems/gitlab-schema-validation/.rspec new file mode 100644 index 00000000000..34c5164d9b5 --- /dev/null +++ b/gems/gitlab-schema-validation/.rspec @@ -0,0 +1,3 @@ +--format documentation +--color +--require spec_helper diff --git a/gems/gitlab-schema-validation/.rubocop.yml b/gems/gitlab-schema-validation/.rubocop.yml new file mode 100644 index 00000000000..1dc800520ca --- /dev/null +++ b/gems/gitlab-schema-validation/.rubocop.yml @@ -0,0 +1,8 @@ +inherit_from: + - ../config/rubocop.yml + +AllCops: + NewCops: enable + +RSpec/MultipleMemoizedHelpers: + Max: 25 diff --git a/gems/gitlab-schema-validation/Gemfile b/gems/gitlab-schema-validation/Gemfile new file mode 100644 index 00000000000..3fa25adbee1 --- /dev/null +++ b/gems/gitlab-schema-validation/Gemfile @@ -0,0 +1,6 @@ +# frozen_string_literal: true + +source "https://rubygems.org" + +# Specify your gem's dependencies in gitlab-schema-validation.gemspec +gemspec diff --git a/gems/gitlab-schema-validation/Gemfile.lock b/gems/gitlab-schema-validation/Gemfile.lock new file mode 100644 index 00000000000..5ad804d3660 --- /dev/null +++ b/gems/gitlab-schema-validation/Gemfile.lock @@ -0,0 +1,137 @@ +PATH + remote: . + specs: + gitlab-schema-validation (0.1.0) + diffy + pg_query + +GEM + remote: https://rubygems.org/ + specs: + activesupport (7.0.6) + concurrent-ruby (~> 1.0, >= 1.0.2) + i18n (>= 1.6, < 2) + minitest (>= 5.1) + tzinfo (~> 2.0) + ast (2.4.2) + benchmark-malloc (0.2.0) + benchmark-perf (0.6.0) + benchmark-trend (0.4.0) + binding_of_caller (1.0.0) + debug_inspector (>= 0.0.1) + coderay (1.1.3) + concurrent-ruby (1.2.2) + debug_inspector (1.1.0) + diff-lcs (1.5.0) + diffy (3.4.2) + gitlab-styles (10.1.0) + rubocop (~> 1.50.2) + rubocop-graphql (~> 0.18) + rubocop-performance (~> 1.15) + rubocop-rails (~> 2.17) + rubocop-rspec (~> 2.22) + google-protobuf (3.23.3) + i18n (1.14.1) + concurrent-ruby (~> 1.0) + json (2.6.3) + method_source (1.0.0) + minitest (5.18.1) + parallel (1.23.0) + parser (3.2.2.3) + ast (~> 2.4.1) + racc + pg_query (4.2.1) + google-protobuf (>= 3.22.3) + proc_to_ast (0.1.0) + coderay + parser + unparser + pry (0.14.2) + coderay (~> 1.1) + method_source (~> 1.0) + racc (1.7.1) + rack (3.0.8) + rainbow (3.1.1) + regexp_parser (2.8.1) + rexml (3.2.5) + rspec (3.12.0) + rspec-core (~> 3.12.0) + rspec-expectations (~> 3.12.0) + rspec-mocks (~> 3.12.0) + rspec-benchmark (0.6.0) + benchmark-malloc (~> 0.2) + benchmark-perf (~> 0.6) + benchmark-trend (~> 0.4) + rspec (>= 3.0) + rspec-core (3.12.2) + rspec-support (~> 3.12.0) + rspec-expectations (3.12.3) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.12.0) + rspec-mocks (3.12.5) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.12.0) + rspec-parameterized (1.0.0) + rspec-parameterized-core (< 2) + rspec-parameterized-table_syntax (< 2) + rspec-parameterized-core (1.0.0) + parser + proc_to_ast + rspec (>= 2.13, < 4) + unparser + rspec-parameterized-table_syntax (1.0.0) + binding_of_caller + rspec-parameterized-core (< 2) + rspec-support (3.12.1) + rubocop (1.50.2) + json (~> 2.3) + parallel (~> 1.10) + parser (>= 3.2.0.0) + rainbow (>= 2.2.2, < 4.0) + regexp_parser (>= 1.8, < 3.0) + rexml (>= 3.2.5, < 4.0) + rubocop-ast (>= 1.28.0, < 2.0) + ruby-progressbar (~> 1.7) + unicode-display_width (>= 2.4.0, < 3.0) + rubocop-ast (1.29.0) + parser (>= 3.2.1.0) + rubocop-capybara (2.18.0) + rubocop (~> 1.41) + rubocop-factory_bot (2.23.1) + rubocop (~> 1.33) + rubocop-graphql (0.19.0) + rubocop (>= 0.87, < 2) + rubocop-performance (1.18.0) + rubocop (>= 1.7.0, < 2.0) + rubocop-ast (>= 0.4.0) + rubocop-rails (2.20.2) + activesupport (>= 4.2.0) + rack (>= 1.1) + rubocop (>= 1.33.0, < 2.0) + rubocop-rspec (2.22.0) + rubocop (~> 1.33) + rubocop-capybara (~> 2.17) + rubocop-factory_bot (~> 2.22) + ruby-progressbar (1.13.0) + tzinfo (2.0.6) + concurrent-ruby (~> 1.0) + unicode-display_width (2.4.2) + unparser (0.6.8) + diff-lcs (~> 1.3) + parser (>= 3.2.0) + +PLATFORMS + ruby + +DEPENDENCIES + gitlab-schema-validation! + gitlab-styles (~> 10.1.0) + pry + rspec (~> 3.0) + rspec-benchmark (~> 0.6.0) + rspec-parameterized (~> 1.0) + rubocop (~> 1.50) + rubocop-rspec (~> 2.22) + +BUNDLED WITH + 2.4.14 diff --git a/gems/gitlab-schema-validation/gitlab-schema-validation.gemspec b/gems/gitlab-schema-validation/gitlab-schema-validation.gemspec new file mode 100644 index 00000000000..47ca8b65b5d --- /dev/null +++ b/gems/gitlab-schema-validation/gitlab-schema-validation.gemspec @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +require_relative "lib/gitlab/schema/validation/version" + +Gem::Specification.new do |spec| + spec.name = "gitlab-schema-validation" + spec.version = Gitlab::Schema::Validation::Version::VERSION + spec.authors = ["group::database"] + spec.email = ["engineering@gitlab.com"] + + spec.summary = "Schema validation framework" + spec.description = "Compares the differences between a structure.sql file and a database + and reports the inconsistencies." + spec.homepage = "https://gitlab.com/gitlab-org/gitlab/-/tree/master/gems/gitlab-schema-validation" + spec.license = "MIT" + spec.required_ruby_version = ">= 3.0" + spec.metadata["rubygems_mfa_required"] = "true" + + spec.files = Dir['lib/**/*.rb'] + spec.require_paths = ["lib"] + + spec.add_runtime_dependency "diffy" + spec.add_runtime_dependency "pg_query" + + spec.add_development_dependency "gitlab-styles", "~> 10.1.0" + spec.add_development_dependency "pry" + spec.add_development_dependency "rspec", "~> 3.0" + spec.add_development_dependency "rspec-benchmark", "~> 0.6.0" + spec.add_development_dependency "rspec-parameterized", "~> 1.0" + spec.add_development_dependency "rubocop", "~> 1.50" + spec.add_development_dependency "rubocop-rspec", "~> 2.22" +end diff --git a/gems/gitlab-schema-validation/lib/gitlab/schema/validation.rb b/gems/gitlab-schema-validation/lib/gitlab/schema/validation.rb new file mode 100644 index 00000000000..5211358a197 --- /dev/null +++ b/gems/gitlab-schema-validation/lib/gitlab/schema/validation.rb @@ -0,0 +1,66 @@ +# frozen_string_literal: true + +require 'pg_query' +require 'diffy' + +require_relative 'validation/version' +require_relative 'validation/inconsistency' +require_relative 'validation/pg_types' + +require_relative 'validation/validators/base' + +require_relative 'validation/validators/different_definition_indexes' +require_relative 'validation/validators/extra_indexes' +require_relative 'validation/validators/missing_indexes' + +require_relative 'validation/validators/extra_table_columns' +require_relative 'validation/validators/missing_table_columns' + +require_relative 'validation/validators/different_definition_foreign_keys' +require_relative 'validation/validators/extra_foreign_keys' +require_relative 'validation/validators/missing_foreign_keys' + +require_relative 'validation/validators/different_definition_tables' +require_relative 'validation/validators/extra_tables' +require_relative 'validation/validators/missing_tables' + +require_relative 'validation/validators/different_definition_triggers' +require_relative 'validation/validators/extra_triggers' +require_relative 'validation/validators/missing_triggers' + +require_relative 'validation/sources/structure_sql' +require_relative 'validation/sources/database' + +require_relative 'validation/schema_objects/base' +require_relative 'validation/schema_objects/column' +require_relative 'validation/schema_objects/index' +require_relative 'validation/schema_objects/table' +require_relative 'validation/schema_objects/trigger' +require_relative 'validation/schema_objects/foreign_key' + +require_relative 'validation/adapters/column_database_adapter' +require_relative 'validation/adapters/column_structure_sql_adapter' +require_relative 'validation/adapters/foreign_key_database_adapter' +require_relative 'validation/adapters/foreign_key_structure_sql_adapter' + +module Gitlab + module Schema + module Validation + class Runner + def initialize(structure_sql, database, validators:) + @structure_sql = structure_sql + @database = database + @validators = validators + end + + def execute + validators.flat_map { |c| c.new(structure_sql, database).execute } + end + + private + + attr_reader :structure_sql, :database, :validators + end + end + end +end diff --git a/gems/gitlab-schema-validation/lib/gitlab/schema/validation/adapters/column_database_adapter.rb b/gems/gitlab-schema-validation/lib/gitlab/schema/validation/adapters/column_database_adapter.rb new file mode 100644 index 00000000000..8b4d07d2e79 --- /dev/null +++ b/gems/gitlab-schema-validation/lib/gitlab/schema/validation/adapters/column_database_adapter.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +module Gitlab + module Schema + module Validation + module Adapters + class ColumnDatabaseAdapter + def initialize(query_result) + @query_result = query_result + end + + def name + @name ||= query_result['column_name'] + end + + def table_name + query_result['table_name'] + end + + def data_type + query_result['data_type'] + end + + def default + return unless query_result['column_default'] + + return if name == 'id' || query_result['column_default'].include?('nextval') + + "DEFAULT #{query_result['column_default']}" + end + + def nullable + 'NOT NULL' if query_result['not_null'] + end + + def partition_key? + query_result['partition_key'] + end + + private + + attr_reader :query_result + end + end + end + end +end diff --git a/gems/gitlab-schema-validation/lib/gitlab/schema/validation/adapters/column_structure_sql_adapter.rb b/gems/gitlab-schema-validation/lib/gitlab/schema/validation/adapters/column_structure_sql_adapter.rb new file mode 100644 index 00000000000..62e501bf16b --- /dev/null +++ b/gems/gitlab-schema-validation/lib/gitlab/schema/validation/adapters/column_structure_sql_adapter.rb @@ -0,0 +1,139 @@ +# frozen_string_literal: true + +module Gitlab + module Schema + module Validation + module Adapters + UndefinedPGType = Class.new(StandardError) + + class ColumnStructureSqlAdapter + NOT_NULL_CONSTR = :CONSTR_NOTNULL + DEFAULT_CONSTR = :CONSTR_DEFAULT + + MAPPINGS = { + 't' => 'true', + 'f' => 'false' + }.freeze + + attr_reader :table_name + + def initialize(table_name, pg_query_stmt, partitioning_stmt) + @table_name = table_name + @pg_query_stmt = pg_query_stmt + @partitioning_stmt = partitioning_stmt + end + + def name + @name ||= pg_query_stmt.colname + end + + def data_type + type(pg_query_stmt.type_name) + end + + def default + return if name == 'id' + + value = parse_node(constraints.find { |node| node.constraint.contype == DEFAULT_CONSTR }) + + return unless value + + "DEFAULT #{value}" + end + + def nullable + 'NOT NULL' if constraints.any? { |node| node.constraint.contype == NOT_NULL_CONSTR } + end + + def partition_key? + partition_keys.include?(name) + end + + private + + attr_reader :pg_query_stmt, :partitioning_stmt + + def constraints + @constraints ||= pg_query_stmt.constraints + end + + # Returns the node type + # + # pg_type:: type alias, used internally by postgres, +int4+, +int8+, +bool+, +varchar+ + # type:: type name, like +integer+, +bigint+, +boolean+, +character varying+. + # array_ext:: adds the +[]+ extension for array types. + # precision_ext:: adds the precision, if have any, like +(255)+, +(6)+. + # + # @info +timestamp+ and +timestamptz+ have a particular case when precision is defined. + # In this case, the order of the statement needs to be re-arranged from + # timestamp without time zone(6) to timestamp(6) without a time zone. + def type(node) + pg_type = parse_node(node.names.last) + type = PgTypes::TYPES.fetch(pg_type).dup + array_ext = '[]' if node.array_bounds.any? + precision_ext = "(#{node.typmods.map { |typmod| parse_node(typmod) }.join(',')})" if node.typmods.any? + + if %w[timestamp timestamptz].include?(pg_type) + type.gsub!('timestamp', ['timestamp', precision_ext].compact.join) + precision_ext = nil + end + + [type, precision_ext, array_ext].compact.join + rescue KeyError => e + raise UndefinedPGType, e.message + end + + # Parses PGQuery nodes recursively + # + # :constraint:: nodes that groups column default info + # :partition_elem:: node that store partition key info + # :func_cal:: nodes that stores functions, like +now()+ + # :a_const:: nodes that stores constant values, like +t+, +f+, +0.0.0.0+, +255+, +1.0+ + # :type_cast:: nodes that stores casting values, like +'name'::text+, +'0.0.0.0'::inet+ + # else:: extract node values in the last iteration of the recursion, like +int4+, +1.0+, +now+, +255+ + # + # @note boolean types types are mapped from +t+, +f+ to +true+, +false+ + def parse_node(node) + return unless node + + case node.node + when :constraint + parse_node(node.constraint.raw_expr) + when :partition_elem + node.partition_elem.name + when :func_call + "#{parse_node(node.func_call.funcname.first)}()" + when :a_const + parse_a_const(node.a_const) + when :type_cast + value = parse_node(node.type_cast.arg) + type = type(node.type_cast.type_name) + separator = MAPPINGS.key?(value) ? '' : "::#{type}" + + [MAPPINGS.fetch(value, "'#{value}'"), separator].compact.join + else + get_value_from_key(node, key: node.node) + end + end + + def parse_a_const(a_const) + return unless a_const + + type = a_const.val + get_value_from_key(a_const, key: type) + end + + def get_value_from_key(node, key:) + node.to_h[key].values.last + end + + def partition_keys + return [] unless partitioning_stmt + + @partition_keys ||= partitioning_stmt.part_params.map { |key_stmt| parse_node(key_stmt) } + end + end + end + end + end +end diff --git a/gems/gitlab-schema-validation/lib/gitlab/schema/validation/adapters/foreign_key_database_adapter.rb b/gems/gitlab-schema-validation/lib/gitlab/schema/validation/adapters/foreign_key_database_adapter.rb new file mode 100644 index 00000000000..ee5d5dc0ce9 --- /dev/null +++ b/gems/gitlab-schema-validation/lib/gitlab/schema/validation/adapters/foreign_key_database_adapter.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module Gitlab + module Schema + module Validation + module Adapters + class ForeignKeyDatabaseAdapter + def initialize(query_result) + @query_result = query_result + end + + def name + "#{query_result['schema']}.#{query_result['foreign_key_name']}" + end + + def table_name + query_result['table_name'] + end + + def statement + query_result['foreign_key_definition'] + end + + private + + attr_reader :query_result + end + end + end + end +end diff --git a/gems/gitlab-schema-validation/lib/gitlab/schema/validation/adapters/foreign_key_structure_sql_adapter.rb b/gems/gitlab-schema-validation/lib/gitlab/schema/validation/adapters/foreign_key_structure_sql_adapter.rb new file mode 100644 index 00000000000..730652c302d --- /dev/null +++ b/gems/gitlab-schema-validation/lib/gitlab/schema/validation/adapters/foreign_key_structure_sql_adapter.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +module Gitlab + module Schema + module Validation + module Adapters + class ForeignKeyStructureSqlAdapter + STATEMENT_REGEX = /\bREFERENCES\s\K\S+\K\s\(/ + EXTRACT_REGEX = /\bFOREIGN KEY.*/ + + def initialize(parsed_stmt) + @parsed_stmt = parsed_stmt + end + + def name + "#{schema_name}.#{foreign_key_name}" + end + + def table_name + parsed_stmt.relation.relname + end + + # PgQuery parses FK statements with an extra space in the referenced table column. + # This extra space needs to be removed. + # + # @example REFERENCES ci_pipelines (id) => REFERENCES ci_pipelines(id) + def statement + deparse_stmt[EXTRACT_REGEX].gsub(STATEMENT_REGEX, '(') + end + + private + + attr_reader :parsed_stmt + + def schema_name + parsed_stmt.relation.schemaname + end + + def foreign_key_name + parsed_stmt.cmds.first.alter_table_cmd.def.constraint.conname + end + + def deparse_stmt + PgQuery.deparse_stmt(parsed_stmt) + end + end + end + end + end +end diff --git a/gems/gitlab-schema-validation/lib/gitlab/schema/validation/inconsistency.rb b/gems/gitlab-schema-validation/lib/gitlab/schema/validation/inconsistency.rb new file mode 100644 index 00000000000..13799b8b9ff --- /dev/null +++ b/gems/gitlab-schema-validation/lib/gitlab/schema/validation/inconsistency.rb @@ -0,0 +1,67 @@ +# frozen_string_literal: true + +module Gitlab + module Schema + module Validation + class Inconsistency + def initialize(validator_class, structure_sql_object, database_object) + @validator_class = validator_class + @structure_sql_object = structure_sql_object + @database_object = database_object + end + + def error_message + format(validator_class::ERROR_MESSAGE, object_name) + end + + def type + validator_class.name + end + + def object_type + object_type = structure_sql_object&.class&.name || database_object&.class&.name + + object_type&.gsub('Gitlab::Schema::Validation::SchemaObjects::', '') + end + + def table_name + structure_sql_object&.table_name || database_object&.table_name + end + + def object_name + structure_sql_object&.name || database_object&.name + end + + def diff + Diffy::Diff.new(structure_sql_statement, database_statement) + end + + def display + <<~MSG + #{'-' * 54} + #{error_message} + Diff: + #{diff.to_s(:color)} + #{'-' * 54} + MSG + end + + def structure_sql_statement + return unless structure_sql_object + + "#{structure_sql_object.statement}\n" + end + + def database_statement + return unless database_object + + "#{database_object.statement}\n" + end + + private + + attr_reader :validator_class, :structure_sql_object, :database_object + end + end + end +end diff --git a/gems/gitlab-schema-validation/lib/gitlab/schema/validation/pg_types.rb b/gems/gitlab-schema-validation/lib/gitlab/schema/validation/pg_types.rb new file mode 100644 index 00000000000..335bbe94cfb --- /dev/null +++ b/gems/gitlab-schema-validation/lib/gitlab/schema/validation/pg_types.rb @@ -0,0 +1,73 @@ +# frozen_string_literal: true + +module Gitlab + module Schema + module Validation + class PgTypes + TYPES = { + 'bool' => 'boolean', + 'bytea' => 'bytea', + 'char' => '"char"', + 'int8' => 'bigint', + 'int2' => 'smallint', + 'int4' => 'integer', + 'regproc' => 'regproc', + 'text' => 'text', + 'oid' => 'oid', + 'tid' => 'tid', + 'xid' => 'xid', + 'cid' => 'cid', + 'json' => 'json', + 'xml' => 'xml', + 'pg_node_tree' => 'pg_node_tree', + 'pg_ndistinct' => 'pg_ndistinct', + 'pg_dependencies' => 'pg_dependencies', + 'pg_mcv_list' => 'pg_mcv_list', + 'xid8' => 'xid8', + 'path' => 'path', + 'polygon' => 'polygon', + 'float4' => 'real', + 'float8' => 'double precision', + 'circle' => 'circle', + 'money' => 'money', + 'macaddr' => 'macaddr', + 'inet' => 'inet', + 'cidr' => 'cidr', + 'macaddr8' => 'macaddr8', + 'aclitem' => 'aclitem', + 'bpchar' => 'character', + 'varchar' => 'character varying', + 'date' => 'date', + 'time' => 'time without time zone', + 'timestamp' => 'timestamp without time zone', + 'timestamptz' => 'timestamp with time zone', + 'interval' => 'interval', + 'timetz' => 'time with time zone', + 'bit' => 'bit', + 'varbit' => 'bit varying', + 'numeric' => 'numeric', + 'refcursor' => 'refcursor', + 'regprocedure' => 'regprocedure', + 'regoper' => 'regoper', + 'regoperator' => 'regoperator', + 'regclass' => 'regclass', + 'regcollation' => 'regcollation', + 'regtype' => 'regtype', + 'regrole' => 'regrole', + 'regnamespace' => 'regnamespace', + 'uuid' => 'uuid', + 'pg_lsn' => 'pg_lsn', + 'tsvector' => 'tsvector', + 'gtsvector' => 'gtsvector', + 'tsquery' => 'tsquery', + 'regconfig' => 'regconfig', + 'regdictionary' => 'regdictionary', + 'jsonb' => 'jsonb', + 'jsonpath' => 'jsonpath', + 'txid_snapshot' => 'txid_snapshot', + 'pg_snapshot' => 'pg_snapshot' + }.freeze + end + end + end +end diff --git a/gems/gitlab-schema-validation/lib/gitlab/schema/validation/schema_objects/base.rb b/gems/gitlab-schema-validation/lib/gitlab/schema/validation/schema_objects/base.rb new file mode 100644 index 00000000000..1af7a67ddb6 --- /dev/null +++ b/gems/gitlab-schema-validation/lib/gitlab/schema/validation/schema_objects/base.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module Gitlab + module Schema + module Validation + module SchemaObjects + class Base + def initialize(parsed_stmt) + @parsed_stmt = parsed_stmt + end + + def name + raise NoMethodError, "subclasses of #{self.class.name} must implement #{__method__}" + end + + def table_name + parsed_stmt.relation.relname + end + + def statement + @statement ||= PgQuery.deparse_stmt(parsed_stmt) + end + + private + + attr_reader :parsed_stmt + end + end + end + end +end diff --git a/gems/gitlab-schema-validation/lib/gitlab/schema/validation/schema_objects/column.rb b/gems/gitlab-schema-validation/lib/gitlab/schema/validation/schema_objects/column.rb new file mode 100644 index 00000000000..0b3687fdb98 --- /dev/null +++ b/gems/gitlab-schema-validation/lib/gitlab/schema/validation/schema_objects/column.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module Gitlab + module Schema + module Validation + module SchemaObjects + class Column + def initialize(adapter) + @adapter = adapter + end + + attr_reader :adapter + + def name + adapter.name + end + + def table_name + adapter.table_name + end + + def partition_key? + adapter.partition_key? + end + + def statement + [adapter.name, adapter.data_type, adapter.default, adapter.nullable].compact.join(' ') + end + end + end + end + end +end diff --git a/gems/gitlab-schema-validation/lib/gitlab/schema/validation/schema_objects/foreign_key.rb b/gems/gitlab-schema-validation/lib/gitlab/schema/validation/schema_objects/foreign_key.rb new file mode 100644 index 00000000000..41e2d30029a --- /dev/null +++ b/gems/gitlab-schema-validation/lib/gitlab/schema/validation/schema_objects/foreign_key.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +module Gitlab + module Schema + module Validation + module SchemaObjects + class ForeignKey + def initialize(adapter) + @adapter = adapter + end + + # Foreign key name should include the schema, as the same name could be used across different schemas + # + # @example public.foreign_key_name + def name + @name ||= adapter.name + end + + def table_name + @table_name ||= adapter.table_name + end + + def statement + @statement ||= adapter.statement + end + + private + + attr_reader :adapter + end + end + end + end +end diff --git a/gems/gitlab-schema-validation/lib/gitlab/schema/validation/schema_objects/index.rb b/gems/gitlab-schema-validation/lib/gitlab/schema/validation/schema_objects/index.rb new file mode 100644 index 00000000000..9f99c6a6e6e --- /dev/null +++ b/gems/gitlab-schema-validation/lib/gitlab/schema/validation/schema_objects/index.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module Schema + module Validation + module SchemaObjects + class Index < Base + def name + parsed_stmt.idxname + end + end + end + end + end +end diff --git a/gems/gitlab-schema-validation/lib/gitlab/schema/validation/schema_objects/table.rb b/gems/gitlab-schema-validation/lib/gitlab/schema/validation/schema_objects/table.rb new file mode 100644 index 00000000000..591131cb220 --- /dev/null +++ b/gems/gitlab-schema-validation/lib/gitlab/schema/validation/schema_objects/table.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +module Gitlab + module Schema + module Validation + module SchemaObjects + class Table + def initialize(name, columns) + @name = name + @columns = columns + end + + attr_reader :name, :columns + + def table_name + name + end + + def statement + format('CREATE TABLE %s (%s)', name, columns_statement) + end + + def fetch_column_by_name(column_name) + columns.find { |column| column.name == column_name } + end + + def column_exists?(column_name) + column = fetch_column_by_name(column_name) + + return false if column.nil? + + true + end + + private + + def columns_statement + columns.reject(&:partition_key?).map(&:statement).join(', ') + end + end + end + end + end +end diff --git a/gems/gitlab-schema-validation/lib/gitlab/schema/validation/schema_objects/trigger.rb b/gems/gitlab-schema-validation/lib/gitlab/schema/validation/schema_objects/trigger.rb new file mode 100644 index 00000000000..7903985a963 --- /dev/null +++ b/gems/gitlab-schema-validation/lib/gitlab/schema/validation/schema_objects/trigger.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module Schema + module Validation + module SchemaObjects + class Trigger < Base + def name + parsed_stmt.trigname + end + end + end + end + end +end diff --git a/gems/gitlab-schema-validation/lib/gitlab/schema/validation/sources/database.rb b/gems/gitlab-schema-validation/lib/gitlab/schema/validation/sources/database.rb new file mode 100644 index 00000000000..8505d1f149a --- /dev/null +++ b/gems/gitlab-schema-validation/lib/gitlab/schema/validation/sources/database.rb @@ -0,0 +1,192 @@ +# frozen_string_literal: true + +module Gitlab + module Schema + module Validation + module Sources + class Database + STATIC_PARTITIONS_SCHEMA = 'gitlab_partitions_static' + + def initialize(connection) + @connection = connection + end + + def fetch_index_by_name(index_name) + index_map[index_name] + end + + def fetch_trigger_by_name(trigger_name) + trigger_map[trigger_name] + end + + def fetch_foreign_key_by_name(foreign_key_name) + foreign_key_map[foreign_key_name] + end + + def fetch_table_by_name(table_name) + table_map[table_name] + end + + def index_exists?(index_name) + index = index_map[index_name] + + return false if index.nil? + + true + end + + def trigger_exists?(trigger_name) + trigger = trigger_map[trigger_name] + + return false if trigger.nil? + + true + end + + def foreign_key_exists?(foreign_key_name) + foreign_key = fetch_foreign_key_by_name(foreign_key_name) + + return false if foreign_key.nil? + + true + end + + def table_exists?(table_name) + table = fetch_table_by_name(table_name) + + return false if table.nil? + + true + end + + def indexes + index_map.values + end + + def triggers + trigger_map.values + end + + def foreign_keys + foreign_key_map.values + end + + def tables + table_map.values + end + + private + + attr_reader :connection + + def schemas + @schemas ||= [STATIC_PARTITIONS_SCHEMA, connection.current_schema] + end + + def trigger_map + @trigger_map ||= + fetch_triggers.transform_values! do |trigger_stmt| + SchemaObjects::Trigger.new(PgQuery.parse(trigger_stmt).tree.stmts.first.stmt.create_trig_stmt) + end + end + + def fetch_triggers + # rubocop:disable Rails/SquishedSQLHeredocs + sql = <<~SQL + SELECT triggers.tgname, pg_get_triggerdef(triggers.oid) + FROM pg_catalog.pg_trigger triggers + INNER JOIN pg_catalog.pg_class rel ON triggers.tgrelid = rel.oid + INNER JOIN pg_catalog.pg_namespace nsp ON nsp.oid = rel.relnamespace + WHERE triggers.tgisinternal IS FALSE + AND nsp.nspname IN ($1, $2) + SQL + # rubocop:enable Rails/SquishedSQLHeredocs + + connection.select_rows(sql, nil, schemas).to_h + end + + def table_map + @table_map ||= fetch_tables.transform_values! do |stmt| + columns = stmt.map { |column| SchemaObjects::Column.new(Adapters::ColumnDatabaseAdapter.new(column)) } + + SchemaObjects::Table.new(stmt.first['table_name'], columns) + end + end + + def fetch_tables + # rubocop:disable Rails/SquishedSQLHeredocs + sql = <<~SQL + SELECT + table_information.relname AS table_name, + col_information.attname AS column_name, + col_information.attnotnull AS not_null, + col_information.attnum = ANY(pg_partitioned_table.partattrs) as partition_key, + format_type(col_information.atttypid, col_information.atttypmod) AS data_type, + pg_get_expr(col_default_information.adbin, col_default_information.adrelid) AS column_default + FROM pg_attribute AS col_information + JOIN pg_class AS table_information ON col_information.attrelid = table_information.oid + JOIN pg_namespace AS schema_information ON table_information.relnamespace = schema_information.oid + LEFT JOIN pg_partitioned_table ON pg_partitioned_table.partrelid = table_information.oid + LEFT JOIN pg_attrdef AS col_default_information ON col_information.attrelid = col_default_information.adrelid + AND col_information.attnum = col_default_information.adnum + WHERE NOT col_information.attisdropped + AND col_information.attnum > 0 + AND table_information.relkind IN ('r', 'p') + AND schema_information.nspname IN ($1, $2) + SQL + # rubocop:enable Rails/SquishedSQLHeredocs + + connection.exec_query(sql, nil, schemas).group_by { |row| row['table_name'] } + end + + def fetch_indexes + # rubocop:disable Rails/SquishedSQLHeredocs + sql = <<~SQL + SELECT indexname, indexdef + FROM pg_indexes + WHERE indexname NOT LIKE '%_pkey' AND schemaname IN ($1, $2); + SQL + # rubocop:enable Rails/SquishedSQLHeredocs + + connection.select_rows(sql, nil, schemas).to_h + end + + def index_map + @index_map ||= + fetch_indexes.transform_values! do |index_stmt| + SchemaObjects::Index.new(PgQuery.parse(index_stmt).tree.stmts.first.stmt.index_stmt) + end + end + + def foreign_key_map + @foreign_key_map ||= fetch_fks.each_with_object({}) do |stmt, result| + adapter = Adapters::ForeignKeyDatabaseAdapter.new(stmt) + + result[adapter.name] = SchemaObjects::ForeignKey.new(adapter) + end + end + + def fetch_fks + # rubocop:disable Rails/SquishedSQLHeredocs + sql = <<~SQL + SELECT + pg_namespace.nspname::text AS schema, + pg_class.relname::text AS table_name, + pg_constraint.conname AS foreign_key_name, + pg_get_constraintdef(pg_constraint.oid) AS foreign_key_definition + FROM pg_constraint + INNER JOIN pg_class ON pg_constraint.conrelid = pg_class.oid + INNER JOIN pg_namespace ON pg_class.relnamespace = pg_namespace.oid + WHERE contype = 'f' + AND pg_namespace.nspname = $1 + AND pg_constraint.conparentid = 0 + SQL + # rubocop:enable Rails/SquishedSQLHeredocs + + connection.exec_query(sql, nil, [connection.current_schema]) + end + end + end + end + end +end diff --git a/gems/gitlab-schema-validation/lib/gitlab/schema/validation/sources/structure_sql.rb b/gems/gitlab-schema-validation/lib/gitlab/schema/validation/sources/structure_sql.rb new file mode 100644 index 00000000000..b2e3fcd63c5 --- /dev/null +++ b/gems/gitlab-schema-validation/lib/gitlab/schema/validation/sources/structure_sql.rb @@ -0,0 +1,143 @@ +# frozen_string_literal: true + +module Gitlab + module Schema + module Validation + module Sources + class StructureSql + DEFAULT_SCHEMA = 'public' + + def initialize(structure_file_path, schema_name = DEFAULT_SCHEMA) + @structure_file_path = structure_file_path + @schema_name = schema_name + end + + def index_exists?(index_name) + index = indexes.find { |index| index.name == index_name } + + return false if index.nil? + + true + end + + def trigger_exists?(trigger_name) + trigger = triggers.find { |trigger| trigger.name == trigger_name } + + return false if trigger.nil? + + true + end + + def foreign_key_exists?(foreign_key_name) + foreign_key = foreign_keys.find { |fk| fk.name == foreign_key_name } + + return false if foreign_key.nil? + + true + end + + def table_exists?(table_name) + table = fetch_table_by_name(table_name) + + return false if table.nil? + + true + end + + def fetch_table_by_name(table_name) + tables.find { |table| table.name == table_name } + end + + def indexes + @indexes ||= map_with_default_schema(index_statements, SchemaObjects::Index) + end + + def triggers + @triggers ||= map_with_default_schema(trigger_statements, SchemaObjects::Trigger) + end + + def foreign_keys + @foreign_keys ||= foreign_key_statements.map do |stmt| + stmt.relation.schemaname = schema_name if stmt.relation.schemaname == '' + + SchemaObjects::ForeignKey.new(Adapters::ForeignKeyStructureSqlAdapter.new(stmt)) + end + end + + def tables + @tables ||= table_statements.map do |stmt| + table_name = stmt.relation.relname + partition_stmt = stmt.partspec + + columns = stmt.table_elts.select { |n| n.node == :column_def }.map do |column| + adapter = Adapters::ColumnStructureSqlAdapter.new(table_name, column.column_def, partition_stmt) + SchemaObjects::Column.new(adapter) + end + + SchemaObjects::Table.new(table_name, columns) + end + end + + private + + attr_reader :structure_file_path, :schema_name + + def index_statements + statements.filter_map { |s| s.stmt.index_stmt } + end + + def trigger_statements + statements.filter_map { |s| s.stmt.create_trig_stmt } + end + + def table_statements + statements.filter_map { |s| s.stmt.create_stmt } + end + + def foreign_key_statements + constraint_statements(:CONSTR_FOREIGN) + end + + # Filter constraint statement nodes + # + # @param constraint_type [Symbol] node type. One of CONSTR_PRIMARY, CONSTR_CHECK, CONSTR_EXCLUSION, + # CONSTR_UNIQUE or CONSTR_FOREIGN. + def constraint_statements(constraint_type) + alter_table_statements(:AT_AddConstraint).filter do |stmt| + stmt.cmds.first.alter_table_cmd.def.constraint.contype == constraint_type + end + end + + # Filter alter table statement nodes + # + # @param subtype [Symbol] node subtype +AT_AttachPartition+, +AT_ColumnDefault+ or +AT_AddConstraint+ + def alter_table_statements(subtype) + statements.filter_map do |statement| + node = statement.stmt.alter_table_stmt + + next unless node + + node if node.cmds.first.alter_table_cmd.subtype == subtype + end + end + + def statements + @statements ||= parsed_structure_file.tree.stmts + end + + def parsed_structure_file + PgQuery.parse(File.read(structure_file_path)) + end + + def map_with_default_schema(statements, validation_class) + statements.map do |statement| + statement.relation.schemaname = schema_name if statement.relation.schemaname == '' + + validation_class.new(statement) + end + end + end + end + end + end +end diff --git a/gems/gitlab-schema-validation/lib/gitlab/schema/validation/validators/base.rb b/gems/gitlab-schema-validation/lib/gitlab/schema/validation/validators/base.rb new file mode 100644 index 00000000000..151af4b61e6 --- /dev/null +++ b/gems/gitlab-schema-validation/lib/gitlab/schema/validation/validators/base.rb @@ -0,0 +1,49 @@ +# frozen_string_literal: true + +module Gitlab + module Schema + module Validation + module Validators + class Base + ERROR_MESSAGE = 'A schema inconsistency has been found' + + def initialize(structure_sql, database) + @structure_sql = structure_sql + @database = database + end + + def self.all_validators + [ + ExtraTables, + ExtraTableColumns, + ExtraIndexes, + ExtraTriggers, + ExtraForeignKeys, + MissingTables, + MissingTableColumns, + MissingIndexes, + MissingTriggers, + MissingForeignKeys, + DifferentDefinitionTables, + DifferentDefinitionIndexes, + DifferentDefinitionTriggers, + DifferentDefinitionForeignKeys + ] + end + + def execute + raise NoMethodError, "subclasses of #{self.class.name} must implement #{__method__}" + end + + private + + attr_reader :structure_sql, :database + + def build_inconsistency(validator_class, structure_sql_object, database_object) + Inconsistency.new(validator_class, structure_sql_object, database_object) + end + end + end + end + end +end diff --git a/gems/gitlab-schema-validation/lib/gitlab/schema/validation/validators/different_definition_foreign_keys.rb b/gems/gitlab-schema-validation/lib/gitlab/schema/validation/validators/different_definition_foreign_keys.rb new file mode 100644 index 00000000000..d8ea7807cc5 --- /dev/null +++ b/gems/gitlab-schema-validation/lib/gitlab/schema/validation/validators/different_definition_foreign_keys.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +module Gitlab + module Schema + module Validation + module Validators + class DifferentDefinitionForeignKeys < Base + ERROR_MESSAGE = "The %s foreign key has a different statement between structure.sql and database" + + def execute + structure_sql.foreign_keys.filter_map do |structure_sql_fk| + database_fk = database.fetch_foreign_key_by_name(structure_sql_fk.name) + + next if database_fk.nil? + next if database_fk.statement == structure_sql_fk.statement + + build_inconsistency(self.class, structure_sql_fk, database_fk) + end + end + end + end + end + end +end diff --git a/gems/gitlab-schema-validation/lib/gitlab/schema/validation/validators/different_definition_indexes.rb b/gems/gitlab-schema-validation/lib/gitlab/schema/validation/validators/different_definition_indexes.rb new file mode 100644 index 00000000000..032e7edd5ab --- /dev/null +++ b/gems/gitlab-schema-validation/lib/gitlab/schema/validation/validators/different_definition_indexes.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +module Gitlab + module Schema + module Validation + module Validators + class DifferentDefinitionIndexes < Base + ERROR_MESSAGE = 'The %s index has a different statement between structure.sql and database' + + def execute + structure_sql.indexes.filter_map do |structure_sql_index| + database_index = database.fetch_index_by_name(structure_sql_index.name) + + next if database_index.nil? + next if database_index.statement == structure_sql_index.statement + + build_inconsistency(self.class, structure_sql_index, database_index) + end + end + end + end + end + end +end diff --git a/gems/gitlab-schema-validation/lib/gitlab/schema/validation/validators/different_definition_tables.rb b/gems/gitlab-schema-validation/lib/gitlab/schema/validation/validators/different_definition_tables.rb new file mode 100644 index 00000000000..f6892a76a12 --- /dev/null +++ b/gems/gitlab-schema-validation/lib/gitlab/schema/validation/validators/different_definition_tables.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +module Gitlab + module Schema + module Validation + module Validators + class DifferentDefinitionTables < Base + ERROR_MESSAGE = "The table %s has a different column statement between structure.sql and database" + + def execute + structure_sql.tables.filter_map do |structure_sql_table| + table_name = structure_sql_table.name + database_table = database.fetch_table_by_name(table_name) + + next unless database_table + + db_diffs, structure_diffs = column_diffs(database_table, structure_sql_table.columns) + + if db_diffs.any? + build_inconsistency(self.class, + SchemaObjects::Table.new(table_name, db_diffs), + SchemaObjects::Table.new(table_name, structure_diffs)) + end + end + end + + private + + def column_diffs(db_table, columns) + db_diffs = [] + structure_diffs = [] + + columns.each do |column| + db_column = db_table.fetch_column_by_name(column.name) + + next unless db_column + + next if db_column.statement == column.statement + + db_diffs << db_column + structure_diffs << column + end + + [db_diffs, structure_diffs] + end + end + end + end + end +end diff --git a/gems/gitlab-schema-validation/lib/gitlab/schema/validation/validators/different_definition_triggers.rb b/gems/gitlab-schema-validation/lib/gitlab/schema/validation/validators/different_definition_triggers.rb new file mode 100644 index 00000000000..eed78924b02 --- /dev/null +++ b/gems/gitlab-schema-validation/lib/gitlab/schema/validation/validators/different_definition_triggers.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +module Gitlab + module Schema + module Validation + module Validators + class DifferentDefinitionTriggers < Base + ERROR_MESSAGE = "The %s trigger has a different statement between structure.sql and database" + + def execute + structure_sql.triggers.filter_map do |structure_sql_trigger| + database_trigger = database.fetch_trigger_by_name(structure_sql_trigger.name) + + next if database_trigger.nil? + next if database_trigger.statement == structure_sql_trigger.statement + + build_inconsistency(self.class, structure_sql_trigger, nil) + end + end + end + end + end + end +end diff --git a/gems/gitlab-schema-validation/lib/gitlab/schema/validation/validators/extra_foreign_keys.rb b/gems/gitlab-schema-validation/lib/gitlab/schema/validation/validators/extra_foreign_keys.rb new file mode 100644 index 00000000000..81968318629 --- /dev/null +++ b/gems/gitlab-schema-validation/lib/gitlab/schema/validation/validators/extra_foreign_keys.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module Gitlab + module Schema + module Validation + module Validators + class ExtraForeignKeys < Base + ERROR_MESSAGE = "The foreign key %s is present in the database, but not in the structure.sql file" + + def execute + database.foreign_keys.filter_map do |database_fk| + next if structure_sql.foreign_key_exists?(database_fk.name) + + build_inconsistency(self.class, nil, database_fk) + end + end + end + end + end + end +end diff --git a/gems/gitlab-schema-validation/lib/gitlab/schema/validation/validators/extra_indexes.rb b/gems/gitlab-schema-validation/lib/gitlab/schema/validation/validators/extra_indexes.rb new file mode 100644 index 00000000000..4b5bd7c820b --- /dev/null +++ b/gems/gitlab-schema-validation/lib/gitlab/schema/validation/validators/extra_indexes.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module Gitlab + module Schema + module Validation + module Validators + class ExtraIndexes < Base + ERROR_MESSAGE = 'The index %s is present in the database, but not in the structure.sql file' + + def execute + database.indexes.filter_map do |database_index| + next if structure_sql.index_exists?(database_index.name) + + build_inconsistency(self.class, nil, database_index) + end + end + end + end + end + end +end diff --git a/gems/gitlab-schema-validation/lib/gitlab/schema/validation/validators/extra_table_columns.rb b/gems/gitlab-schema-validation/lib/gitlab/schema/validation/validators/extra_table_columns.rb new file mode 100644 index 00000000000..517d6ae957f --- /dev/null +++ b/gems/gitlab-schema-validation/lib/gitlab/schema/validation/validators/extra_table_columns.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module Gitlab + module Schema + module Validation + module Validators + class ExtraTableColumns < Base + ERROR_MESSAGE = "The table %s has columns present in the database, but not in the structure.sql file" + + def execute + database.tables.filter_map do |database_table| + table_name = database_table.name + structure_sql_table = structure_sql.fetch_table_by_name(table_name) + + next unless structure_sql_table + + inconsistencies = database_table.columns.filter_map do |database_table_column| + next if structure_sql_table.column_exists?(database_table_column.name) + + database_table_column + end + + if inconsistencies.any? + build_inconsistency(self.class, nil, SchemaObjects::Table.new(table_name, inconsistencies)) + end + end + end + end + end + end + end +end diff --git a/gems/gitlab-schema-validation/lib/gitlab/schema/validation/validators/extra_tables.rb b/gems/gitlab-schema-validation/lib/gitlab/schema/validation/validators/extra_tables.rb new file mode 100644 index 00000000000..d297464a01c --- /dev/null +++ b/gems/gitlab-schema-validation/lib/gitlab/schema/validation/validators/extra_tables.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module Gitlab + module Schema + module Validation + module Validators + class ExtraTables < Base + ERROR_MESSAGE = "The table %s is present in the database, but not in the structure.sql file" + + def execute + database.tables.filter_map do |database_table| + next if structure_sql.table_exists?(database_table.name) + + build_inconsistency(self.class, nil, database_table) + end + end + end + end + end + end +end diff --git a/gems/gitlab-schema-validation/lib/gitlab/schema/validation/validators/extra_triggers.rb b/gems/gitlab-schema-validation/lib/gitlab/schema/validation/validators/extra_triggers.rb new file mode 100644 index 00000000000..d06747989fc --- /dev/null +++ b/gems/gitlab-schema-validation/lib/gitlab/schema/validation/validators/extra_triggers.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module Gitlab + module Schema + module Validation + module Validators + class ExtraTriggers < Base + ERROR_MESSAGE = "The trigger %s is present in the database, but not in the structure.sql file" + + def execute + database.triggers.filter_map do |database_trigger| + next if structure_sql.trigger_exists?(database_trigger.name) + + build_inconsistency(self.class, nil, database_trigger) + end + end + end + end + end + end +end diff --git a/gems/gitlab-schema-validation/lib/gitlab/schema/validation/validators/missing_foreign_keys.rb b/gems/gitlab-schema-validation/lib/gitlab/schema/validation/validators/missing_foreign_keys.rb new file mode 100644 index 00000000000..daebd458282 --- /dev/null +++ b/gems/gitlab-schema-validation/lib/gitlab/schema/validation/validators/missing_foreign_keys.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module Gitlab + module Schema + module Validation + module Validators + class MissingForeignKeys < Base + ERROR_MESSAGE = "The foreign key %s is missing from the database" + + def execute + structure_sql.foreign_keys.filter_map do |structure_sql_fk| + next if database.foreign_key_exists?(structure_sql_fk.name) + + build_inconsistency(self.class, structure_sql_fk, nil) + end + end + end + end + end + end +end diff --git a/gems/gitlab-schema-validation/lib/gitlab/schema/validation/validators/missing_indexes.rb b/gems/gitlab-schema-validation/lib/gitlab/schema/validation/validators/missing_indexes.rb new file mode 100644 index 00000000000..655c462aeaa --- /dev/null +++ b/gems/gitlab-schema-validation/lib/gitlab/schema/validation/validators/missing_indexes.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module Gitlab + module Schema + module Validation + module Validators + class MissingIndexes < Base + ERROR_MESSAGE = "The index %s is missing from the database" + + def execute + structure_sql.indexes.filter_map do |structure_sql_index| + next if database.index_exists?(structure_sql_index.name) + + build_inconsistency(self.class, structure_sql_index, nil) + end + end + end + end + end + end +end diff --git a/gems/gitlab-schema-validation/lib/gitlab/schema/validation/validators/missing_table_columns.rb b/gems/gitlab-schema-validation/lib/gitlab/schema/validation/validators/missing_table_columns.rb new file mode 100644 index 00000000000..8b441e19654 --- /dev/null +++ b/gems/gitlab-schema-validation/lib/gitlab/schema/validation/validators/missing_table_columns.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module Gitlab + module Schema + module Validation + module Validators + class MissingTableColumns < Base + ERROR_MESSAGE = "The table %s has columns missing from the database" + + def execute + structure_sql.tables.filter_map do |structure_sql_table| + table_name = structure_sql_table.name + database_table = database.fetch_table_by_name(table_name) + + next unless database_table + + inconsistencies = structure_sql_table.columns.filter_map do |structure_table_column| + next if database_table.column_exists?(structure_table_column.name) + + structure_table_column + end + + if inconsistencies.any? + build_inconsistency(self.class, nil, SchemaObjects::Table.new(table_name, inconsistencies)) + end + end + end + end + end + end + end +end diff --git a/gems/gitlab-schema-validation/lib/gitlab/schema/validation/validators/missing_tables.rb b/gems/gitlab-schema-validation/lib/gitlab/schema/validation/validators/missing_tables.rb new file mode 100644 index 00000000000..facf9135dfb --- /dev/null +++ b/gems/gitlab-schema-validation/lib/gitlab/schema/validation/validators/missing_tables.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module Gitlab + module Schema + module Validation + module Validators + class MissingTables < Base + ERROR_MESSAGE = "The table %s is missing from the database" + + def execute + structure_sql.tables.filter_map do |structure_sql_table| + next if database.table_exists?(structure_sql_table.name) + + build_inconsistency(self.class, structure_sql_table, nil) + end + end + end + end + end + end +end diff --git a/gems/gitlab-schema-validation/lib/gitlab/schema/validation/validators/missing_triggers.rb b/gems/gitlab-schema-validation/lib/gitlab/schema/validation/validators/missing_triggers.rb new file mode 100644 index 00000000000..1640d4304c3 --- /dev/null +++ b/gems/gitlab-schema-validation/lib/gitlab/schema/validation/validators/missing_triggers.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module Gitlab + module Schema + module Validation + module Validators + class MissingTriggers < Base + ERROR_MESSAGE = "The trigger %s is missing from the database" + + def execute + structure_sql.triggers.filter_map do |structure_sql_trigger| + next if database.trigger_exists?(structure_sql_trigger.name) + + build_inconsistency(self.class, structure_sql_trigger, nil) + end + end + end + end + end + end +end diff --git a/gems/gitlab-schema-validation/lib/gitlab/schema/validation/version.rb b/gems/gitlab-schema-validation/lib/gitlab/schema/validation/version.rb new file mode 100644 index 00000000000..40220578c97 --- /dev/null +++ b/gems/gitlab-schema-validation/lib/gitlab/schema/validation/version.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +module Gitlab + module Schema + module Validation + module Version + VERSION = "0.1.0" + end + end + end +end diff --git a/gems/gitlab-schema-validation/spec/fixtures/structure.sql b/gems/gitlab-schema-validation/spec/fixtures/structure.sql new file mode 100644 index 00000000000..421fb6c3593 --- /dev/null +++ b/gems/gitlab-schema-validation/spec/fixtures/structure.sql @@ -0,0 +1,108 @@ +CREATE INDEX missing_index ON events USING btree (created_at, author_id); + +CREATE UNIQUE INDEX wrong_index ON table_name (column_name, column_name_2); + +CREATE UNIQUE INDEX "index" ON achievements USING btree (namespace_id, lower(name)); + +CREATE INDEX index_namespaces_public_groups_name_id ON namespaces USING btree (name, id) WHERE (((type)::text = 'Group'::text) AND (visibility_level = 20)); + +CREATE UNIQUE INDEX index_on_deploy_keys_id_and_type_and_public ON keys USING btree (id, type) WHERE (public = true); + +CREATE INDEX index_users_on_public_email_excluding_null_and_empty ON users USING btree (public_email) WHERE (((public_email)::text <> ''::text) AND (public_email IS NOT NULL)); + +CREATE TABLE test_table ( + id bigint NOT NULL, + integer_column integer, + integer_with_default_column integer DEFAULT 1, + smallint_column smallint, + smallint_with_default_column smallint DEFAULT 0 NOT NULL, + numeric_column numeric NOT NULL, + numeric_with_default_column numeric DEFAULT 1.0 NOT NULL, + boolean_colum boolean, + boolean_with_default_colum boolean DEFAULT true NOT NULL, + double_precision_column double precision, + double_precision_with_default_column double precision DEFAULT 1.0, + varying_column character varying, + varying_with_default_column character varying DEFAULT 'DEFAULT'::character varying NOT NULL, + varying_with_limit_column character varying(255), + varying_with_limit_and_default_column character varying(255) DEFAULT 'DEFAULT'::character varying, + text_column text NOT NULL, + text_with_default_column text DEFAULT ''::text NOT NULL, + array_column character varying(255)[] NOT NULL, + array_with_default_column character varying(255)[] DEFAULT '{one,two}'::character varying[] NOT NULL, + jsonb_column jsonb, + jsonb_with_default_column jsonb DEFAULT '[]'::jsonb NOT NULL, + timestamptz_column timestamp with time zone, + timestamptz_with_default_column timestamp(6) with time zone DEFAULT now(), + timestamp_column timestamp(6) without time zone NOT NULL, + timestamp_with_default_column timestamp(6) without time zone DEFAULT '2022-01-23 00:00:00+00'::timestamp without time zone NOT NULL, + date_column date, + date_with_default_column date DEFAULT '2023-04-05', + inet_column inet NOT NULL, + inet_with_default_column inet DEFAULT '0.0.0.0'::inet NOT NULL, + macaddr_column macaddr, + macaddr_with_default_column macaddr DEFAULT '00-00-00-00-00-000'::macaddr NOT NULL, + uuid_column uuid NOT NULL, + uuid_with_default_column uuid DEFAULT '00000000-0000-0000-0000-000000000000'::uuid NOT NULL, + bytea_column bytea, + bytea_with_default_column bytea DEFAULT '\xDEADBEEF'::bytea, + unmapped_column_type anyarray, + partition_key bigint DEFAULT 1 NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL +) PARTITION BY HASH (partition_key, created_at); + +CREATE TABLE ci_project_mirrors ( + id bigint NOT NULL, + project_id integer NOT NULL, + namespace_id integer NOT NULL +); + +CREATE TABLE wrong_table ( + id bigint NOT NULL, + description character varying(255) NOT NULL +); + +CREATE TABLE extra_table_columns ( + id bigint NOT NULL, + name character varying(255) NOT NULL +); + +CREATE TABLE missing_table ( + id bigint NOT NULL, + description text NOT NULL +); + +CREATE TABLE missing_table_columns ( + id bigint NOT NULL, + email character varying(255) NOT NULL +); + +CREATE TABLE operations_user_lists ( + id bigint NOT NULL, + project_id bigint NOT NULL, + created_at timestamp with time zone NOT NULL, + updated_at timestamp with time zone NOT NULL, + iid integer NOT NULL, + name character varying(255) NOT NULL, + user_xids text DEFAULT ''::text NOT NULL +); + +CREATE TRIGGER trigger AFTER INSERT ON public.t1 FOR EACH ROW EXECUTE FUNCTION t1(); + +CREATE TRIGGER wrong_trigger BEFORE UPDATE ON public.t2 FOR EACH ROW EXECUTE FUNCTION my_function(); + +CREATE TRIGGER missing_trigger_1 BEFORE INSERT OR UPDATE ON public.t3 FOR EACH ROW EXECUTE FUNCTION t3(); + +CREATE TRIGGER projects_loose_fk_trigger AFTER DELETE ON projects REFERENCING OLD TABLE AS old_table FOR EACH STATEMENT EXECUTE FUNCTION insert_into_loose_foreign_keys_deleted_records(); + +ALTER TABLE web_hooks + ADD CONSTRAINT web_hooks_project_id_fkey FOREIGN KEY (project_id) REFERENCES projects(id) ON DELETE CASCADE; + +ALTER TABLE ONLY issues + ADD CONSTRAINT wrong_definition_fk FOREIGN KEY (author_id) REFERENCES users(id) ON DELETE SET NULL; + +ALTER TABLE ONLY issues + ADD CONSTRAINT missing_fk FOREIGN KEY (author_id) REFERENCES users(id) ON DELETE SET NULL; + +ALTER TABLE ONLY bulk_import_configurations + ADD CONSTRAINT fk_rails_536b96bff1 FOREIGN KEY (bulk_import_id) REFERENCES bulk_imports(id) ON DELETE CASCADE; diff --git a/gems/gitlab-schema-validation/spec/gitlab/schema/validation_spec.rb b/gems/gitlab-schema-validation/spec/gitlab/schema/validation_spec.rb new file mode 100644 index 00000000000..f4a06abab48 --- /dev/null +++ b/gems/gitlab-schema-validation/spec/gitlab/schema/validation_spec.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +RSpec.describe Gitlab::Schema::Validation do + it "has a version number" do + expect(Gitlab::Schema::Validation::Version::VERSION).not_to be_nil + end +end diff --git a/gems/gitlab-schema-validation/spec/lib/gitlab/schema/validation/adapters/column_database_adapter_spec.rb b/gems/gitlab-schema-validation/spec/lib/gitlab/schema/validation/adapters/column_database_adapter_spec.rb new file mode 100644 index 00000000000..ce16d8468b5 --- /dev/null +++ b/gems/gitlab-schema-validation/spec/lib/gitlab/schema/validation/adapters/column_database_adapter_spec.rb @@ -0,0 +1,72 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Schema::Validation::Adapters::ColumnDatabaseAdapter, feature_category: :database do + subject(:adapter) { described_class.new(db_result) } + + let(:column_name) { 'email' } + let(:column_default) { "'no-reply@gitlab.com'::character varying" } + let(:not_null) { true } + let(:partition_key) { false } + let(:db_result) do + { + 'table_name' => 'projects', + 'column_name' => column_name, + 'data_type' => 'character varying', + 'column_default' => column_default, + 'not_null' => not_null, + 'partition_key' => partition_key + } + end + + describe '#name' do + it { expect(adapter.name).to eq('email') } + end + + describe '#table_name' do + it { expect(adapter.table_name).to eq('projects') } + end + + describe '#data_type' do + it { expect(adapter.data_type).to eq('character varying') } + end + + describe '#default' do + context "when there's no default value in the column" do + let(:column_default) { nil } + + it { expect(adapter.default).to be_nil } + end + + context 'when the column name is id' do + let(:column_name) { 'id' } + + it { expect(adapter.default).to be_nil } + end + + context 'when the column default includes nextval' do + let(:column_default) { "nextval('my_seq'::regclass)" } + + it { expect(adapter.default).to be_nil } + end + + it { expect(adapter.default).to eq("DEFAULT 'no-reply@gitlab.com'::character varying") } + end + + describe '#nullable' do + context 'when column is not null' do + it { expect(adapter.nullable).to eq('NOT NULL') } + end + + context 'when column is nullable' do + let(:not_null) { false } + + it { expect(adapter.nullable).to be_nil } + end + end + + describe '#partition_key?' do + it { expect(adapter.partition_key?).to be(false) } + end +end diff --git a/gems/gitlab-schema-validation/spec/lib/gitlab/schema/validation/adapters/column_structure_sql_adapter_spec.rb b/gems/gitlab-schema-validation/spec/lib/gitlab/schema/validation/adapters/column_structure_sql_adapter_spec.rb new file mode 100644 index 00000000000..ae0d635e8ca --- /dev/null +++ b/gems/gitlab-schema-validation/spec/lib/gitlab/schema/validation/adapters/column_structure_sql_adapter_spec.rb @@ -0,0 +1,78 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Schema::Validation::Adapters::ColumnStructureSqlAdapter, feature_category: :database do + subject(:adapter) { described_class.new(table_name, column_def, partition_stmt) } + + let(:table_name) { 'test_table' } + let(:file_path) { 'spec/fixtures/structure.sql' } + let(:table_stmts) { PgQuery.parse(File.read(file_path)).tree.stmts.filter_map { |s| s.stmt.create_stmt } } + let(:table) { table_stmts.find { |table| table.relation.relname == table_name } } + let(:partition_stmt) { table.partspec } + let(:column_stmts) { table.table_elts } + let(:column_def) { column_stmts.find { |col| col.column_def.colname == column_name }.column_def } + + where(:column_name, :data_type, :default_value, :nullable, :partition_key) do + [ + ['id', 'bigint', nil, 'NOT NULL', false], + ['integer_column', 'integer', nil, nil, false], + ['integer_with_default_column', 'integer', 'DEFAULT 1', nil, false], + ['smallint_with_default_column', 'smallint', 'DEFAULT 0', 'NOT NULL', false], + ['double_precision_with_default_column', 'double precision', 'DEFAULT 1.0', nil, false], + ['numeric_with_default_column', 'numeric', 'DEFAULT 1.0', 'NOT NULL', false], + ['boolean_with_default_colum', 'boolean', 'DEFAULT true', 'NOT NULL', false], + ['varying_with_default_column', 'character varying', "DEFAULT 'DEFAULT'::character varying", 'NOT NULL', false], + ['varying_with_limit_and_default_column', 'character varying(255)', "DEFAULT 'DEFAULT'::character varying", + nil, false], + ['text_with_default_column', 'text', "DEFAULT ''::text", 'NOT NULL', false], + ['array_with_default_column', 'character varying(255)[]', "DEFAULT '{one,two}'::character varying[]", + 'NOT NULL', false], + ['jsonb_with_default_column', 'jsonb', "DEFAULT '[]'::jsonb", 'NOT NULL', false], + ['timestamptz_with_default_column', 'timestamp(6) with time zone', 'DEFAULT now()', nil, false], + ['timestamp_with_default_column', 'timestamp(6) without time zone', + "DEFAULT '2022-01-23 00:00:00+00'::timestamp without time zone", 'NOT NULL', false], + ['date_with_default_column', 'date', 'DEFAULT 2023-04-05', nil, false], + ['inet_with_default_column', 'inet', "DEFAULT '0.0.0.0'::inet", 'NOT NULL', false], + ['macaddr_with_default_column', 'macaddr', "DEFAULT '00-00-00-00-00-000'::macaddr", 'NOT NULL', false], + ['uuid_with_default_column', 'uuid', "DEFAULT '00000000-0000-0000-0000-000000000000'::uuid", 'NOT NULL', false], + ['partition_key', 'bigint', 'DEFAULT 1', 'NOT NULL', true], + ['created_at', 'timestamp with time zone', 'DEFAULT now()', 'NOT NULL', true] + ] + end + + with_them do + describe '#name' do + it { expect(adapter.name).to eq(column_name) } + end + + describe '#table_name' do + it { expect(adapter.table_name).to eq(table_name) } + end + + describe '#data_type' do + it { expect(adapter.data_type).to eq(data_type) } + end + + describe '#nullable' do + it { expect(adapter.nullable).to eq(nullable) } + end + + describe '#default' do + it { expect(adapter.default).to eq(default_value) } + end + + describe '#partition_key?' do + it { expect(adapter.partition_key?).to eq(partition_key) } + end + end + + context 'when the data type is not mapped' do + let(:column_name) { 'unmapped_column_type' } + let(:error_class) { Gitlab::Schema::Validation::Adapters::UndefinedPGType } + + describe '#data_type' do + it { expect { adapter.data_type }.to raise_error(error_class) } + end + end +end diff --git a/gems/gitlab-schema-validation/spec/lib/gitlab/schema/validation/adapters/foreign_key_database_adapter_spec.rb b/gems/gitlab-schema-validation/spec/lib/gitlab/schema/validation/adapters/foreign_key_database_adapter_spec.rb new file mode 100644 index 00000000000..52689c0f0ec --- /dev/null +++ b/gems/gitlab-schema-validation/spec/lib/gitlab/schema/validation/adapters/foreign_key_database_adapter_spec.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Schema::Validation::Adapters::ForeignKeyDatabaseAdapter, feature_category: :database do + subject(:adapter) { described_class.new(query_result) } + + let(:query_result) do + { + 'schema' => 'public', + 'foreign_key_name' => 'fk_2e88fb7ce9', + 'table_name' => 'members', + 'foreign_key_definition' => 'FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE' + } + end + + describe '#name' do + it { expect(adapter.name).to eq('public.fk_2e88fb7ce9') } + end + + describe '#table_name' do + it { expect(adapter.table_name).to eq('members') } + end + + describe '#statement' do + it { expect(adapter.statement).to eq('FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE') } + end +end diff --git a/gems/gitlab-schema-validation/spec/lib/gitlab/schema/validation/adapters/foreign_key_structure_sql_adapter_spec.rb b/gems/gitlab-schema-validation/spec/lib/gitlab/schema/validation/adapters/foreign_key_structure_sql_adapter_spec.rb new file mode 100644 index 00000000000..001786b9fbe --- /dev/null +++ b/gems/gitlab-schema-validation/spec/lib/gitlab/schema/validation/adapters/foreign_key_structure_sql_adapter_spec.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Schema::Validation::Adapters::ForeignKeyStructureSqlAdapter, feature_category: :database do + subject(:adapter) { described_class.new(stmt) } + + let(:stmt) { PgQuery.parse(sql).tree.stmts.first.stmt.alter_table_stmt } + + where(:sql, :name, :table_name, :statement) do + [ + [ + 'ALTER TABLE ONLY public.issues ADD CONSTRAINT fk_05f1e72feb FOREIGN KEY (author_id) REFERENCES users (id) ' \ + 'ON DELETE SET NULL', + 'public.fk_05f1e72feb', + 'issues', + 'FOREIGN KEY (author_id) REFERENCES users(id) ON DELETE SET NULL' + ], + [ + 'ALTER TABLE public.import_failures ADD CONSTRAINT fk_9a9b9ba21c FOREIGN KEY (user_id) REFERENCES users(id) ' \ + 'ON DELETE CASCADE', + 'public.fk_9a9b9ba21c', + 'import_failures', + 'FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE' + ] + ] + end + + with_them do + describe '#name' do + it { expect(adapter.name).to eq(name) } + end + + describe '#table_name' do + it { expect(adapter.table_name).to eq(table_name) } + end + + describe '#statement' do + it { expect(adapter.statement).to eq(statement) } + end + end +end diff --git a/gems/gitlab-schema-validation/spec/lib/gitlab/schema/validation/inconsistency_spec.rb b/gems/gitlab-schema-validation/spec/lib/gitlab/schema/validation/inconsistency_spec.rb new file mode 100644 index 00000000000..268bb4556e3 --- /dev/null +++ b/gems/gitlab-schema-validation/spec/lib/gitlab/schema/validation/inconsistency_spec.rb @@ -0,0 +1,96 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Schema::Validation::Inconsistency do + let(:validator) { Gitlab::Schema::Validation::Validators::DifferentDefinitionIndexes } + + let(:database_statement) { 'CREATE INDEX index_name ON public.achievements USING btree (namespace_id)' } + let(:structure_sql_statement) { 'CREATE INDEX index_name ON public.achievements USING btree (id)' } + + let(:structure_stmt) { PgQuery.parse(structure_sql_statement).tree.stmts.first.stmt.index_stmt } + let(:database_stmt) { PgQuery.parse(database_statement).tree.stmts.first.stmt.index_stmt } + + let(:structure_sql_object) { Gitlab::Schema::Validation::SchemaObjects::Index.new(structure_stmt) } + let(:database_object) { Gitlab::Schema::Validation::SchemaObjects::Index.new(database_stmt) } + + subject(:inconsistency) { described_class.new(validator, structure_sql_object, database_object) } + + describe '#object_name' do + it 'returns the index name' do + expect(inconsistency.object_name).to eq('index_name') + end + end + + describe '#diff' do + it 'returns a diff between the structure.sql and the database' do + expect(inconsistency.diff).to be_a(Diffy::Diff) + expect(inconsistency.diff.string1).to eq("#{structure_sql_statement}\n") + expect(inconsistency.diff.string2).to eq("#{database_statement}\n") + end + end + + describe '#error_message' do + it 'returns the error message' do + stub_const "#{validator}::ERROR_MESSAGE", 'error message %s' + + expect(inconsistency.error_message).to eq('error message index_name') + end + end + + describe '#type' do + it 'returns the type of the validator' do + expect(inconsistency.type).to eq('Gitlab::Schema::Validation::Validators::DifferentDefinitionIndexes') + end + end + + describe '#table_name' do + it 'returns the table name' do + expect(inconsistency.table_name).to eq('achievements') + end + end + + describe '#object_type' do + it 'returns the structure sql object type' do + expect(inconsistency.object_type).to eq('Index') + end + + context 'when the structure sql object is not available' do + subject(:inconsistency) { described_class.new(validator, nil, database_object) } + + it 'returns the database object type' do + expect(inconsistency.object_type).to eq('Index') + end + end + end + + describe '#structure_sql_statement' do + it 'returns structure sql statement' do + expect(inconsistency.structure_sql_statement).to eq("#{structure_sql_statement}\n") + end + end + + describe '#database_statement' do + it 'returns database statement' do + expect(inconsistency.database_statement).to eq("#{database_statement}\n") + end + end + + describe '#display' do + let(:expected_output) do + <<~MSG + ------------------------------------------------------ + The index_name index has a different statement between structure.sql and database + Diff: + \e[31m-CREATE INDEX index_name ON public.achievements USING btree (id)\e[0m + \e[32m+CREATE INDEX index_name ON public.achievements USING btree (namespace_id)\e[0m + + ------------------------------------------------------ + MSG + end + + it 'prints the inconsistency message' do + expect(inconsistency.display).to eql(expected_output) + end + end +end diff --git a/gems/gitlab-schema-validation/spec/lib/gitlab/schema/validation/schema_objects/column_spec.rb b/gems/gitlab-schema-validation/spec/lib/gitlab/schema/validation/schema_objects/column_spec.rb new file mode 100644 index 00000000000..c002903e765 --- /dev/null +++ b/gems/gitlab-schema-validation/spec/lib/gitlab/schema/validation/schema_objects/column_spec.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Schema::Validation::SchemaObjects::Column, feature_category: :database do + subject(:column) { described_class.new(adapter) } + + let(:database_adapter) { 'Gitlab::Schema::Validation::Adapters::ColumnDatabaseAdapter' } + let(:adapter) do + instance_double(database_adapter, name: 'id', table_name: 'projects', + data_type: 'bigint', default: nil, nullable: 'NOT NULL') + end + + describe '#name' do + it { expect(column.name).to eq('id') } + end + + describe '#table_name' do + it { expect(column.table_name).to eq('projects') } + end + + describe '#statement' do + it { expect(column.statement).to eq('id bigint NOT NULL') } + end +end diff --git a/gems/gitlab-schema-validation/spec/lib/gitlab/schema/validation/schema_objects/foreign_key_spec.rb b/gems/gitlab-schema-validation/spec/lib/gitlab/schema/validation/schema_objects/foreign_key_spec.rb new file mode 100644 index 00000000000..bfe337b6e7c --- /dev/null +++ b/gems/gitlab-schema-validation/spec/lib/gitlab/schema/validation/schema_objects/foreign_key_spec.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Schema::Validation::SchemaObjects::ForeignKey, feature_category: :database do + subject(:foreign_key) { described_class.new(adapter) } + + let(:database_adapter) { 'Gitlab::Schema::Validation::Adapters::ForeignKeyDatabaseAdapter' } + let(:adapter) do + instance_double(database_adapter, name: 'public.fk_1d37cddf91', table_name: 'vulnerabilities', + statement: 'FOREIGN KEY (epic_id) REFERENCES epics(id) ON DELETE SET NULL') + end + + describe '#name' do + it { expect(foreign_key.name).to eq('public.fk_1d37cddf91') } + end + + describe '#table_name' do + it { expect(foreign_key.table_name).to eq('vulnerabilities') } + end + + describe '#statement' do + it { expect(foreign_key.statement).to eq('FOREIGN KEY (epic_id) REFERENCES epics(id) ON DELETE SET NULL') } + end +end diff --git a/gems/gitlab-schema-validation/spec/lib/gitlab/schema/validation/schema_objects/index_spec.rb b/gems/gitlab-schema-validation/spec/lib/gitlab/schema/validation/schema_objects/index_spec.rb new file mode 100644 index 00000000000..dfef440d99e --- /dev/null +++ b/gems/gitlab-schema-validation/spec/lib/gitlab/schema/validation/schema_objects/index_spec.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Schema::Validation::SchemaObjects::Index, feature_category: :database do + let(:statement) { 'CREATE INDEX index_name ON public.achievements USING btree (namespace_id)' } + let(:name) { 'index_name' } + let(:table_name) { 'achievements' } + + include_examples 'schema objects assertions for', 'index_stmt' +end diff --git a/gems/gitlab-schema-validation/spec/lib/gitlab/schema/validation/schema_objects/table_spec.rb b/gems/gitlab-schema-validation/spec/lib/gitlab/schema/validation/schema_objects/table_spec.rb new file mode 100644 index 00000000000..87555c88edf --- /dev/null +++ b/gems/gitlab-schema-validation/spec/lib/gitlab/schema/validation/schema_objects/table_spec.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Schema::Validation::SchemaObjects::Table, feature_category: :database do + subject(:table) { described_class.new(name, columns) } + + let(:name) { 'my_table' } + let(:column_class) { 'Gitlab::Schema::Validation::SchemaObjects::Column' } + let(:columns) do + [ + instance_double(column_class, name: 'id', statement: 'id bigint NOT NULL', partition_key?: false), + instance_double(column_class, name: 'col', statement: 'col text', partition_key?: false), + instance_double(column_class, name: 'partition', statement: 'partition integer DEFAULT 1', partition_key?: true) + ] + end + + describe '#name' do + it { expect(table.name).to eq('my_table') } + end + + describe '#table_name' do + it { expect(table.table_name).to eq('my_table') } + end + + describe '#statement' do + it { expect(table.statement).to eq('CREATE TABLE my_table (id bigint NOT NULL, col text)') } + + it 'ignores the partition column' do + expect(table.statement).not_to include('partition integer DEFAULT 1') + end + end + + describe '#fetch_column_by_name' do + it { expect(table.fetch_column_by_name('col')).not_to be_nil } + + it { expect(table.fetch_column_by_name('invalid')).to be_nil } + end + + describe '#column_exists?' do + it { expect(table.column_exists?('col')).to be(true) } + + it { expect(table.column_exists?('invalid')).to be(false) } + end +end diff --git a/gems/gitlab-schema-validation/spec/lib/gitlab/schema/validation/schema_objects/trigger_spec.rb b/gems/gitlab-schema-validation/spec/lib/gitlab/schema/validation/schema_objects/trigger_spec.rb new file mode 100644 index 00000000000..b6d0ba38ebb --- /dev/null +++ b/gems/gitlab-schema-validation/spec/lib/gitlab/schema/validation/schema_objects/trigger_spec.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Schema::Validation::SchemaObjects::Trigger, feature_category: :database do + let(:statement) { 'CREATE TRIGGER my_trigger BEFORE INSERT ON todos FOR EACH ROW EXECUTE FUNCTION trigger()' } + let(:name) { 'my_trigger' } + let(:table_name) { 'todos' } + + include_examples 'schema objects assertions for', 'create_trig_stmt' +end diff --git a/gems/gitlab-schema-validation/spec/lib/gitlab/schema/validation/sources/structure_sql_spec.rb b/gems/gitlab-schema-validation/spec/lib/gitlab/schema/validation/sources/structure_sql_spec.rb new file mode 100644 index 00000000000..7d4a23b1619 --- /dev/null +++ b/gems/gitlab-schema-validation/spec/lib/gitlab/schema/validation/sources/structure_sql_spec.rb @@ -0,0 +1,66 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.shared_examples 'structure sql schema assertions for' do |object_exists_method, all_objects_method| + subject(:structure_sql) { described_class.new(structure_file_path, schema_name) } + + let(:structure_file_path) { 'spec/fixtures/structure.sql' } + let(:schema_name) { 'public' } + + describe "##{object_exists_method}" do + it 'returns true when schema object exists' do + expect(structure_sql.public_send(object_exists_method, valid_schema_object_name)).to be_truthy + end + + it 'returns false when schema object does not exists' do + expect(structure_sql.public_send(object_exists_method, 'invalid-object-name')).to be_falsey + end + end + + describe "##{all_objects_method}" do + it 'returns all the schema objects' do + schema_objects = structure_sql.public_send(all_objects_method) + + expect(schema_objects).to all(be_a(schema_object)) + expect(schema_objects.map(&:name)).to eq(expected_objects) + end + end +end + +RSpec.describe Gitlab::Schema::Validation::Sources::StructureSql, feature_category: :database do + let(:structure_file_path) { 'spec/fixtures/structure.sql' } + let(:schema_name) { 'public' } + + subject(:structure_sql) { described_class.new(structure_file_path, schema_name) } + + context 'when having indexes' do + let(:schema_object) { Gitlab::Schema::Validation::SchemaObjects::Index } + let(:valid_schema_object_name) { 'index' } + let(:expected_objects) do + %w[missing_index wrong_index index index_namespaces_public_groups_name_id + index_on_deploy_keys_id_and_type_and_public index_users_on_public_email_excluding_null_and_empty] + end + + include_examples 'structure sql schema assertions for', 'index_exists?', 'indexes' + end + + context 'when having triggers' do + let(:schema_object) { Gitlab::Schema::Validation::SchemaObjects::Trigger } + let(:valid_schema_object_name) { 'trigger' } + let(:expected_objects) { %w[trigger wrong_trigger missing_trigger_1 projects_loose_fk_trigger] } + + include_examples 'structure sql schema assertions for', 'trigger_exists?', 'triggers' + end + + context 'when having tables' do + let(:schema_object) { Gitlab::Schema::Validation::SchemaObjects::Table } + let(:valid_schema_object_name) { 'test_table' } + let(:expected_objects) do + %w[test_table ci_project_mirrors wrong_table extra_table_columns missing_table missing_table_columns + operations_user_lists] + end + + include_examples 'structure sql schema assertions for', 'table_exists?', 'tables' + end +end diff --git a/gems/gitlab-schema-validation/spec/lib/gitlab/schema/validation/validators/base_spec.rb b/gems/gitlab-schema-validation/spec/lib/gitlab/schema/validation/validators/base_spec.rb new file mode 100644 index 00000000000..50be1f1b373 --- /dev/null +++ b/gems/gitlab-schema-validation/spec/lib/gitlab/schema/validation/validators/base_spec.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Schema::Validation::Validators::Base, feature_category: :database do + describe '#execute' do + let(:structure_sql) { instance_double(Gitlab::Schema::Validation::Sources::StructureSql) } + let(:database) { instance_double(Gitlab::Schema::Validation::Sources::Database) } + + subject(:inconsistencies) { described_class.new(structure_sql, database).execute } + + describe '.all_validators' do + subject(:all_validators) { described_class.all_validators } + + it 'returns an array of all validators' do + expect(all_validators).to eq([ + Gitlab::Schema::Validation::Validators::ExtraTables, + Gitlab::Schema::Validation::Validators::ExtraTableColumns, + Gitlab::Schema::Validation::Validators::ExtraIndexes, + Gitlab::Schema::Validation::Validators::ExtraTriggers, + Gitlab::Schema::Validation::Validators::ExtraForeignKeys, + Gitlab::Schema::Validation::Validators::MissingTables, + Gitlab::Schema::Validation::Validators::MissingTableColumns, + Gitlab::Schema::Validation::Validators::MissingIndexes, + Gitlab::Schema::Validation::Validators::MissingTriggers, + Gitlab::Schema::Validation::Validators::MissingForeignKeys, + Gitlab::Schema::Validation::Validators::DifferentDefinitionTables, + Gitlab::Schema::Validation::Validators::DifferentDefinitionIndexes, + Gitlab::Schema::Validation::Validators::DifferentDefinitionTriggers, + Gitlab::Schema::Validation::Validators::DifferentDefinitionForeignKeys + ]) + end + end + + it 'raises an exception' do + expect { inconsistencies }.to raise_error(NoMethodError) + end + end +end diff --git a/gems/gitlab-schema-validation/spec/lib/gitlab/schema/validation/validators/different_definition_indexes_spec.rb b/gems/gitlab-schema-validation/spec/lib/gitlab/schema/validation/validators/different_definition_indexes_spec.rb new file mode 100644 index 00000000000..c1795a56063 --- /dev/null +++ b/gems/gitlab-schema-validation/spec/lib/gitlab/schema/validation/validators/different_definition_indexes_spec.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Schema::Validation::Validators::DifferentDefinitionIndexes do + include_examples 'index validators', described_class, ['wrong_index'] +end diff --git a/gems/gitlab-schema-validation/spec/lib/gitlab/schema/validation/validators/different_definition_tables_spec.rb b/gems/gitlab-schema-validation/spec/lib/gitlab/schema/validation/validators/different_definition_tables_spec.rb new file mode 100644 index 00000000000..d1c9169a59a --- /dev/null +++ b/gems/gitlab-schema-validation/spec/lib/gitlab/schema/validation/validators/different_definition_tables_spec.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Schema::Validation::Validators::DifferentDefinitionTables, feature_category: :database do + include_examples 'table validators', described_class, ['wrong_table'] +end diff --git a/gems/gitlab-schema-validation/spec/lib/gitlab/schema/validation/validators/different_definition_triggers_spec.rb b/gems/gitlab-schema-validation/spec/lib/gitlab/schema/validation/validators/different_definition_triggers_spec.rb new file mode 100644 index 00000000000..a2597e3e55e --- /dev/null +++ b/gems/gitlab-schema-validation/spec/lib/gitlab/schema/validation/validators/different_definition_triggers_spec.rb @@ -0,0 +1,8 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Schema::Validation::Validators::DifferentDefinitionTriggers, + feature_category: :database do + include_examples 'trigger validators', described_class, ['wrong_trigger'] +end diff --git a/gems/gitlab-schema-validation/spec/lib/gitlab/schema/validation/validators/extra_foreign_keys_spec.rb b/gems/gitlab-schema-validation/spec/lib/gitlab/schema/validation/validators/extra_foreign_keys_spec.rb new file mode 100644 index 00000000000..499f2578fd2 --- /dev/null +++ b/gems/gitlab-schema-validation/spec/lib/gitlab/schema/validation/validators/extra_foreign_keys_spec.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Schema::Validation::Validators::ExtraForeignKeys, feature_category: :database do + include_examples 'foreign key validators', described_class, ['public.extra_fk'] +end diff --git a/gems/gitlab-schema-validation/spec/lib/gitlab/schema/validation/validators/extra_indexes_spec.rb b/gems/gitlab-schema-validation/spec/lib/gitlab/schema/validation/validators/extra_indexes_spec.rb new file mode 100644 index 00000000000..01498444ba4 --- /dev/null +++ b/gems/gitlab-schema-validation/spec/lib/gitlab/schema/validation/validators/extra_indexes_spec.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Schema::Validation::Validators::ExtraIndexes, feature_category: :database do + include_examples 'index validators', described_class, ['extra_index'] +end diff --git a/gems/gitlab-schema-validation/spec/lib/gitlab/schema/validation/validators/extra_table_columns_spec.rb b/gems/gitlab-schema-validation/spec/lib/gitlab/schema/validation/validators/extra_table_columns_spec.rb new file mode 100644 index 00000000000..f5d06e9941a --- /dev/null +++ b/gems/gitlab-schema-validation/spec/lib/gitlab/schema/validation/validators/extra_table_columns_spec.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Schema::Validation::Validators::ExtraTableColumns, feature_category: :database do + include_examples 'table validators', described_class, ['extra_table_columns'] +end diff --git a/gems/gitlab-schema-validation/spec/lib/gitlab/schema/validation/validators/extra_tables_spec.rb b/gems/gitlab-schema-validation/spec/lib/gitlab/schema/validation/validators/extra_tables_spec.rb new file mode 100644 index 00000000000..15c52fe4719 --- /dev/null +++ b/gems/gitlab-schema-validation/spec/lib/gitlab/schema/validation/validators/extra_tables_spec.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Schema::Validation::Validators::ExtraTables, feature_category: :database do + include_examples 'table validators', described_class, ['extra_table'] +end diff --git a/gems/gitlab-schema-validation/spec/lib/gitlab/schema/validation/validators/extra_triggers_spec.rb b/gems/gitlab-schema-validation/spec/lib/gitlab/schema/validation/validators/extra_triggers_spec.rb new file mode 100644 index 00000000000..97126aebf05 --- /dev/null +++ b/gems/gitlab-schema-validation/spec/lib/gitlab/schema/validation/validators/extra_triggers_spec.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Schema::Validation::Validators::ExtraTriggers, feature_category: :database do + include_examples 'trigger validators', described_class, ['extra_trigger'] +end diff --git a/gems/gitlab-schema-validation/spec/lib/gitlab/schema/validation/validators/missing_foreign_keys_spec.rb b/gems/gitlab-schema-validation/spec/lib/gitlab/schema/validation/validators/missing_foreign_keys_spec.rb new file mode 100644 index 00000000000..6682c3f623d --- /dev/null +++ b/gems/gitlab-schema-validation/spec/lib/gitlab/schema/validation/validators/missing_foreign_keys_spec.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Schema::Validation::Validators::MissingForeignKeys, feature_category: :database do + include_examples 'foreign key validators', described_class, %w[public.fk_rails_536b96bff1 public.missing_fk] +end diff --git a/gems/gitlab-schema-validation/spec/lib/gitlab/schema/validation/validators/missing_indexes_spec.rb b/gems/gitlab-schema-validation/spec/lib/gitlab/schema/validation/validators/missing_indexes_spec.rb new file mode 100644 index 00000000000..c1cb9a2416b --- /dev/null +++ b/gems/gitlab-schema-validation/spec/lib/gitlab/schema/validation/validators/missing_indexes_spec.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Schema::Validation::Validators::MissingIndexes, feature_category: :database do + missing_indexes = %w[ + missing_index + index_namespaces_public_groups_name_id + index_on_deploy_keys_id_and_type_and_public + index_users_on_public_email_excluding_null_and_empty + ] + + include_examples 'index validators', described_class, missing_indexes +end diff --git a/gems/gitlab-schema-validation/spec/lib/gitlab/schema/validation/validators/missing_table_columns_spec.rb b/gems/gitlab-schema-validation/spec/lib/gitlab/schema/validation/validators/missing_table_columns_spec.rb new file mode 100644 index 00000000000..3866bdce071 --- /dev/null +++ b/gems/gitlab-schema-validation/spec/lib/gitlab/schema/validation/validators/missing_table_columns_spec.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Schema::Validation::Validators::MissingTableColumns, feature_category: :database do + include_examples 'table validators', described_class, ['missing_table_columns'] +end diff --git a/gems/gitlab-schema-validation/spec/lib/gitlab/schema/validation/validators/missing_tables_spec.rb b/gems/gitlab-schema-validation/spec/lib/gitlab/schema/validation/validators/missing_tables_spec.rb new file mode 100644 index 00000000000..8a73d67ab7d --- /dev/null +++ b/gems/gitlab-schema-validation/spec/lib/gitlab/schema/validation/validators/missing_tables_spec.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Schema::Validation::Validators::MissingTables, feature_category: :database do + missing_tables = %w[ci_project_mirrors missing_table operations_user_lists test_table] + + include_examples 'table validators', described_class, missing_tables +end diff --git a/gems/gitlab-schema-validation/spec/lib/gitlab/schema/validation/validators/missing_triggers_spec.rb b/gems/gitlab-schema-validation/spec/lib/gitlab/schema/validation/validators/missing_triggers_spec.rb new file mode 100644 index 00000000000..82b9b034503 --- /dev/null +++ b/gems/gitlab-schema-validation/spec/lib/gitlab/schema/validation/validators/missing_triggers_spec.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Schema::Validation::Validators::MissingTriggers, feature_category: :database do + missing_triggers = %w[missing_trigger_1 projects_loose_fk_trigger] + + include_examples 'trigger validators', described_class, missing_triggers +end diff --git a/gems/gitlab-schema-validation/spec/spec_helper.rb b/gems/gitlab-schema-validation/spec/spec_helper.rb new file mode 100644 index 00000000000..c11c5021e3b --- /dev/null +++ b/gems/gitlab-schema-validation/spec/spec_helper.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +require "gitlab/schema/validation" +require 'rspec-parameterized' + +RSpec.configure do |config| + # Enable flags like --only-failures and --next-failure + config.example_status_persistence_file_path = ".rspec_status" + + # Disable RSpec exposing methods globally on `Module` and `main` + config.disable_monkey_patching! + + Dir['./spec/support/**/*.rb'].each { |f| require f } + + config.expect_with :rspec do |c| + c.syntax = :expect + end +end diff --git a/gems/gitlab-schema-validation/spec/support/shared_examples/foreign_key_validators_shared_examples.rb b/gems/gitlab-schema-validation/spec/support/shared_examples/foreign_key_validators_shared_examples.rb new file mode 100644 index 00000000000..1f33c8bd760 --- /dev/null +++ b/gems/gitlab-schema-validation/spec/support/shared_examples/foreign_key_validators_shared_examples.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.shared_examples 'foreign key validators' do |validator, expected_result| + subject(:result) { validator.new(structure_file, database).execute } + + let(:structure_file_path) { 'spec/fixtures/structure.sql' } + let(:structure_file) { Gitlab::Schema::Validation::Sources::StructureSql.new(structure_file_path, schema) } + let(:inconsistency_type) { validator.to_s } + let(:database_name) { 'main' } + let(:schema) { 'public' } + # rubocop:disable RSpec/VerifiedDoubleReference + let(:connection) { instance_double('connection', exec_query: database_query, current_schema: 'public') } + # rubocop:enable RSpec/VerifiedDoubleReference + + let(:database) { Gitlab::Schema::Validation::Sources::Database.new(connection) } + + let(:database_query) do + [ + { + 'schema' => schema, + 'table_name' => 'web_hooks', + 'foreign_key_name' => 'web_hooks_project_id_fkey', + 'foreign_key_definition' => 'FOREIGN KEY (project_id) REFERENCES projects(id) ON DELETE CASCADE' + }, + { + 'schema' => schema, + 'table_name' => 'issues', + 'foreign_key_name' => 'wrong_definition_fk', + 'foreign_key_definition' => 'FOREIGN KEY (author_id) REFERENCES users(id) ON DELETE CASCADE' + }, + { + 'schema' => schema, + 'table_name' => 'projects', + 'foreign_key_name' => 'extra_fk', + 'foreign_key_definition' => 'FOREIGN KEY (creator_id) REFERENCES users(id) ON DELETE CASCADE' + } + ] + end + + it 'returns trigger inconsistencies' do + expect(result.map(&:object_name)).to match_array(expected_result) + expect(result.map(&:type)).to all(eql inconsistency_type) + end +end diff --git a/gems/gitlab-schema-validation/spec/support/shared_examples/index_validators_shared_examples.rb b/gems/gitlab-schema-validation/spec/support/shared_examples/index_validators_shared_examples.rb new file mode 100644 index 00000000000..cc20c0dc765 --- /dev/null +++ b/gems/gitlab-schema-validation/spec/support/shared_examples/index_validators_shared_examples.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.shared_examples 'index validators' do |validator, expected_result| + let(:structure_file_path) { 'spec/fixtures/structure.sql' } + let(:database_indexes) do + [ + ['wrong_index', 'CREATE UNIQUE INDEX wrong_index ON public.table_name (column_name)'], + ['extra_index', 'CREATE INDEX extra_index ON public.table_name (column_name)'], + ['index', 'CREATE UNIQUE INDEX "index" ON public.achievements USING btree (namespace_id, lower(name))'] + ] + end + + let(:inconsistency_type) { validator.name } + + # rubocop:disable RSpec/VerifiedDoubleReference + let(:connection) { instance_double('connection', select_rows: database_indexes, current_schema: 'public') } + # rubocop:enable RSpec/VerifiedDoubleReference + + let(:schema) { 'public' } + + let(:database) { Gitlab::Schema::Validation::Sources::Database.new(connection) } + let(:structure_file) { Gitlab::Schema::Validation::Sources::StructureSql.new(structure_file_path, schema) } + + subject(:result) { validator.new(structure_file, database).execute } + + it 'returns index inconsistencies' do + expect(result.map(&:object_name)).to match_array(expected_result) + expect(result.map(&:type)).to all(eql inconsistency_type) + end +end diff --git a/gems/gitlab-schema-validation/spec/support/shared_examples/schema_objects_shared_examples.rb b/gems/gitlab-schema-validation/spec/support/shared_examples/schema_objects_shared_examples.rb new file mode 100644 index 00000000000..994b30b0941 --- /dev/null +++ b/gems/gitlab-schema-validation/spec/support/shared_examples/schema_objects_shared_examples.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.shared_examples 'schema objects assertions for' do |stmt_name| + let(:stmt) { PgQuery.parse(statement).tree.stmts.first.stmt } + let(:schema_object) { described_class.new(stmt.public_send(stmt_name)) } + + describe '#name' do + it 'returns schema object name' do + expect(schema_object.name).to eq(name) + end + end + + describe '#statement' do + it 'returns schema object statement' do + expect(schema_object.statement).to eq(statement) + end + end + + describe '#table_name' do + it 'returns schema object table_name' do + expect(schema_object.table_name).to eq(table_name) + end + end +end diff --git a/gems/gitlab-schema-validation/spec/support/shared_examples/table_validators_shared_examples.rb b/gems/gitlab-schema-validation/spec/support/shared_examples/table_validators_shared_examples.rb new file mode 100644 index 00000000000..d2a51a9b202 --- /dev/null +++ b/gems/gitlab-schema-validation/spec/support/shared_examples/table_validators_shared_examples.rb @@ -0,0 +1,81 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.shared_examples "table validators" do |validator, expected_result| + subject(:result) { validator.new(structure_file, database).execute } + + let(:structure_file_path) { 'spec/fixtures/structure.sql' } + let(:inconsistency_type) { validator.to_s } + # rubocop:disable RSpec/VerifiedDoubleReference + let(:connection) { instance_double('connection', exec_query: database_tables, current_schema: 'public') } + # rubocop:enable RSpec/VerifiedDoubleReference + let(:schema) { 'public' } + let(:database) { Gitlab::Schema::Validation::Sources::Database.new(connection) } + let(:structure_file) { Gitlab::Schema::Validation::Sources::StructureSql.new(structure_file_path, schema) } + let(:database_tables) do + [ + { + 'table_name' => 'wrong_table', + 'column_name' => 'id', + 'not_null' => true, + 'data_type' => 'integer', + 'column_default' => "nextval('audit_events_id_seq'::regclass)" + }, + { + 'table_name' => 'wrong_table', + 'column_name' => 'description', + 'not_null' => true, + 'data_type' => 'character varying', + 'column_default' => nil + }, + { + 'table_name' => 'extra_table', + 'column_name' => 'id', + 'not_null' => true, + 'data_type' => 'integer', + 'column_default' => "nextval('audit_events_id_seq'::regclass)" + }, + { + 'table_name' => 'extra_table', + 'column_name' => 'email', + 'not_null' => true, + 'data_type' => 'character varying', + 'column_default' => nil + }, + { + 'table_name' => 'extra_table_columns', + 'column_name' => 'id', + 'not_null' => true, + 'data_type' => 'bigint', + 'column_default' => "nextval('audit_events_id_seq'::regclass)" + }, + { + 'table_name' => 'extra_table_columns', + 'column_name' => 'name', + 'not_null' => true, + 'data_type' => 'character varying(255)', + 'column_default' => nil + }, + { + 'table_name' => 'extra_table_columns', + 'column_name' => 'extra_column', + 'not_null' => true, + 'data_type' => 'character varying(255)', + 'column_default' => nil + }, + { + 'table_name' => 'missing_table_columns', + 'column_name' => 'id', + 'not_null' => true, + 'data_type' => 'bigint', + 'column_default' => 'NOT NULL' + } + ] + end + + it 'returns table inconsistencies' do + expect(result.map(&:object_name)).to match_array(expected_result) + expect(result.map(&:type)).to all(eql inconsistency_type) + end +end diff --git a/gems/gitlab-schema-validation/spec/support/shared_examples/trigger_validators_shared_examples.rb b/gems/gitlab-schema-validation/spec/support/shared_examples/trigger_validators_shared_examples.rb new file mode 100644 index 00000000000..45ed87082bb --- /dev/null +++ b/gems/gitlab-schema-validation/spec/support/shared_examples/trigger_validators_shared_examples.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.shared_examples 'trigger validators' do |validator, expected_result| + subject(:result) { validator.new(structure_file, database).execute } + + let(:structure_file_path) { 'spec/fixtures/structure.sql' } + let(:structure_file) { Gitlab::Schema::Validation::Sources::StructureSql.new(structure_file_path, schema) } + let(:inconsistency_type) { validator.to_s } + let(:database_name) { 'main' } + let(:schema) { 'public' } + let(:database) { Gitlab::Schema::Validation::Sources::Database.new(connection) } + + # rubocop:disable RSpec/VerifiedDoubleReference + let(:connection) { instance_double('connection', select_rows: database_triggers, current_schema: 'public') } + # rubocop:enable RSpec/VerifiedDoubleReference + + let(:database_triggers) do + [ + ['trigger', 'CREATE TRIGGER trigger AFTER INSERT ON public.t1 FOR EACH ROW EXECUTE FUNCTION t1()'], + ['wrong_trigger', 'CREATE TRIGGER wrong_trigger BEFORE UPDATE ON public.t2 FOR EACH ROW EXECUTE FUNCTION t2()'], + ['extra_trigger', 'CREATE TRIGGER extra_trigger BEFORE INSERT ON public.t4 FOR EACH ROW EXECUTE FUNCTION t4()'] + ] + end + + it 'returns trigger inconsistencies' do + expect(result.map(&:object_name)).to match_array(expected_result) + expect(result.map(&:type)).to all(eql inconsistency_type) + end +end diff --git a/gems/gitlab-utils/.gitignore b/gems/gitlab-utils/.gitignore new file mode 100644 index 00000000000..b04a8c840df --- /dev/null +++ b/gems/gitlab-utils/.gitignore @@ -0,0 +1,11 @@ +/.bundle/ +/.yardoc +/_yardoc/ +/coverage/ +/doc/ +/pkg/ +/spec/reports/ +/tmp/ + +# rspec failure tracking +.rspec_status diff --git a/gems/gitlab-utils/.gitlab-ci.yml b/gems/gitlab-utils/.gitlab-ci.yml new file mode 100644 index 00000000000..a9e984d4785 --- /dev/null +++ b/gems/gitlab-utils/.gitlab-ci.yml @@ -0,0 +1,4 @@ +include: + - local: gems/gem.gitlab-ci.yml + inputs: + gem_name: "gitlab-utils" diff --git a/gems/gitlab-utils/.rspec b/gems/gitlab-utils/.rspec new file mode 100644 index 00000000000..34c5164d9b5 --- /dev/null +++ b/gems/gitlab-utils/.rspec @@ -0,0 +1,3 @@ +--format documentation +--color +--require spec_helper diff --git a/gems/gitlab-utils/.rubocop.yml b/gems/gitlab-utils/.rubocop.yml new file mode 100644 index 00000000000..eeafd850c9b --- /dev/null +++ b/gems/gitlab-utils/.rubocop.yml @@ -0,0 +1,27 @@ +inherit_from: + - ../config/rubocop.yml + +RSpec/InstanceVariable: + Exclude: + - spec/**/*.rb + +Lint/BinaryOperatorWithIdenticalOperands: + Exclude: + - spec/**/*.rb + +# We use EnforcedStyle of comparison here due to it being better +# performing code as seen in https://gitlab.com/gitlab-org/gitlab/-/merge_requests/36221#note_375659681 +Style/NumericPredicate: + EnforcedStyle: comparison + +# FIXME: When enabled, there's a spec failure in ee/spec/requests/api/graphql/mutations/merge_requests/update_approval_rule_spec.rb:51, +# due to the `default_value` of `remove_hidden_groups` set to `[]`, most probably instead of `false`, in ee/app/graphql/mutations/merge_requests/update_approval_rule.rb. +# The problem is that `Object#=~` exists (even though it's deprecated), hence calling it on an `Array` doesn't blow up, but `Array#match?` doesn't exist. +Performance/RegexpMatch: + Exclude: + - lib/gitlab/utils.rb + +Rails/OutputSafety: + Details: grace period + Exclude: + - 'lib/gitlab/utils.rb' diff --git a/gems/gitlab-utils/Gemfile b/gems/gitlab-utils/Gemfile new file mode 100644 index 00000000000..2c7228c874c --- /dev/null +++ b/gems/gitlab-utils/Gemfile @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +source "https://rubygems.org" + +# Specify your gem's dependencies in gitlab-utils.gemspec +gemspec + +group :development, :test do + gem 'gitlab-rspec', path: '../gitlab-rspec' +end diff --git a/gems/gitlab-utils/Gemfile.lock b/gems/gitlab-utils/Gemfile.lock new file mode 100644 index 00000000000..971d90ce146 --- /dev/null +++ b/gems/gitlab-utils/Gemfile.lock @@ -0,0 +1,199 @@ +PATH + remote: ../gitlab-rspec + specs: + gitlab-rspec (0.1.0) + activesupport (>= 6.1, < 7.1) + rspec (~> 3.0) + +PATH + remote: . + specs: + gitlab-utils (0.1.0) + actionview (>= 6.1.7.2) + activesupport (>= 6.1.7.2) + addressable (~> 2.8) + nokogiri (~> 1.15.2) + rake (~> 13.0) + +GEM + remote: https://rubygems.org/ + specs: + actionpack (7.0.5) + actionview (= 7.0.5) + activesupport (= 7.0.5) + rack (~> 2.0, >= 2.2.4) + rack-test (>= 0.6.3) + rails-dom-testing (~> 2.0) + rails-html-sanitizer (~> 1.0, >= 1.2.0) + actionview (7.0.5) + activesupport (= 7.0.5) + builder (~> 3.1) + erubi (~> 1.4) + rails-dom-testing (~> 2.0) + rails-html-sanitizer (~> 1.1, >= 1.2.0) + activesupport (7.0.5) + concurrent-ruby (~> 1.0, >= 1.0.2) + i18n (>= 1.6, < 2) + minitest (>= 5.1) + tzinfo (~> 2.0) + addressable (2.8.1) + public_suffix (>= 2.0.2, < 6.0) + ast (2.4.2) + benchmark-malloc (0.2.0) + benchmark-perf (0.6.0) + benchmark-trend (0.4.0) + binding_of_caller (1.0.0) + debug_inspector (>= 0.0.1) + builder (3.2.4) + coderay (1.1.3) + concurrent-ruby (1.2.2) + crass (1.0.6) + debug_inspector (1.1.0) + diff-lcs (1.5.0) + erubi (1.12.0) + factory_bot (6.2.1) + activesupport (>= 5.0.0) + factory_bot_rails (6.2.0) + factory_bot (~> 6.2.0) + railties (>= 5.0.0) + gitlab-styles (10.1.0) + rubocop (~> 1.50.2) + rubocop-graphql (~> 0.18) + rubocop-performance (~> 1.15) + rubocop-rails (~> 2.17) + rubocop-rspec (~> 2.22) + i18n (1.14.1) + concurrent-ruby (~> 1.0) + json (2.6.3) + loofah (2.21.3) + crass (~> 1.0.2) + nokogiri (>= 1.12.0) + method_source (1.0.0) + mini_portile2 (2.8.2) + minitest (5.18.1) + nokogiri (1.15.2) + mini_portile2 (~> 2.8.2) + racc (~> 1.4) + parallel (1.23.0) + parser (3.2.2.3) + ast (~> 2.4.1) + racc + proc_to_ast (0.1.0) + coderay + parser + unparser + public_suffix (5.0.0) + racc (1.7.1) + rack (2.2.7) + rack-test (2.1.0) + rack (>= 1.3) + rails-dom-testing (2.0.3) + activesupport (>= 4.2.0) + nokogiri (>= 1.6) + rails-html-sanitizer (1.6.0) + loofah (~> 2.21) + nokogiri (~> 1.14) + railties (7.0.5) + actionpack (= 7.0.5) + activesupport (= 7.0.5) + method_source + rake (>= 12.2) + thor (~> 1.0) + zeitwerk (~> 2.5) + rainbow (3.1.1) + rake (13.0.6) + regexp_parser (2.8.1) + rexml (3.2.5) + rspec (3.12.0) + rspec-core (~> 3.12.0) + rspec-expectations (~> 3.12.0) + rspec-mocks (~> 3.12.0) + rspec-benchmark (0.6.0) + benchmark-malloc (~> 0.2) + benchmark-perf (~> 0.6) + benchmark-trend (~> 0.4) + rspec (>= 3.0) + rspec-core (3.12.0) + rspec-support (~> 3.12.0) + rspec-expectations (3.12.2) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.12.0) + rspec-mocks (3.12.3) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.12.0) + rspec-parameterized (1.0.0) + rspec-parameterized-core (< 2) + rspec-parameterized-table_syntax (< 2) + rspec-parameterized-core (1.0.0) + parser + proc_to_ast + rspec (>= 2.13, < 4) + unparser + rspec-parameterized-table_syntax (1.0.0) + binding_of_caller + rspec-parameterized-core (< 2) + rspec-rails (6.0.1) + actionpack (>= 6.1) + activesupport (>= 6.1) + railties (>= 6.1) + rspec-core (~> 3.11) + rspec-expectations (~> 3.11) + rspec-mocks (~> 3.11) + rspec-support (~> 3.11) + rspec-support (3.12.0) + rubocop (1.50.2) + json (~> 2.3) + parallel (~> 1.10) + parser (>= 3.2.0.0) + rainbow (>= 2.2.2, < 4.0) + regexp_parser (>= 1.8, < 3.0) + rexml (>= 3.2.5, < 4.0) + rubocop-ast (>= 1.28.0, < 2.0) + ruby-progressbar (~> 1.7) + unicode-display_width (>= 2.4.0, < 3.0) + rubocop-ast (1.29.0) + parser (>= 3.2.1.0) + rubocop-capybara (2.18.0) + rubocop (~> 1.41) + rubocop-factory_bot (2.23.1) + rubocop (~> 1.33) + rubocop-graphql (0.19.0) + rubocop (>= 0.87, < 2) + rubocop-performance (1.18.0) + rubocop (>= 1.7.0, < 2.0) + rubocop-ast (>= 0.4.0) + rubocop-rails (2.20.2) + activesupport (>= 4.2.0) + rack (>= 1.1) + rubocop (>= 1.33.0, < 2.0) + rubocop-rspec (2.22.0) + rubocop (~> 1.33) + rubocop-capybara (~> 2.17) + rubocop-factory_bot (~> 2.22) + ruby-progressbar (1.13.0) + thor (1.2.2) + tzinfo (2.0.6) + concurrent-ruby (~> 1.0) + unicode-display_width (2.4.2) + unparser (0.6.7) + diff-lcs (~> 1.3) + parser (>= 3.2.0) + zeitwerk (2.6.8) + +PLATFORMS + ruby + +DEPENDENCIES + factory_bot_rails (~> 6.2.0) + gitlab-rspec! + gitlab-styles (~> 10.1.0) + gitlab-utils! + rspec (~> 3.12) + rspec-benchmark (~> 0.6.0) + rspec-parameterized (~> 1.0) + rspec-rails (~> 6.0.1) + rubocop (~> 1.50) + rubocop-rspec (~> 2.22) + +BUNDLED WITH + 2.4.4 diff --git a/gems/gitlab-utils/README.md b/gems/gitlab-utils/README.md new file mode 100644 index 00000000000..f7c7d83888b --- /dev/null +++ b/gems/gitlab-utils/README.md @@ -0,0 +1,8 @@ +# Gitlab::Utils + +This Gem contains all code that is not dependent on application code +or business logic and provides a generic functions like: + +- safe parsing of YAML +- version comparisions +- `strong_memoize` diff --git a/gems/gitlab-utils/Rakefile b/gems/gitlab-utils/Rakefile new file mode 100644 index 00000000000..cca71754493 --- /dev/null +++ b/gems/gitlab-utils/Rakefile @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +require "bundler/gem_tasks" +require "rspec/core/rake_task" + +RSpec::Core::RakeTask.new(:spec) + +require "rubocop/rake_task" + +RuboCop::RakeTask.new + +task default: %i[spec rubocop] diff --git a/gems/gitlab-utils/gitlab-utils.gemspec b/gems/gitlab-utils/gitlab-utils.gemspec new file mode 100644 index 00000000000..d5f6deb7fe6 --- /dev/null +++ b/gems/gitlab-utils/gitlab-utils.gemspec @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +require_relative "lib/gitlab/utils/version" + +Gem::Specification.new do |spec| + spec.name = "gitlab-utils" + spec.version = Gitlab::Utils::Version::VERSION + spec.authors = ["group::tenant scale"] + spec.email = ["engineering@gitlab.com"] + + spec.summary = "GitLab common helper methods" + spec.description = "A set of useful helpers methods to perform various conversions and checks." + spec.homepage = "https://gitlab.com/gitlab-org/gitlab/-/tree/master/gems/gitlab-utils" + spec.license = "MIT" + spec.required_ruby_version = ">= 3.0" + spec.metadata["rubygems_mfa_required"] = "true" + + spec.files = Dir["lib/**/*.rb"] + spec.require_paths = ["lib"] + + spec.add_runtime_dependency "actionview", ">= 6.1.7.2" + spec.add_runtime_dependency "activesupport", ">= 6.1.7.2" + spec.add_runtime_dependency "addressable", "~> 2.8" + spec.add_runtime_dependency "nokogiri", "~> 1.15.2" + spec.add_runtime_dependency "rake", "~> 13.0" + + spec.add_development_dependency "factory_bot_rails", "~> 6.2.0" + spec.add_development_dependency "gitlab-styles", "~> 10.1.0" + spec.add_development_dependency "rspec", "~> 3.12" + spec.add_development_dependency "rspec-benchmark", "~> 0.6.0" + spec.add_development_dependency "rspec-parameterized", "~> 1.0" + spec.add_development_dependency "rspec-rails", "~> 6.0.1" + spec.add_development_dependency "rubocop", "~> 1.50" + spec.add_development_dependency "rubocop-rspec", "~> 2.22" +end diff --git a/gems/gitlab-utils/lib/gitlab/utils.rb b/gems/gitlab-utils/lib/gitlab/utils.rb new file mode 100644 index 00000000000..4e08ee8fcaf --- /dev/null +++ b/gems/gitlab-utils/lib/gitlab/utils.rb @@ -0,0 +1,263 @@ +# frozen_string_literal: true + +require "addressable/uri" +require "active_support/all" +require "action_view" + +module Gitlab + module Utils + extend self + DoubleEncodingError = Class.new(StandardError) + + def allowlisted?(absolute_path, allowlist) + path = absolute_path.downcase + + allowlist.map(&:downcase).any? do |allowed_path| + path.start_with?(allowed_path) + end + end + + def decode_path(encoded_path) + decoded = CGI.unescape(encoded_path) + if decoded != CGI.unescape(decoded) # rubocop:disable Style/IfUnlessModifier + raise DoubleEncodingError, "path #{encoded_path} is not allowed" + end + + decoded + end + + def force_utf8(str) + str.dup.force_encoding(Encoding::UTF_8) + end + + def ensure_utf8_size(str, bytes:) + raise ArgumentError, 'Empty string provided!' if str.empty? + raise ArgumentError, 'Negative string size provided!' if bytes < 0 + + truncated = str.each_char.each_with_object(+'') do |char, object| + if object.bytesize + char.bytesize > bytes # rubocop:disable Style/GuardClause + break object + else + object.concat(char) + end + end + + truncated + ('0' * (bytes - truncated.bytesize)) + end + + # Append path to host, making sure there's one single / in between + def append_path(host, path) + "#{host.to_s.sub(%r{\/+$}, '')}/#{remove_leading_slashes(path)}" # rubocop:disable Style/RedundantRegexpEscape + end + + def remove_leading_slashes(str) + str.to_s.sub(%r{^/+}, '') + end + + # A slugified version of the string, suitable for inclusion in URLs and + # domain names. Rules: + # + # * Lowercased + # * Anything not matching [a-z0-9-] is replaced with a - + # * Maximum length is 63 bytes + # * First/Last Character is not a hyphen + def slugify(str) + str.downcase + .gsub(/[^a-z0-9]/, '-')[0..62] + .gsub(/(\A-+|-+\z)/, '') + end + + # Converts newlines into HTML line break elements + def nlbr(str) + ActionView::Base.full_sanitizer.sanitize(+str, tags: []).gsub(/\r?\n/, '<br>').html_safe + end + + def remove_line_breaks(str) + str.gsub(/\r?\n/, '') + end + + def to_boolean(value, default: nil) + value = value.to_s if [0, 1].include?(value) + + return value if [true, false].include?(value) + return true if value =~ /^(true|t|yes|y|1|on)$/i + return false if value =~ /^(false|f|no|n|0|off)$/i + + default + end + + def boolean_to_yes_no(bool) + if bool + 'Yes' + else + 'No' + end + end + + # Behaves like `which` on Linux machines: given PATH, try to resolve the given + # executable name to an absolute path, or return nil. + # + # which('ruby') #=> /usr/bin/ruby + def which(filename) + ENV['PATH']&.split(File::PATH_SEPARATOR)&.each do |path| + full_path = File.join(path, filename) + return full_path if File.executable?(full_path) + end + + nil + end + + def try_megabytes_to_bytes(size) + Integer(size).megabytes + rescue ArgumentError + size + end + + def bytes_to_megabytes(bytes) + bytes.to_f / Numeric::MEGABYTE + end + + def ms_to_round_sec(ms) + (ms.to_f / 1000).round(6) + end + + # Used in EE + # Accepts either an Array or a String and returns an array + def ensure_array_from_string(string_or_array) + return string_or_array if string_or_array.is_a?(Array) + + string_or_array.split(',').map(&:strip) + end + + def deep_indifferent_access(data) + case data + when Array + data.map { |item| deep_indifferent_access(item) } + when Hash + data.with_indifferent_access + else + data + end + end + + def deep_symbolized_access(data) + case data + when Array + data.map { |item| deep_symbolized_access(item) } + when Hash + data.deep_symbolize_keys + else + data + end + end + + def string_to_ip_object(str) + return unless str + + IPAddr.new(str) + rescue IPAddr::InvalidAddressError + end + + # A safe alternative to String#downcase! + # + # This will make copies of frozen strings but downcase unfrozen + # strings in place, reducing allocations. + def safe_downcase!(str) + if str.frozen? + str.downcase + else + str.downcase! || str + end + end + + # Converts a string to an Addressable::URI object. + # If the string is not a valid URI, it returns nil. + # Param uri_string should be a String object. + # This method returns an Addressable::URI object or nil. + def parse_url(uri_string) + Addressable::URI.parse(uri_string) + rescue Addressable::URI::InvalidURIError, TypeError + end + + def add_url_parameters(url, params) + uri = parse_url(url.to_s) + uri.query_values = uri.query_values.to_h.merge(params.to_h.stringify_keys) + uri.query_values = nil if uri.query_values.empty? + uri.to_s + end + + def removes_sensitive_data_from_url(uri_string) + uri = parse_url(uri_string) + + return unless uri + return uri_string unless uri.fragment + + stripped_params = CGI.parse(uri.fragment) + if stripped_params['access_token'] + stripped_params['access_token'] = 'filtered' + filtered_query = Addressable::URI.new + filtered_query.query_values = stripped_params + + uri.fragment = filtered_query.query + end + + uri.to_s + end + + # Invert a hash, collecting all keys that map to a given value in an array. + # + # Unlike `Hash#invert`, where the last encountered pair wins, and which has the + # type `Hash[k, v] => Hash[v, k]`, `multiple_key_invert` does not lose any + # information, has the type `Hash[k, v] => Hash[v, Array[k]]`, and the original + # hash can always be reconstructed. + # + # example: + # + # multiple_key_invert({ a: 1, b: 2, c: 1 }) + # # => { 1 => [:a, :c], 2 => [:b] } + # + def multiple_key_invert(hash) + hash.flat_map { |k, v| Array.wrap(v).zip([k].cycle) } + .group_by(&:first) + .transform_values { |kvs| kvs.map(&:last) } + end + + # This sort is stable (see https://en.wikipedia.org/wiki/Sorting_algorithm#Stability) + # contrary to the bare Ruby sort_by method. Using just sort_by leads to + # instability across different platforms (e.g., x86_64-linux and x86_64-darwin18) + # which in turn leads to different sorting results for the equal elements across + # these platforms. + # This method uses a list item's original index position to break ties. + def stable_sort_by(list) + list.sort_by.with_index { |x, idx| [yield(x), idx] } + end + + # Check for valid brackets (`[` and `]`) in a string using this aspects: + # * open brackets count == closed brackets count + # * (optionally) reject nested brackets via `allow_nested: false` + # * open / close brackets coherence, eg. ][[] -> invalid + def valid_brackets?(string = '', allow_nested: true) + # remove everything except brackets + brackets = string.remove(/[^\[\]]/) + + return true if brackets.empty? + # balanced counts check + return false if brackets.size.odd? + + unless allow_nested + # nested brackets check + return false if brackets.include?('[[') || brackets.include?(']]') # rubocop:disable Style/SoleNestedConditional + end + + # open / close brackets coherence check + untrimmed = brackets + loop do + trimmed = untrimmed.gsub('[]', '') + return true if trimmed.empty? + return false if trimmed == untrimmed + + untrimmed = trimmed + end + end + end +end diff --git a/gems/gitlab-utils/lib/gitlab/utils/all.rb b/gems/gitlab-utils/lib/gitlab/utils/all.rb new file mode 100644 index 00000000000..200a21aad88 --- /dev/null +++ b/gems/gitlab-utils/lib/gitlab/utils/all.rb @@ -0,0 +1,6 @@ +# frozen_string_literal: true + +require_relative "../utils" +require_relative "../version_info" +require_relative "version" +require_relative "strong_memoize" diff --git a/gems/gitlab-utils/lib/gitlab/utils/strong_memoize.rb b/gems/gitlab-utils/lib/gitlab/utils/strong_memoize.rb new file mode 100644 index 00000000000..2b3841b8f09 --- /dev/null +++ b/gems/gitlab-utils/lib/gitlab/utils/strong_memoize.rb @@ -0,0 +1,147 @@ +# frozen_string_literal: true + +module Gitlab + module Utils + module StrongMemoize + # Instead of writing patterns like this: + # + # def trigger_from_token + # return @trigger if defined?(@trigger) + # + # @trigger = Ci::Trigger.find_by_token(params[:token].to_s) + # end + # + # We could write it like: + # + # include Gitlab::Utils::StrongMemoize + # + # def trigger_from_token + # Ci::Trigger.find_by_token(params[:token].to_s) + # end + # strong_memoize_attr :trigger_from_token + # + # def enabled? + # Feature.enabled?(:some_feature) + # end + # strong_memoize_attr :enabled? + # + def strong_memoize(name) + key = ivar(name) + + if instance_variable_defined?(key) + instance_variable_get(key) + else + instance_variable_set(key, yield) + end + end + + # Works the same way as "strong_memoize" but takes + # a second argument - expire_in. This allows invalidate + # the data after specified number of seconds + def strong_memoize_with_expiration(name, expire_in) + key = ivar(name) + expiration_key = "#{key}_expired_at" + + if instance_variable_defined?(expiration_key) + expire_at = instance_variable_get(expiration_key) + clear_memoization(name) if Time.current > expire_at + end + + if instance_variable_defined?(key) + instance_variable_get(key) + else + value = instance_variable_set(key, yield) + instance_variable_set(expiration_key, Time.current + expire_in) + value + end + end + + def strong_memoize_with(name, *args) + container = strong_memoize(name) { {} } + + if container.key?(args) + container[args] + else + container[args] = yield + end + end + + def strong_memoized?(name) + key = ivar(StrongMemoize.normalize_key(name)) + instance_variable_defined?(key) + end + + def clear_memoization(name) + key = ivar(StrongMemoize.normalize_key(name)) + remove_instance_variable(key) if instance_variable_defined?(key) + end + + module StrongMemoizeClassMethods + def strong_memoize_attr(method_name) + member_name = StrongMemoize.normalize_key(method_name) + + StrongMemoize.send(:do_strong_memoize, self, method_name, member_name) # rubocop:disable GitlabSecurity/PublicSend + end + end + + def self.included(base) + base.singleton_class.prepend(StrongMemoizeClassMethods) + end + + private + + # Convert `"name"`/`:name` into `:@name` + # + # Depending on a type ensure that there's a single memory allocation + def ivar(name) + case name + when Symbol + name.to_s.prepend("@").to_sym + when String + :"@#{name}" + else + raise ArgumentError, "Invalid type of '#{name}'" + end + end + + class << self + def normalize_key(key) + return key unless key.end_with?('!', '?') + + # Replace invalid chars like `!` and `?` with allowed Unicode codeparts. + key.to_s.tr('!?', "\uFF01\uFF1F") + end + + private + + def do_strong_memoize(klass, method_name, member_name) + method = klass.instance_method(method_name) + + unless method.arity == 0 + raise <<~ERROR + Using `strong_memoize_attr` on methods with parameters is not supported. + + Use `strong_memoize_with` instead. + See https://docs.gitlab.com/ee/development/utilities.html#strongmemoize + ERROR + end + + # Methods defined within a class method are already public by default, so we don't need to + # explicitly make them public. + scope = %i[private protected].find do |scope| + klass.send("#{scope}_instance_methods") # rubocop:disable GitlabSecurity/PublicSend + .include? method_name + end + + klass.define_method(method_name) do |&block| + strong_memoize(member_name) do + method.bind_call(self, &block) + end + end + + klass.send(scope, method_name) if scope # rubocop:disable GitlabSecurity/PublicSend + end + end + end + end +end diff --git a/gems/gitlab-utils/lib/gitlab/utils/version.rb b/gems/gitlab-utils/lib/gitlab/utils/version.rb new file mode 100644 index 00000000000..a9afe5bf845 --- /dev/null +++ b/gems/gitlab-utils/lib/gitlab/utils/version.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +module Gitlab + module Utils + module Version + VERSION = "0.1.0" + end + end +end diff --git a/gems/gitlab-utils/lib/gitlab/version_info.rb b/gems/gitlab-utils/lib/gitlab/version_info.rb new file mode 100644 index 00000000000..00a9b4ddc6e --- /dev/null +++ b/gems/gitlab-utils/lib/gitlab/version_info.rb @@ -0,0 +1,101 @@ +# frozen_string_literal: true + +module Gitlab + class VersionInfo + include Comparable + + attr_reader :major, :minor, :patch + + VERSION_REGEX = /(\d+)\.(\d+)\.(\d+)/ + # To mitigate ReDoS, limit the length of the version string we're + # willing to check + MAX_VERSION_LENGTH = 128 + + def self.parse(str, parse_suffix: false) + return str if str.is_a?(self) + + if str && str.length <= MAX_VERSION_LENGTH + match = str.match(VERSION_REGEX) + if match + return VersionInfo.new(match[1].to_i, match[2].to_i, match[3].to_i, parse_suffix ? match.post_match : nil) + end + end + + VersionInfo.new + end + + def initialize(major = 0, minor = 0, patch = 0, suffix = nil) # rubocop:disable Metrics/ParameterLists + @major = major + @minor = minor + @patch = patch + @suffix_s = suffix.to_s + end + + # rubocop:disable Metrics/CyclomaticComplexity + # rubocop:disable Metrics/PerceivedComplexity + def <=>(other) + return unless other.is_a? VersionInfo + return unless valid? && other.valid? + + if other.major < @major + 1 + elsif @major < other.major + -1 + elsif other.minor < @minor + 1 + elsif @minor < other.minor + -1 + elsif other.patch < @patch + 1 + elsif @patch < other.patch + -1 + elsif @suffix_s.empty? && other.suffix.present? + 1 + elsif other.suffix.empty? && @suffix_s.present? + -1 + else + suffix <=> other.suffix + end + end + # rubocop:enable Metrics/CyclomaticComplexity + # rubocop:enable Metrics/PerceivedComplexity + + def to_s + if valid? + "%d.%d.%d%s" % [@major, @minor, @patch, @suffix_s] # rubocop:disable Style/FormatString + else + 'Unknown' + end + end + + def to_json(*_args) + { major: @major, minor: @minor, patch: @patch }.to_json + end + + def suffix + @suffix ||= @suffix_s.strip.gsub('-', '.pre.').scan(/\d+|[a-z]+/i).map do |s| + /^\d+$/.match?(s) ? s.to_i : s + end.freeze + end + + def valid? + @major >= 0 && @minor >= 0 && @patch >= 0 && @major + @minor + @patch > 0 + end + + def hash + [self.class, to_s].hash + end + + def eql?(other) + (self <=> other) == 0 + end + + def same_minor_version?(other) + @major == other.major && @minor == other.minor + end + + def without_patch + self.class.new(@major, @minor, 0) + end + end +end diff --git a/gems/gitlab-utils/spec/gitlab/utils/strong_memoize_spec.rb b/gems/gitlab-utils/spec/gitlab/utils/strong_memoize_spec.rb new file mode 100644 index 00000000000..f23a12ca6a2 --- /dev/null +++ b/gems/gitlab-utils/spec/gitlab/utils/strong_memoize_spec.rb @@ -0,0 +1,368 @@ +# frozen_string_literal: true + +require 'spec_helper' +require 'active_support/testing/time_helpers' + +RSpec.describe Gitlab::Utils::StrongMemoize, feature_category: :shared do + include ActiveSupport::Testing::TimeHelpers + + let(:klass) do + strong_memoize_class = described_class + + Struct.new(:value) do + include strong_memoize_class + + def self.method_added_list + @method_added_list ||= [] + end + + def self.method_added(name) + method_added_list << name + end + + def method_name + strong_memoize(:method_name) do # rubocop:disable Gitlab/StrongMemoizeAttr + trace << value + value + end + end + + def method_name_with_expiration + strong_memoize_with_expiration(:method_name_with_expiration, 1) do + trace << value + value + end + end + + def method_name_attr + trace << value + value + end + strong_memoize_attr :method_name_attr + + def enabled? + trace << value + value + end + strong_memoize_attr :enabled? + + def method_name_with_args(*args) + strong_memoize_with(:method_name_with_args, args) do + trace << [value, args] + value + end + end + + def trace + @trace ||= [] + end + + protected + + def private_method; end + private :private_method + strong_memoize_attr :private_method + + public + + def protected_method; end + protected :protected_method + strong_memoize_attr :protected_method + + private + + def public_method; end + public :public_method + strong_memoize_attr :public_method + end + end + + subject(:object) { klass.new(value) } + + shared_examples 'caching the value' do + let(:member_name) { described_class.normalize_key(method_name) } + + it 'only calls the block once' do + value0 = object.public_send(method_name) + value1 = object.public_send(method_name) + + expect(value0).to eq(value) + expect(value1).to eq(value) + expect(object.trace).to contain_exactly(value) + end + + it 'returns and defines the instance variable for the exact value' do + returned_value = object.public_send(method_name) + memoized_value = object.instance_variable_get(:"@#{member_name}") + + expect(returned_value).to eql(value) + expect(memoized_value).to eql(value) + end + end + + describe '#strong_memoize' do + [nil, false, true, 'value', 0, [0]].each do |value| + context "with value #{value}" do + let(:value) { value } + let(:method_name) { :method_name } + + it_behaves_like 'caching the value' + + it 'raises exception for invalid type as key' do + expect { object.strong_memoize(10) { 20 } }.to raise_error(/Invalid type of '10'/) + end + + it 'raises exception for invalid characters in key' do + expect { object.strong_memoize(:enabled?) { 20 } } + .to raise_error(/is not allowed as an instance variable name/) + end + end + end + + context "with memory allocation", type: :benchmark do + let(:value) { 'aaa' } + + before do + object.method_name # warmup + end + + [:method_name, "method_name"].each do |argument| + context "when argument is a #{argument.class}" do + it 'does allocate exactly one string when fetching value' do + expect do + object.strong_memoize(argument) { 10 } + end.to perform_allocation(1) + end + + it 'does allocate exactly one string when storing value' do + object.clear_memoization(:method_name) # clear to force set + + expect do + object.strong_memoize(argument) { 10 } + end.to perform_allocation(1) + end + end + end + end + end + + describe '#strong_memoize_with_expiration' do + [nil, false, true, 'value', 0, [0]].each do |value| + context "with value #{value}" do + let(:value) { value } + let(:method_name) { :method_name_with_expiration } + + it_behaves_like 'caching the value' + + it 'raises exception for invalid type as key' do + expect { object.strong_memoize_with_expiration(10, 1) { 20 } }.to raise_error(/Invalid type of '10'/) + end + + it 'raises exception for invalid characters in key' do + expect { object.strong_memoize_with_expiration(:enabled?, 1) { 20 } } + .to raise_error(/is not allowed as an instance variable name/) + end + end + end + + context 'with value memoization test' do + let(:value) { 'value' } + + it 'caches the value for specified number of seconds' do + object.method_name_with_expiration + object.method_name_with_expiration + + expect(object.trace.count).to eq(1) + + travel_to(Time.current + 2.seconds) do + object.method_name_with_expiration + + expect(object.trace.count).to eq(2) + end + end + end + end + + describe '#strong_memoize_with' do + [nil, false, true, 'value', 0, [0]].each do |value| + context "with value #{value}" do + let(:value) { value } + + it 'only calls the block once' do + value0 = object.method_name_with_args(1) + value1 = object.method_name_with_args(1) + value2 = object.method_name_with_args([2, 3]) + value3 = object.method_name_with_args([2, 3]) + + expect(value0).to eq(value) + expect(value1).to eq(value) + expect(value2).to eq(value) + expect(value3).to eq(value) + + expect(object.trace).to contain_exactly([value, [1]], [value, [[2, 3]]]) + end + + it 'returns and defines the instance variable for the exact value' do + returned_value = object.method_name_with_args(1, 2, 3) + memoized_value = object.instance_variable_get(:@method_name_with_args) + + expect(returned_value).to eql(value) + expect(memoized_value).to eql({ [[1, 2, 3]] => value }) + end + end + end + end + + describe '#strong_memoized?' do + shared_examples 'memoization check' do |method_name| + context "when method is :#{method_name}" do + let(:value) { :anything } + + subject { object.strong_memoized?(method_name) } + + it 'returns false if the value is uncached' do + expect(subject).to be(false) + end + + it 'returns true if the value is cached' do + object.public_send(method_name) + + expect(subject).to be(true) + end + end + end + + it_behaves_like 'memoization check', :method_name + it_behaves_like 'memoization check', :enabled? + end + + describe '#clear_memoization' do + shared_examples 'clearing memoization' do |method_name| + let(:member_name) { described_class.normalize_key(method_name) } + let(:value) { 'mepmep' } + + it 'removes the instance variable' do + object.public_send(method_name) + + object.clear_memoization(method_name) + + expect(object.instance_variable_defined?(:"@#{member_name}")).to be(false) + end + end + + it_behaves_like 'clearing memoization', :method_name + it_behaves_like 'clearing memoization', :enabled? + end + + describe '.strong_memoize_attr' do + [nil, false, true, 'value', 0, [0]].each do |value| + context "with value '#{value}'" do + let(:value) { value } + + context 'with memoized after method definition' do + let(:method_name) { :method_name_attr } + + it_behaves_like 'caching the value' + + it 'calls the existing .method_added' do + expect(klass.method_added_list).to include(:method_name_attr) + end + + it 'retains method arity' do + expect(klass.instance_method(method_name).arity).to eq(0) + end + end + end + end + + describe 'method visibility' do + it 'sets private visibility' do + expect(klass.private_instance_methods).to include(:private_method) + expect(klass.protected_instance_methods).not_to include(:private_method) + expect(klass.public_instance_methods).not_to include(:private_method) + end + + it 'sets protected visibility' do + expect(klass.private_instance_methods).not_to include(:protected_method) + expect(klass.protected_instance_methods).to include(:protected_method) + expect(klass.public_instance_methods).not_to include(:protected_method) + end + + it 'sets public visibility' do + expect(klass.private_instance_methods).not_to include(:public_method) + expect(klass.protected_instance_methods).not_to include(:public_method) + expect(klass.public_instance_methods).to include(:public_method) + end + end + + context "when method doesn't exist" do + let(:klass) do + strong_memoize_class = described_class + + Struct.new(:value) do + include strong_memoize_class + end + end + + subject { klass.strong_memoize_attr(:nonexistent_method) } + + it 'fails when strong-memoizing a nonexistent method' do + expect { subject }.to raise_error(NameError, /undefined method `nonexistent_method' for class/) + end + end + + context 'when memoized method has parameters' do + it 'raises an error' do + expected_message = /Using `strong_memoize_attr` on methods with parameters is not supported/ + + expect do + strong_memoize_class = described_class + + Class.new do + include strong_memoize_class + + def method_with_parameters(params); end + strong_memoize_attr :method_with_parameters + end + end.to raise_error(RuntimeError, expected_message) + end + end + end + + describe '.normalize_key' do + using RSpec::Parameterized::TableSyntax + + subject { described_class.normalize_key(input) } + + where(:input, :output, :valid) do + :key | :key | true + "key" | "key" | true + :key? | "key？" | true + "key?" | "key？" | true + :key! | "key！" | true + "key!" | "key！" | true + # invalid cases caught elsewhere + :"ke?y" | :"ke?y" | false + "ke?y" | "ke?y" | false + :"ke!y" | :"ke!y" | false + "ke!y" | "ke!y" | false + end + + with_them do + let(:ivar) { "@#{output}" } + + it { is_expected.to eq(output) } + + if params[:valid] + it 'is a valid ivar name' do + expect { instance_variable_defined?(ivar) }.not_to raise_error + end + else + it 'raises a NameError error' do + expect { instance_variable_defined?(ivar) } + .to raise_error(NameError, /not allowed as an instance/) + end + end + end + end +end diff --git a/gems/gitlab-utils/spec/gitlab/utils_spec.rb b/gems/gitlab-utils/spec/gitlab/utils_spec.rb new file mode 100644 index 00000000000..53593190eea --- /dev/null +++ b/gems/gitlab-utils/spec/gitlab/utils_spec.rb @@ -0,0 +1,479 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Utils, feature_category: :shared do + using RSpec::Parameterized::TableSyntax + include StubENV + + delegate :to_boolean, :boolean_to_yes_no, :slugify, :which, + :ensure_array_from_string, :bytes_to_megabytes, + :append_path, :remove_leading_slashes, :allowlisted?, + :decode_path, :ms_to_round_sec, to: :described_class + + describe '.allowlisted?' do + let(:allowed_paths) { ['/home/foo', '/foo/bar', '/etc/passwd'] } + + it 'returns true if path is allowed' do + expect(allowlisted?('/foo/bar', allowed_paths)).to be(true) + end + + it 'returns false if path is not allowed' do + expect(allowlisted?('/test/test', allowed_paths)).to be(false) + end + end + + describe '.decode_path' do + it 'returns path unencoded for singled-encoded paths' do + expect(decode_path('%2Fhome%2Fbar%3Fasd%3Dqwe')).to eq('/home/bar?asd=qwe') + end + + it 'returns path when it is unencoded' do + expect(decode_path('/home/bar?asd=qwe')).to eq('/home/bar?asd=qwe') + end + + [ + '..%252F..%252F..%252Fetc%252Fpasswd', + '%25252Fresult%25252Fchosennickname%25253D%252522jj%252522' + ].each do |multiple_encoded_path| + it 'raises an exception when the path is multiple-encoded' do + expect { decode_path(multiple_encoded_path) }.to raise_error(/path #{multiple_encoded_path} is not allowed/) + end + end + end + + describe '.slugify' do + { + 'TEST' => 'test', + 'project_with_underscores' => 'project-with-underscores', + 'namespace/project' => 'namespace-project', + 'a' * 70 => 'a' * 63, + 'test_trailing_' => 'test-trailing' + }.each do |original, expected| + it "slugifies #{original} to #{expected}" do + expect(slugify(original)).to eq(expected) + end + end + end + + describe '.ms_to_round_sec' do + where(:original, :expected) do + 1999.8999 | 1.9999 + 12384 | 12.384 + 333 | 0.333 + 1333.33333333 | 1.333333 + end + + with_them do + it "returns rounded seconds" do + expect(ms_to_round_sec(original)).to eq(expected) + end + end + end + + describe '.nlbr' do + it 'replaces new lines with <br>' do + expect(described_class.nlbr("<b>hello</b>\n<i>world</i>")).to eq("hello<br>world") + end + end + + describe '.remove_line_breaks' do + where(:original, :expected) do + "foo\nbar\nbaz" | "foobarbaz" + "foo\r\nbar\r\nbaz" | "foobarbaz" + "foobar" | "foobar" + end + + with_them do + it "replace line breaks with an empty string" do + expect(described_class.remove_line_breaks(original)).to eq(expected) + end + end + end + + describe '.to_boolean' do + it 'accepts booleans' do + expect(to_boolean(true)).to be(true) + expect(to_boolean(false)).to be(false) + end + + it 'converts a valid value to a boolean' do + expect(to_boolean(true)).to be(true) + expect(to_boolean('true')).to be(true) + expect(to_boolean('YeS')).to be(true) + expect(to_boolean('t')).to be(true) + expect(to_boolean('1')).to be(true) + expect(to_boolean(1)).to be(true) + expect(to_boolean('ON')).to be(true) + + expect(to_boolean('FaLse')).to be(false) + expect(to_boolean('F')).to be(false) + expect(to_boolean('NO')).to be(false) + expect(to_boolean('n')).to be(false) + expect(to_boolean('0')).to be(false) + expect(to_boolean(0)).to be(false) + expect(to_boolean('oFF')).to be(false) + end + + it 'converts an invalid value to nil' do + expect(to_boolean('fals')).to be_nil + expect(to_boolean('yeah')).to be_nil + expect(to_boolean('')).to be_nil + expect(to_boolean(nil)).to be_nil + end + + it 'accepts a default value, and does not return it when a valid value is given' do + expect(to_boolean(true, default: false)).to be(true) + expect(to_boolean('true', default: false)).to be(true) + expect(to_boolean('YeS', default: false)).to be(true) + expect(to_boolean('t', default: false)).to be(true) + expect(to_boolean('1', default: 'any value')).to be(true) + expect(to_boolean('ON', default: 42)).to be(true) + + expect(to_boolean('FaLse', default: true)).to be(false) + expect(to_boolean('F', default: true)).to be(false) + expect(to_boolean('NO', default: true)).to be(false) + expect(to_boolean('n', default: true)).to be(false) + expect(to_boolean('0', default: 'any value')).to be(false) + expect(to_boolean('oFF', default: 42)).to be(false) + end + + it 'accepts a default value, and returns it when an invalid value is given' do + expect(to_boolean('fals', default: true)).to eq(true) + expect(to_boolean('yeah', default: false)).to eq(false) + expect(to_boolean('', default: 'any value')).to eq('any value') + expect(to_boolean(nil, default: 42)).to eq(42) + end + end + + describe '.boolean_to_yes_no' do + it 'converts booleans to Yes or No' do + expect(boolean_to_yes_no(true)).to eq('Yes') + expect(boolean_to_yes_no(false)).to eq('No') + end + end + + describe '.which' do + before do + stub_env('PATH', '/sbin:/usr/bin:/home/joe/bin') + end + + it 'finds the full path to an executable binary in order of appearance' do + expect(File).to receive(:executable?).with('/sbin/tool').ordered.and_return(false) + expect(File).to receive(:executable?).with('/usr/bin/tool').ordered.and_return(true) + expect(File).not_to receive(:executable?).with('/home/joe/bin/tool') + + expect(which('tool')).to eq('/usr/bin/tool') + end + end + + describe '.ensure_array_from_string' do + it 'returns the same array if given one' do + arr = ['a', 4, true, { test: 1 }] + + expect(ensure_array_from_string(arr)).to eq(arr) + end + + it 'turns comma-separated strings into arrays' do + str = 'seven, eight, 9, 10' + + expect(ensure_array_from_string(str)).to eq(%w[seven eight 9 10]) + end + end + + describe '.bytes_to_megabytes' do + it 'converts bytes to megabytes' do + bytes = 1.megabyte + + expect(bytes_to_megabytes(bytes)).to eq(1) + end + end + + describe '.append_path' do + where(:host, :path, :result) do + 'http://test/' | '/foo/bar' | 'http://test/foo/bar' + 'http://test/' | '//foo/bar' | 'http://test/foo/bar' + 'http://test//' | '/foo/bar' | 'http://test/foo/bar' + 'http://test' | 'foo/bar' | 'http://test/foo/bar' + 'http://test//' | '' | 'http://test/' + 'http://test//' | nil | 'http://test/' + '' | '/foo/bar' | '/foo/bar' + nil | '/foo/bar' | '/foo/bar' + end + + with_them do + it 'makes sure there is only one slash as path separator' do + expect(append_path(host, path)).to eq(result) + end + end + end + + describe '.remove_leading_slashes' do + where(:str, :result) do + '/foo/bar' | 'foo/bar' + '//foo/bar' | 'foo/bar' + '/foo/bar/' | 'foo/bar/' + 'foo/bar' | 'foo/bar' + '' | '' + nil | '' + end + + with_them do + it 'removes leading slashes' do + expect(remove_leading_slashes(str)).to eq(result) + end + end + end + + describe '.ensure_utf8_size' do + context 'with string is has less bytes than expected' do + it 'backfills string with null characters' do + transformed = described_class.ensure_utf8_size('a' * 10, bytes: 32) + + expect(transformed.bytesize).to eq 32 + expect(transformed).to eq(('a' * 10) + ('0' * 22)) + end + end + + context 'with string size is exactly the one that is expected' do + it 'returns original value' do + transformed = described_class.ensure_utf8_size('a' * 32, bytes: 32) + + expect(transformed).to eq 'a' * 32 + expect(transformed.bytesize).to eq 32 + end + end + + context 'when string contains a few multi-byte UTF characters' do + it 'backfills string with null characters' do + transformed = described_class.ensure_utf8_size('❤' * 6, bytes: 32) + + expect(transformed).to eq '❤❤❤❤❤❤' + ('0' * 14) # rubocop:disable Style/StringConcatenation + expect(transformed.bytesize).to eq 32 + end + end + + context 'when string has multiple multi-byte UTF chars exceeding 32 bytes' do + it 'truncates string to 32 characters and backfills it if needed' do + transformed = described_class.ensure_utf8_size('❤' * 18, bytes: 32) + + expect(transformed).to eq(('❤' * 10) + ('0' * 2)) + expect(transformed.bytesize).to eq 32 + end + end + end + + describe '.deep_indifferent_access' do + let(:hash) do + { "variables" => [{ "key" => "VAR1", "value" => "VALUE2" }] } + end + + subject { described_class.deep_indifferent_access(hash) } + + it 'allows to access hash keys with symbols' do + expect(subject[:variables]).to be_a(Array) + end + + it 'allows to access array keys with symbols' do + expect(subject[:variables].first[:key]).to eq('VAR1') + end + end + + describe '.deep_symbolized_access' do + let(:hash) do + { "variables" => [{ "key" => "VAR1", "value" => "VALUE2" }] } + end + + subject { described_class.deep_symbolized_access(hash) } + + it 'allows to access hash keys with symbols' do + expect(subject[:variables]).to be_a(Array) + end + + it 'allows to access array keys with symbols' do + expect(subject[:variables].first[:key]).to eq('VAR1') + end + end + + describe '.try_megabytes_to_bytes' do + context 'when the size can be converted to megabytes' do + it 'returns the size in megabytes' do + size = described_class.try_megabytes_to_bytes(1) + + expect(size).to eq(1.megabytes) + end + end + + context 'when the size can not be converted to megabytes' do + it 'returns the input size' do + size = described_class.try_megabytes_to_bytes('foo') + + expect(size).to eq('foo') + end + end + end + + describe '.string_to_ip_object' do + it 'returns nil when string is nil' do + expect(described_class.string_to_ip_object(nil)).to eq(nil) + end + + it 'returns nil when string is invalid IP' do + expect(described_class.string_to_ip_object('invalid ip')).to eq(nil) + expect(described_class.string_to_ip_object('')).to eq(nil) + end + + it 'returns IP object when string is valid IP' do + expect(described_class.string_to_ip_object('192.168.1.1')).to eq(IPAddr.new('192.168.1.1')) + expect(described_class.string_to_ip_object('::ffff:a9fe:a864')).to eq(IPAddr.new('::ffff:a9fe:a864')) + expect(described_class.string_to_ip_object('[::ffff:a9fe:a864]')).to eq(IPAddr.new('::ffff:a9fe:a864')) + expect(described_class.string_to_ip_object('127.0.0.0/28')).to eq(IPAddr.new('127.0.0.0/28')) + expect(described_class.string_to_ip_object('1:0:0:0:0:0:0:0/124')).to eq(IPAddr.new('1:0:0:0:0:0:0:0/124')) + end + end + + describe ".safe_downcase!" do + where(:str, :result) do + "test" | "test" + "Test" | "test" + "test" | "test" + "Test" | "test" + end + + with_them do + it "downcases the string" do + expect(described_class.safe_downcase!(str)).to eq(result) + end + end + end + + describe '.parse_url' do + it 'returns Addressable::URI object' do + expect(described_class.parse_url('http://gitlab.com')).to be_instance_of(Addressable::URI) + end + + it 'returns nil when URI cannot be parsed' do + expect(described_class.parse_url('://gitlab.com')).to be nil + end + + it 'returns nil with invalid parameter' do + expect(described_class.parse_url(1)).to be nil + end + end + + describe '.add_url_parameters' do + subject { described_class.add_url_parameters(url, params) } + + where(:url, :params, :expected_url) do + nil | nil | '' + nil | { b: 3, a: 2 } | '?a=2&b=3' + 'https://gitlab.com' | nil | 'https://gitlab.com' + 'https://gitlab.com' | { b: 3, a: 2 } | 'https://gitlab.com?a=2&b=3' + 'https://gitlab.com?a=1#foo' | { b: 3, 'a' => 2 } | 'https://gitlab.com?a=2&b=3#foo' + 'https://gitlab.com?a=1#foo' | [[:b, 3], [:a, 2]] | 'https://gitlab.com?a=2&b=3#foo' + end + + with_them do + it { is_expected.to eq(expected_url) } + end + end + + describe '.removes_sensitive_data_from_url' do + it 'returns string object' do + expect(described_class.removes_sensitive_data_from_url('http://gitlab.com')).to be_instance_of(String) + end + + it 'returns nil when URI cannot be parsed' do + expect(described_class.removes_sensitive_data_from_url('://gitlab.com')).to be nil + end + + it 'returns nil with invalid parameter' do + expect(described_class.removes_sensitive_data_from_url(1)).to be nil + end + + it 'returns string with filtered access_token param' do + expect(described_class.removes_sensitive_data_from_url('http://gitlab.com/auth.html#access_token=secret_token')) + .to eq('http://gitlab.com/auth.html#access_token=filtered') + end + + it 'returns string with filtered access_token param but other params preserved' do + expect(described_class.removes_sensitive_data_from_url('http://gitlab.com/auth.html#access_token=secret_token&token_type=Bearer&state=test')) + .to include('&token_type=Bearer', '&state=test') + end + end + + describe 'multiple_key_invert' do + it 'invert keys with array values' do + hash = { + dast: [:vulnerabilities_count, :scanned_resources_count], + sast: [:vulnerabilities_count] + } + expect(described_class.multiple_key_invert(hash)).to eq({ + vulnerabilities_count: [:dast, :sast], + scanned_resources_count: [:dast] + }) + end + end + + describe '.stable_sort_by' do + subject(:sorted_list) { described_class.stable_sort_by(list) { |obj| obj[:priority] } } + + context 'when items have the same priority' do + let(:list) do + [ + { name: 'obj 1', priority: 1 }, + { name: 'obj 2', priority: 1 }, + { name: 'obj 3', priority: 1 } + ] + end + + it 'does not change order in cases of ties' do + expect(sorted_list).to eq(list) + end + end + + context 'when items have different priorities' do + let(:list) do + [ + { name: 'obj 1', priority: 2 }, + { name: 'obj 2', priority: 1 }, + { name: 'obj 3', priority: 3 } + ] + end + + it 'sorts items like the regular sort_by' do + expect(sorted_list).to eq( + [ + { name: 'obj 2', priority: 1 }, + { name: 'obj 1', priority: 2 }, + { name: 'obj 3', priority: 3 } + ]) + end + end + end + + describe '.valid_brackets?' do + where(:input, :allow_nested, :valid) do + 'no brackets' | true | true + 'no brackets' | false | true + 'user[avatar]' | true | true + 'user[avatar]' | false | true + 'user[avatar][friends]' | true | true + 'user[avatar][friends]' | false | true + 'user[avatar[image[url]]]' | true | true + 'user[avatar[image[url]]]' | false | false + 'user[avatar[]friends]' | true | true + 'user[avatar[]friends]' | false | false + 'user[avatar]]' | true | false + 'user[avatar]]' | false | false + 'user][avatar]]' | true | false + 'user][avatar]]' | false | false + 'user[avatar' | true | false + 'user[avatar' | false | false + end + + with_them do + it { expect(described_class.valid_brackets?(input, allow_nested: allow_nested)).to eq(valid) } + end + end +end diff --git a/gems/gitlab-utils/spec/gitlab/version_info_spec.rb b/gems/gitlab-utils/spec/gitlab/version_info_spec.rb new file mode 100644 index 00000000000..2b5f6bcb4c1 --- /dev/null +++ b/gems/gitlab-utils/spec/gitlab/version_info_spec.rb @@ -0,0 +1,193 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::VersionInfo, feature_category: :shared do + before do + @unknown = described_class.new + @v0_0_1 = described_class.new(0, 0, 1) + @v0_1_0 = described_class.new(0, 1, 0) + @v1_0_0 = described_class.new(1, 0, 0) + @v1_0_1 = described_class.new(1, 0, 1) + @v1_0_1_b1 = described_class.new(1, 0, 1, '-b1') + @v1_0_1_rc1 = described_class.new(1, 0, 1, '-rc1') + @v1_0_1_rc2 = described_class.new(1, 0, 1, '-rc2') + @v1_1_0 = described_class.new(1, 1, 0) + @v1_1_0_beta1 = described_class.new(1, 1, 0, '-beta1') + @v2_0_0 = described_class.new(2, 0, 0) + @v13_10_1_1574_89 = described_class.parse("v13.10.1~beta.1574.gf6ea9389", parse_suffix: true) + @v13_10_1_1575_89 = described_class.parse("v13.10.1~beta.1575.gf6ea9389", parse_suffix: true) + @v13_10_1_1575_90 = described_class.parse("v13.10.1~beta.1575.gf6ea9390", parse_suffix: true) + end + + describe '>' do + it { expect(@v2_0_0).to be > @v1_1_0 } + it { expect(@v1_1_0).to be > @v1_0_1 } + it { expect(@v1_0_1_b1).to be > @v1_0_0 } + it { expect(@v1_0_1_rc1).to be > @v1_0_0 } + it { expect(@v1_0_1_rc1).to be > @v1_0_1_b1 } + it { expect(@v1_0_1_rc2).to be > @v1_0_1_rc1 } + it { expect(@v1_0_1).to be > @v1_0_1_rc1 } + it { expect(@v1_0_1).to be > @v1_0_1_rc2 } + it { expect(@v1_0_1).to be > @v1_0_0 } + it { expect(@v1_0_0).to be > @v0_1_0 } + it { expect(@v1_1_0_beta1).to be > @v1_0_1_rc2 } + it { expect(@v1_1_0).to be > @v1_1_0_beta1 } + it { expect(@v0_1_0).to be > @v0_0_1 } + end + + describe '>=' do + it { expect(@v2_0_0).to be >= described_class.new(2, 0, 0) } + it { expect(@v2_0_0).to be >= @v1_1_0 } + it { expect(@v1_0_1_rc2).to be >= @v1_0_1_rc1 } + end + + describe '<' do + it { expect(@v0_0_1).to be < @v0_1_0 } + it { expect(@v0_1_0).to be < @v1_0_0 } + it { expect(@v1_0_0).to be < @v1_0_1 } + it { expect(@v1_0_1).to be < @v1_1_0 } + it { expect(@v1_0_0).to be < @v1_0_1_rc2 } + it { expect(@v1_0_1_rc1).to be < @v1_0_1 } + it { expect(@v1_0_1_rc1).to be < @v1_0_1_rc2 } + it { expect(@v1_0_1_rc2).to be < @v1_0_1 } + it { expect(@v1_1_0).to be < @v2_0_0 } + it { expect(@v13_10_1_1574_89).to be < @v13_10_1_1575_89 } + it { expect(@v13_10_1_1575_89).to be < @v13_10_1_1575_90 } + end + + describe '<=' do + it { expect(@v0_0_1).to be <= described_class.new(0, 0, 1) } + it { expect(@v0_0_1).to be <= @v0_1_0 } + it { expect(@v1_0_1_b1).to be <= @v1_0_1_rc1 } + it { expect(@v1_0_1_rc1).to be <= @v1_0_1_rc2 } + it { expect(@v1_1_0_beta1).to be <= @v1_1_0 } + end + + describe '==' do + it { expect(@v0_0_1).to eq(described_class.new(0, 0, 1)) } + it { expect(@v0_1_0).to eq(described_class.new(0, 1, 0)) } + it { expect(@v1_0_0).to eq(described_class.new(1, 0, 0)) } + it { expect(@v1_0_1_rc1).to eq(described_class.new(1, 0, 1, '-rc1')) } + end + + describe '!=' do + it { expect(@v0_0_1).not_to eq(@v0_1_0) } + it { expect(@v1_0_1_rc1).not_to eq(@v1_0_1_rc2) } + end + + describe '.unknown' do + it { expect(@unknown).not_to be @v0_0_1 } + it { expect(@unknown).not_to be described_class.new } + it { expect { @unknown > @v0_0_1 }.to raise_error(ArgumentError) } + it { expect { @unknown < @v0_0_1 }.to raise_error(ArgumentError) } + end + + describe '.parse' do + it { expect(described_class.parse(described_class.new(1, 0, 0))).to eq(@v1_0_0) } + it { expect(described_class.parse("1.0.0")).to eq(@v1_0_0) } + it { expect(described_class.parse("1.0.0.1")).to eq(@v1_0_0) } + it { expect(described_class.parse("1.0.0-ee")).to eq(@v1_0_0) } + it { expect(described_class.parse("1.0.0-rc1")).to eq(@v1_0_0) } + it { expect(described_class.parse("1.0.0-rc1-ee")).to eq(@v1_0_0) } + it { expect(described_class.parse("git 1.0.0b1")).to eq(@v1_0_0) } + it { expect(described_class.parse("git 1.0b1")).not_to be_valid } + it { expect(described_class.parse("1.1.#{'1' * described_class::MAX_VERSION_LENGTH}")).not_to be_valid } + it { expect(described_class.parse(nil)).not_to be_valid } + + context 'with parse_suffix: true' do + let(:versions) do + <<-VERSIONS.lines + 0.0.1 + 0.1.0 + 1.0.0 + 1.0.1-b1 + 1.0.1-rc1 + 1.0.1-rc2 + 1.0.1 + 1.1.0-beta1 + 1.1.0 + 2.0.0 + v13.10.0-pre + v13.10.0-rc1 + v13.10.0-rc2 + v13.10.0 + v13.10.1~beta.1574.gf6ea9389 + v13.10.1~beta.1575.gf6ea9389 + v13.10.1-rc1 + v13.10.1-rc2 + v13.10.1 + VERSIONS + end + + let(:parsed_versions) do + versions.map(&:strip).map { |version| described_class.parse(version, parse_suffix: true) } + end + + it 'versions are returned in a correct order' do + expect(parsed_versions.shuffle.sort).to eq(parsed_versions) + end + end + end + + describe '.to_s' do + it { expect(@v1_0_0.to_s).to eq("1.0.0") } + it { expect(@v1_0_1_rc1.to_s).to eq("1.0.1-rc1") } + it { expect(@unknown.to_s).to eq("Unknown") } + end + + describe '.to_json' do + let(:correct_version) do + "{\"major\":1,\"minor\":0,\"patch\":1}" + end + + let(:unknown_version) do + "{\"major\":0,\"minor\":0,\"patch\":0}" + end + + it { expect(@v1_0_1.to_json).to eq(correct_version) } + it { expect(@v1_0_1_rc2.to_json).to eq(correct_version) } + it { expect(@unknown.to_json).to eq(unknown_version) } + end + + describe '.hash' do + it { expect(described_class.parse("1.0.0").hash).to eq(@v1_0_0.hash) } + it { expect(described_class.parse("1.0.0.1").hash).to eq(@v1_0_0.hash) } + it { expect(described_class.parse("1.0.1b1").hash).to eq(@v1_0_1.hash) } + it { expect(described_class.parse("1.0.1-rc1", parse_suffix: true).hash).to eq(@v1_0_1_rc1.hash) } + end + + describe '.eql?' do + it { expect(described_class.parse("1.0.0").eql?(@v1_0_0)).to be_truthy } + it { expect(described_class.parse("1.0.0.1").eql?(@v1_0_0)).to be_truthy } + it { expect(@v1_0_1_rc1.eql?(@v1_0_1_rc1)).to be_truthy } + it { expect(@v1_0_1_rc1.eql?(@v1_0_1_rc2)).to be_falsey } + it { expect(@v1_0_1_rc1.eql?(@v1_0_1)).to be_falsey } + it { expect(@v1_0_1.eql?(@v1_0_0)).to be_falsey } + it { expect(@v1_1_0.eql?(@v1_0_0)).to be_falsey } + it { expect(@v1_0_0.eql?(@v1_0_0)).to be_truthy } + it { expect([@v1_0_0, @v1_1_0, @v1_0_0, @v1_0_1_rc1, @v1_0_1_rc1].uniq).to eq [@v1_0_0, @v1_1_0, @v1_0_1_rc1] } + end + + describe '.same_minor_version?' do + it { expect(@v0_1_0.same_minor_version?(@v0_0_1)).to be_falsey } + it { expect(@v1_0_1.same_minor_version?(@v1_0_0)).to be_truthy } + it { expect(@v1_0_1_rc1.same_minor_version?(@v1_0_0)).to be_truthy } + it { expect(@v1_0_0.same_minor_version?(@v1_0_1)).to be_truthy } + it { expect(@v1_1_0.same_minor_version?(@v1_0_0)).to be_falsey } + it { expect(@v2_0_0.same_minor_version?(@v1_0_0)).to be_falsey } + end + + describe '.without_patch' do + it { expect(@v0_1_0.without_patch).to eq(@v0_1_0) } + it { expect(@v1_0_0.without_patch).to eq(@v1_0_0) } + it { expect(@v1_0_1.without_patch).to eq(@v1_0_0) } + it { expect(@v1_0_1_rc1.without_patch).to eq(@v1_0_0) } + end + + describe 'MAX_VERSION_LENGTH' do + subject { described_class::MAX_VERSION_LENGTH } + + it { is_expected.to eq(128) } + end +end diff --git a/gems/gitlab-utils/spec/spec_helper.rb b/gems/gitlab-utils/spec/spec_helper.rb new file mode 100644 index 00000000000..5dc3859f77d --- /dev/null +++ b/gems/gitlab-utils/spec/spec_helper.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +require 'rails' +require 'rspec/mocks' +require 'rspec-benchmark' +require 'rspec-parameterized' + +require 'gitlab/rspec/all' +require 'gitlab/utils/all' + +RSpec.configure do |config| + config.include RSpec::Benchmark::Matchers + + # Enable flags like --only-failures and --next-failure + config.example_status_persistence_file_path = ".rspec_status" + + # Disable RSpec exposing methods globally on `Module` and `main` + config.disable_monkey_patching! + + config.expect_with :rspec do |c| + c.syntax = :expect + end +end diff --git a/gems/ipynbdiff/.gitignore b/gems/ipynbdiff/.gitignore new file mode 100644 index 00000000000..fae8d1c772f --- /dev/null +++ b/gems/ipynbdiff/.gitignore @@ -0,0 +1,3 @@ +*.gem +coverage +.bundle diff --git a/gems/ipynbdiff/.gitlab-ci.yml b/gems/ipynbdiff/.gitlab-ci.yml new file mode 100644 index 00000000000..de5c989a4a4 --- /dev/null +++ b/gems/ipynbdiff/.gitlab-ci.yml @@ -0,0 +1,4 @@ +include: + - local: gems/gem.gitlab-ci.yml + inputs: + gem_name: "ipynbdiff" diff --git a/gems/ipynbdiff/.rubocop.yml b/gems/ipynbdiff/.rubocop.yml new file mode 100644 index 00000000000..00a3ed337f1 --- /dev/null +++ b/gems/ipynbdiff/.rubocop.yml @@ -0,0 +1,21 @@ +inherit_from: + - ../config/rubocop.yml + +CodeReuse/ActiveRecord: + Enabled: false + +Gitlab/Json: + Enabled: false + +Naming/FileName: + Exclude: + - spec/**/*.rb + - lib/gitlab/rspec.rb + - lib/gitlab/rspec/all.rb + +Rails/Pluck: + Enabled: false + +RSpec/MultipleMemoizedHelpers: + Max: 6 + AllowSubject: true diff --git a/gems/ipynbdiff/Gemfile b/gems/ipynbdiff/Gemfile new file mode 100644 index 00000000000..7f4f5e950d1 --- /dev/null +++ b/gems/ipynbdiff/Gemfile @@ -0,0 +1,5 @@ +# frozen_string_literal: true + +source 'https://rubygems.org' + +gemspec diff --git a/gems/ipynbdiff/Gemfile.lock b/gems/ipynbdiff/Gemfile.lock new file mode 100644 index 00000000000..9a583bdf274 --- /dev/null +++ b/gems/ipynbdiff/Gemfile.lock @@ -0,0 +1,139 @@ +PATH + remote: . + specs: + ipynbdiff (0.4.7) + diffy (~> 3.4) + oj (~> 3.13.16) + +GEM + remote: https://rubygems.org/ + specs: + activesupport (7.0.6) + concurrent-ruby (~> 1.0, >= 1.0.2) + i18n (>= 1.6, < 2) + minitest (>= 5.1) + tzinfo (~> 2.0) + ast (2.4.2) + benchmark-memory (0.2.0) + memory_profiler (~> 1) + binding_of_caller (1.0.0) + debug_inspector (>= 0.0.1) + coderay (1.1.3) + concurrent-ruby (1.2.2) + debug_inspector (1.1.0) + diff-lcs (1.5.0) + diffy (3.4.2) + docile (1.4.0) + gitlab-styles (10.1.0) + rubocop (~> 1.50.2) + rubocop-graphql (~> 0.18) + rubocop-performance (~> 1.15) + rubocop-rails (~> 2.17) + rubocop-rspec (~> 2.22) + i18n (1.14.1) + concurrent-ruby (~> 1.0) + json (2.6.3) + memory_profiler (1.0.0) + method_source (1.0.0) + minitest (5.18.1) + oj (3.13.23) + parallel (1.23.0) + parser (3.2.2.3) + ast (~> 2.4.1) + racc + proc_to_ast (0.1.0) + coderay + parser + unparser + pry (0.14.1) + coderay (~> 1.1) + method_source (~> 1.0) + racc (1.7.1) + rack (3.0.8) + rainbow (3.1.1) + rake (13.0.6) + regexp_parser (2.8.1) + rexml (3.2.5) + rspec (3.11.0) + rspec-core (~> 3.11.0) + rspec-expectations (~> 3.11.0) + rspec-mocks (~> 3.11.0) + rspec-core (3.11.0) + rspec-support (~> 3.11.0) + rspec-expectations (3.11.0) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.11.0) + rspec-mocks (3.11.1) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.11.0) + rspec-parameterized (1.0.0) + rspec-parameterized-core (< 2) + rspec-parameterized-table_syntax (< 2) + rspec-parameterized-core (1.0.0) + parser + proc_to_ast + rspec (>= 2.13, < 4) + unparser + rspec-parameterized-table_syntax (1.0.0) + binding_of_caller + rspec-parameterized-core (< 2) + rspec-support (3.11.0) + rubocop (1.50.2) + json (~> 2.3) + parallel (~> 1.10) + parser (>= 3.2.0.0) + rainbow (>= 2.2.2, < 4.0) + regexp_parser (>= 1.8, < 3.0) + rexml (>= 3.2.5, < 4.0) + rubocop-ast (>= 1.28.0, < 2.0) + ruby-progressbar (~> 1.7) + unicode-display_width (>= 2.4.0, < 3.0) + rubocop-ast (1.29.0) + parser (>= 3.2.1.0) + rubocop-capybara (2.18.0) + rubocop (~> 1.41) + rubocop-factory_bot (2.23.1) + rubocop (~> 1.33) + rubocop-graphql (0.19.0) + rubocop (>= 0.87, < 2) + rubocop-performance (1.18.0) + rubocop (>= 1.7.0, < 2.0) + rubocop-ast (>= 0.4.0) + rubocop-rails (2.20.2) + activesupport (>= 4.2.0) + rack (>= 1.1) + rubocop (>= 1.33.0, < 2.0) + rubocop-rspec (2.22.0) + rubocop (~> 1.33) + rubocop-capybara (~> 2.17) + rubocop-factory_bot (~> 2.22) + ruby-progressbar (1.13.0) + simplecov (0.22.0) + docile (~> 1.1) + simplecov-html (~> 0.11) + simplecov_json_formatter (~> 0.1) + simplecov-html (0.12.3) + simplecov_json_formatter (0.1.4) + tzinfo (2.0.6) + concurrent-ruby (~> 1.0) + unicode-display_width (2.4.2) + unparser (0.6.8) + diff-lcs (~> 1.3) + parser (>= 3.2.0) + +PLATFORMS + ruby + +DEPENDENCIES + benchmark-memory (~> 0.2.0) + bundler (~> 2.2) + gitlab-styles (~> 10.1.0) + ipynbdiff! + pry (~> 0.14) + rake (~> 13.0) + rspec (~> 3.10) + rspec-parameterized (~> 1.0) + simplecov (~> 0.22.0) + +BUNDLED WITH + 2.3.16 diff --git a/gems/ipynbdiff/LICENSE b/gems/ipynbdiff/LICENSE new file mode 100644 index 00000000000..e6de2f90864 --- /dev/null +++ b/gems/ipynbdiff/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2016-2021 GitLab B.V. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/gems/ipynbdiff/README.md b/gems/ipynbdiff/README.md new file mode 100644 index 00000000000..f046f678a4d --- /dev/null +++ b/gems/ipynbdiff/README.md @@ -0,0 +1,56 @@ +# IpynbDiff: Better diff for Jupyter Notebooks + +This is a simple diff tool that cleans up Jupyter notebooks, transforming each [notebook](example/1/from.ipynb) +into a [readable markdown file](example/1/from_html.md), keeping the output of cells, and running the +diff after. Markdowns are generated using an opinionated Jupyter to Markdown conversion. This means +that the entire file is readable on the diff. + +The result are diffs that are much easier to read: + +| Diff | IpynbDiff | +| ----------------------------------- | ----------------------------------------------------- | +| [Diff text](example/diff.txt) | [IpynbDiff text](example/ipynbdiff_percent.txt) | +|  |  | + +This started as a port of [ipynbdiff](https://gitlab.com/gitlab-org/incubation-engineering/mlops/poc/ipynbdiff), +but now has extended functionality although not working as git driver. + +## Usage + +### Generating diffs + +```ruby +IpynbDiff.diff(from_path, to_path, options) +``` + +Options: + +```ruby +@default_transform_options = { + preprocess_input: true, # Whether the input should be transformed + write_output_to: nil, # Pass a path to save the output to a file + format: :text, # These are the formats Diffy accepts https://github.com/samg/diffy + sources_are_files: false, # Weather to use the from/to as string or path to a file + raise_if_invalid_notebook: false, # Raises an error if the notebooks are invalid, otherwise returns nil + transform_options: @default_transform_options, # See below for transform options + diff_opts: { + include_diff_info: false # These are passed to Diffy https://github.com/samg/diffy + } +} +``` + +### Transforming the notebooks + +It might be necessary to have the transformed files in addition to the diff. + +```ruby +IpynbDiff.transform(notebook, options) +``` + +Options: + +```ruby +@default_transform_options = { + include_frontmatter: false, # Whether to include or not the notebook metadata (kernel, language, etc) +} +``` diff --git a/gems/ipynbdiff/ipynbdiff.gemspec b/gems/ipynbdiff/ipynbdiff.gemspec new file mode 100644 index 00000000000..8bc3e1b142d --- /dev/null +++ b/gems/ipynbdiff/ipynbdiff.gemspec @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +$LOAD_PATH.push File.expand_path('lib', __dir__ || '') + +require_relative 'lib/ipynb_diff/version' + +Gem::Specification.new do |s| + s.name = 'ipynbdiff' + s.version = IpynbDiff::Version::VERSION + s.summary = 'Human Readable diffs for Jupyter Notebooks' + s.description = 'Better diff for Jupyter Notebooks by first preprocessing them and removing clutter' + s.authors = ['Eduardo Bonet'] + s.email = 'ebonet@gitlab.com' + # Specify which files should be added to the gem when it is released. + # The `git ls-files -z` loads the files in the RubyGem that have been added into git. + s.files = Dir['lib/**/*.rb'] + s.require_paths = ["lib"] + s.required_ruby_version = ">= 3.0" + s.homepage = 'https://gitlab.com/gitlab-org/gitlab/-/tree/master/gems/ipynbdiff' + s.license = 'MIT' + + s.add_runtime_dependency 'diffy', '~> 3.4' + s.add_runtime_dependency 'oj', '~> 3.13.16' + + s.add_development_dependency 'benchmark-memory', '~>0.2.0' + s.add_development_dependency 'bundler', '~> 2.2' + s.add_development_dependency 'gitlab-styles', '~> 10.1.0' + s.add_development_dependency 'pry', '~> 0.14' + s.add_development_dependency 'rake', '~> 13.0' + s.add_development_dependency 'rspec', '~> 3.10' + s.add_development_dependency 'rspec-parameterized', '~> 1.0' + s.add_development_dependency 'simplecov', '~> 0.22.0' +end diff --git a/gems/ipynbdiff/lib/ipynb_diff.rb b/gems/ipynbdiff/lib/ipynb_diff.rb new file mode 100644 index 00000000000..605ff6e4a75 --- /dev/null +++ b/gems/ipynbdiff/lib/ipynb_diff.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +require 'ipynb_diff/transformer' +require 'ipynb_diff/diff' +require 'ipynb_diff/symbol_map' + +# Human Readable Jupyter Diffs +module IpynbDiff + def self.diff(from, to, raise_if_invalid_nb: false, include_frontmatter: false, hide_images: false, diffy_opts: {}) + transformer = Transformer.new(include_frontmatter: include_frontmatter, hide_images: hide_images) + + Diff.new(transformer.transform(from), transformer.transform(to), diffy_opts) + rescue InvalidNotebookError + raise if raise_if_invalid_nb + end + + def self.transform(notebook, raise_errors: false, include_frontmatter: true, hide_images: false) + return unless notebook + + Transformer.new(include_frontmatter: include_frontmatter, hide_images: hide_images).transform(notebook).as_text + rescue InvalidNotebookError + raise if raise_errors + end +end diff --git a/gems/ipynbdiff/lib/ipynb_diff/diff.rb b/gems/ipynbdiff/lib/ipynb_diff/diff.rb new file mode 100644 index 00000000000..3554ac55d99 --- /dev/null +++ b/gems/ipynbdiff/lib/ipynb_diff/diff.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +# Custom differ for Jupyter Notebooks +module IpynbDiff + require 'delegate' + + # The result of a diff object + class Diff < SimpleDelegator + require 'diffy' + + attr_reader :from, :to + + def initialize(from, to, diffy_opts) + super(Diffy::Diff.new(from.as_text, to.as_text, **diffy_opts)) + + @from = from + @to = to + end + end +end diff --git a/gems/ipynbdiff/lib/ipynb_diff/output_transformer.rb b/gems/ipynbdiff/lib/ipynb_diff/output_transformer.rb new file mode 100644 index 00000000000..95dbcecf95c --- /dev/null +++ b/gems/ipynbdiff/lib/ipynb_diff/output_transformer.rb @@ -0,0 +1,73 @@ +# frozen_string_literal: true + +require 'ipynb_diff/symbolized_markdown_helper' + +module IpynbDiff + # Transforms Jupyter output data into markdown + class OutputTransformer + include SymbolizedMarkdownHelper + + HIDDEN_IMAGE_OUTPUT = ' [Hidden Image Output]' + + ORDERED_KEYS = { + 'execute_result' => %w[image/png image/svg+xml image/jpeg text/markdown text/latex text/plain], + 'display_data' => %w[image/png image/svg+xml image/jpeg text/markdown text/latex], + 'stream' => %w[text] + }.freeze + + def initialize(hide_images = false) + @hide_images = hide_images + end + + def transform(output, symbol) + case (output_type = output['output_type']) + when 'error' + transform_error(output['traceback'], symbol / 'traceback') + when 'execute_result', 'display_data' + transform_non_error(ORDERED_KEYS[output_type], output['data'], symbol / 'data') + when 'stream' + transform_element('text', output['text'], symbol) + end + end + + def transform_error(traceback, symbol) + traceback.map.with_index do |t, idx| + t.split("\n").map do |l| + ___(symbol / idx, l.gsub(/\[[0-9][0-9;]*m/, '').sub("\u001B", ' ').delete("\u001B").rstrip) + end + end + end + + def transform_non_error(accepted_keys, elements, symbol) + accepted_keys.filter { |key| elements.key?(key) }.map do |key| + transform_element(key, elements[key], symbol) + end + end + + def transform_element(output_type, output_element, symbol_prefix) + new_symbol = symbol_prefix / output_type + case output_type + when 'image/png', 'image/jpeg' + transform_image("#{output_type};base64", output_element, new_symbol) + when 'image/svg+xml' + transform_image("#{output_type};utf8", output_element, new_symbol) + when 'text/markdown', 'text/latex', 'text/plain', 'text' + transform_text(output_element, new_symbol) + end + end + + def transform_image(image_type, image_content, symbol) + return ___(nil, HIDDEN_IMAGE_OUTPUT) if @hide_images + + lines = image_content.is_a?(Array) ? image_content : [image_content] + + single_line = lines.map(&:strip).join.gsub(/\s+/, ' ') + + ___(symbol, " ") + end + + def transform_text(text_content, symbol) + symbolize_array(symbol, text_content) { |l| " #{l.rstrip}" } + end + end +end diff --git a/gems/ipynbdiff/lib/ipynb_diff/symbol_map.rb b/gems/ipynbdiff/lib/ipynb_diff/symbol_map.rb new file mode 100644 index 00000000000..383f1de5c18 --- /dev/null +++ b/gems/ipynbdiff/lib/ipynb_diff/symbol_map.rb @@ -0,0 +1,109 @@ +# frozen_string_literal: true + +module IpynbDiff + require 'oj' + + # Creates a map from a symbol to the line number it appears in a Json file + # + # Example: + # + # Input: + # + # 1. { + # 2. 'obj1': [ + # 3. { + # 4. 'obj2': 5 + # 5. }, + # 6. 3, + # 7. { + # 8. 'obj3': { + # 9. 'obj4': 'b' + # 10. } + # 11. } + # 12. ] + # 13.} + # + # Output: + # + # Symbol Line Number + # .obj1 -> 2 + # .obj1.0 -> 3 + # .obj1.0 -> 3 + # .obj1.0.obj2 -> 4 + # .obj1.1 -> 6 + # .obj1.2 -> 7 + # .obj1.2.obj3 -> 8 + # .obj1.2.obj3.obj4 -> 9 + # + class SymbolMap + # rubocop:disable Lint/UnusedMethodArgument + class << self + def handler + @handler ||= SymbolMap.new + end + + def parser + @parser ||= Oj::Parser.new(:saj).tap { |p| p.handler = handler } + end + + def parse(notebook, *args) + handler.reset + parser.parse(notebook) + handler.symbols + end + end + + attr_accessor :symbols + + def hash_start(key, line, column) + add_symbol(key_or_index(key), line) + end + + def hash_end(key, line, column) + @current_path.pop + end + + def array_start(key, line, column) + @current_array_index << 0 + + add_symbol(key, line) + end + + def array_end(key, line, column) + @current_path.pop + @current_array_index.pop + end + + def add_value(value, key, line, column) + add_symbol(key_or_index(key), line) + + @current_path.pop + end + + def add_symbol(symbol, line) + @symbols[@current_path.append(symbol).join('.')] = line if symbol + end + + def key_or_index(key) + if key.nil? # value in an array + if @current_path.empty? + @current_path = [''] + return + end + + symbol = @current_array_index.last + @current_array_index[-1] += 1 + symbol + else + key + end + end + + def reset + @current_path = [] + @symbols = {} + @current_array_index = [] + end + # rubocop:enable Lint/UnusedMethodArgument + end +end diff --git a/gems/ipynbdiff/lib/ipynb_diff/symbolized_markdown_helper.rb b/gems/ipynbdiff/lib/ipynb_diff/symbolized_markdown_helper.rb new file mode 100644 index 00000000000..991c9e493bc --- /dev/null +++ b/gems/ipynbdiff/lib/ipynb_diff/symbolized_markdown_helper.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module IpynbDiff + # Helper functions + module SymbolizedMarkdownHelper + def ___(symbol = nil, content = '') + { symbol: symbol, content: content } + end + + def symbolize_array(symbol, content) + if content.is_a?(Array) + content.map.with_index { |l, idx| ___(symbol / idx, yield(l)) } + else + content.split("\n").map { |c| ___(symbol, c) } + end + end + end + + # Simple wrapper for a string + class JsonSymbol < String + def /(other) + JsonSymbol.new((other.is_a?(Array) ? [self, *other] : [self, other]).join('.')) + end + end +end diff --git a/gems/ipynbdiff/lib/ipynb_diff/transformed_notebook.rb b/gems/ipynbdiff/lib/ipynb_diff/transformed_notebook.rb new file mode 100644 index 00000000000..f98e5f68086 --- /dev/null +++ b/gems/ipynbdiff/lib/ipynb_diff/transformed_notebook.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +module IpynbDiff + # Notebook that was transformed into md, including location of source cells + class TransformedNotebook + attr_reader :blocks + + def as_text + @blocks.map { |b| b[:content].gsub(/\n/, '\\n') }.join("\n") + end + + private + + def initialize(lines = [], symbol_map = {}) + @blocks = lines.map do |line| + { content: line[:content], source_symbol: (symbol = line[:symbol]), source_line: symbol && symbol_map[symbol] } + end + end + end +end diff --git a/gems/ipynbdiff/lib/ipynb_diff/transformer.rb b/gems/ipynbdiff/lib/ipynb_diff/transformer.rb new file mode 100644 index 00000000000..2b386168b5d --- /dev/null +++ b/gems/ipynbdiff/lib/ipynb_diff/transformer.rb @@ -0,0 +1,111 @@ +# frozen_string_literal: true + +require 'json' +require 'yaml' +require 'ipynb_diff/output_transformer' +require 'ipynb_diff/symbolized_markdown_helper' +require 'ipynb_diff/symbol_map' +require 'ipynb_diff/transformed_notebook' +require 'oj' + +module IpynbDiff + InvalidNotebookError = Class.new(StandardError) + + # Returns a markdown version of the Jupyter Notebook + class Transformer + include SymbolizedMarkdownHelper + + @include_frontmatter = true + + def initialize(include_frontmatter: true, hide_images: false) + @include_frontmatter = include_frontmatter + @hide_images = hide_images + @out_transformer = OutputTransformer.new(hide_images) + end + + def validate_notebook(notebook) + notebook_json = Oj::Parser.usual.parse(notebook) + + return notebook_json if notebook_json&.key?('cells') + + raise InvalidNotebookError + rescue EncodingError, Oj::ParseError, JSON::ParserError + raise InvalidNotebookError + end + + def transform(notebook) + return TransformedNotebook.new unless notebook + + notebook_json = validate_notebook(notebook) + transformed = transform_document(notebook_json) + symbol_map = SymbolMap.parse(notebook) + + TransformedNotebook.new(transformed, symbol_map) + end + + def transform_document(notebook) + symbol = JsonSymbol.new('.cells') + + transformed_blocks = notebook['cells'].map.with_index do |cell, idx| + decorate_cell(transform_cell(cell, notebook, symbol / idx), cell, symbol / idx) + end + + transformed_blocks.prepend(transform_metadata(notebook)) if @include_frontmatter + transformed_blocks.flatten + end + + def decorate_cell(rows, cell, symbol) + tags = cell['metadata']&.fetch('tags', []) + type = cell['cell_type'] || 'raw' + + [ + ___(symbol, %(%% Cell type:#{type} id:#{cell['id']} tags:#{tags&.join(',')})), + ___, + rows, + ___ + ] + end + + def transform_cell(cell, notebook, symbol) + cell['cell_type'] == 'code' ? transform_code_cell(cell, notebook, symbol) : transform_text_cell(cell, symbol) + end + + def transform_code_cell(cell, notebook, symbol) + [ + ___(symbol / 'source', %(``` #{notebook.dig('metadata', 'kernelspec', 'language') || ''})), + symbolize_array(symbol / 'source', cell['source'], &:rstrip), + ___(nil, '```'), + transform_outputs(cell['outputs'], symbol) + ] + end + + def transform_outputs(outputs, symbol) + transformed = outputs.map + .with_index { |output, i| @out_transformer.transform(output, symbol / ['outputs', i]) } + .compact + .map { |el| [___, el] } + + [ + transformed.empty? ? [] : [___, ___(symbol / 'outputs', '%% Output')], + transformed + ] + end + + def transform_text_cell(cell, symbol) + symbolize_array(symbol / 'source', cell['source'], &:rstrip) + end + + def transform_metadata(notebook_json) + as_yaml = { + 'jupyter' => { + 'kernelspec' => notebook_json['metadata']['kernelspec'], + 'language_info' => notebook_json['metadata']['language_info'], + 'nbformat' => notebook_json['nbformat'], + 'nbformat_minor' => notebook_json['nbformat_minor'] + } + }.to_yaml + + as_yaml.split("\n").map { |l| ___(nil, l) }.append(___(nil, '---'), ___) + end + end +end diff --git a/gems/ipynbdiff/lib/ipynb_diff/version.rb b/gems/ipynbdiff/lib/ipynb_diff/version.rb new file mode 100644 index 00000000000..1a407f9c0fa --- /dev/null +++ b/gems/ipynbdiff/lib/ipynb_diff/version.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +module IpynbDiff + module Version + VERSION = '0.4.7' + end +end diff --git a/gems/ipynbdiff/spec/benchmark.rb b/gems/ipynbdiff/spec/benchmark.rb new file mode 100644 index 00000000000..514c8700183 --- /dev/null +++ b/gems/ipynbdiff/spec/benchmark.rb @@ -0,0 +1,68 @@ +# frozen_string_literal: true + +require 'ipynbdiff' +require 'benchmark' +require 'benchmark/memory' +require_relative 'test_helper' + +# rubocop:disable Layout/LineLength +large_cell = '{ + "cell_type": "code", + "execution_count": 9, + "id": "24f32781-48bf-4378-b30c-ccdce7b05ba0", + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAABj0AAAHwCAYAAAD91q10AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAA9hAAAPYQGoP6dpAABPSUlEQVR4nO39eZCfd34f+L2fxkmAAAiCBO/7PobHkCAxIEjcV/eWHTl/yOVKvFaloo3kTZXjlKUa2bVS4tiyna2JElmudaUqK+0m9m4ceVVxutEAiIMXeIEEObyG4PC+QPAEQIA4+8kfD1pf/DgkhwC78XT/+vWqYmHQnybxLml6COCNz/dT1XUdAAAAAACA8a6n7QAAAAAAAAAjQekBAAAAAAB0BaUHAAAAAADQFZQeAAAAAABAV1B6AAAAAAAAXUHpAQAAAAAAdAWlBwAAAAAA0BUmtx3g66qqqpJcnGR/21kAAAAAAIAxYVaSD+q6rr/rk8Zc6ZGm8Hiv7RAAAAAAAMCYcmmS97/rE8Zi6bE/Sd59993Mnj277SwAAAAAAECL9u3bl8suuyz5Hi9EjcXSI0kye/ZspQcAAAAAAPC9OWQOAAAAAAB0BaUHAAAAAADQFZQeAAAAAABAV1B6AAAAAAAAXUHpAQAAAAAAdAWlBwAAAAAA0BWUHgAAAAAAQFdQegAAAAAAAF1B6QEAAAAAAHQFpQcAAAAAANAVlB4AAAAAAEBXUHoAAAAAAABdQekBAAAAAAB0BaUHAAAAAADQFZQeAAAAAABAV1B6AAAAAAAAXeGUSo+qqn5aVdXTVVXtr6pqT1VVf1VV1Q1f+5w/r6qq/tpfT4xsbAAAAAAA6DKffJJs2dJ2inFt8il+/pIkf5bk6RN/7z9LsrGqqpvruj5w0ucNJvmtk75/5AelBAAAAACAblPXyXPPJf39ycBA8sQTyfTpyaefJmed1Xa6cemUSo+6rtee/P2qqn4ryZ4kdyV5+KTR4bqud//weAAAAAAA0EX27082by5FxwcfdM6vvz55773kuuvayTfOneqmx9fNOfHtZ1/7+NKqqvYk+SLJQ0n+cV3Xe77pH1BV1bQk00760KwfmAkAAAAAAMaO115rSo7+/uThh5MjJz2ONGNGsnJl0teX9PYml17aXs4ucNqlR1VVVZKfJXm0rusXTxqtT/Ifkryd5Kok/zTJlqqq7qrr+vA3/KN+muQPTzcHAAAAAACMKYcPN+XG8DbHa691zq+5pik5+vqSBx5onrRiRFR1XZ/e31hVf5akL8niuq7f+47PuyhNAfK367r+j98w/6ZNj/f27t2b2bNnn1Y2AAAAAAA4o95/vyk4BgaSTZuSAyedwZ4ypSk3enubouP665Oqai/rOLNv377MmTMnSebUdb3vuz73tDY9qqr60yR/I8kD31V4JEld1x9WVfV2km98gOzE9sdfb4BU/h8NAAAAAMBYd/x48tRT5dmq557rnF94YSk5Vq5M/CH/M+KUSo8TT1r9aZLfSLK0rus3v8ffMy/JZUk+PK2EAAAAAAAwFnz2WbJhQ1NyDA4mn35aZlWV3HNPebbqjjuSnp7Wok5Up7rp8WdJ/k6Sv5lkf1VVF574+N66rr+qqursJH+U5C/TlBxXJvnnST5J8j+NRGAAAAAAADgj6jp54YXmyar+/mT79mRoqMznzEnWrGlKjrVrk/nz28tKklMvPX7nxLfbvvbx30ry50mOJ/lRkr+b5Jw0xcfWJL9Z1/X+0w0JAAAAAABnxIEDyZYt5Qj5u+92zm+5pWxzLFqUTD6tKxKMklP6/0Zd1995cKOu66+SrPlBiQAAAAAA4Ex6441ym2PbtuTw4TKbPj1ZsaIpOXp7kyuuaC0mv54KCgAAAACAieXo0eTRR0vR8YtfdM6vuKJscyxblpx1Vjs5OWVKDwAAAAAAut/u3cn69U3JsWlTsm9fmU2alCxeXIqOm25qDpMz7ig9AAAAAADoPkNDyY4d5Qj5jh2d8/PPb56r6u1NVq9OzjmnlZiMLKUHAAAAAADdYe/eZOPGpuRYvz7Zs6dzfvfdTcnR19f8556ednIyapQeAAAAAACMT3WdvPJKU3IMDDR3Oo4dK/NZs5otjr6+ZN265MIL28vKGaH0AAAAAABg/Pjqq2TbtnKE/K23Ouc33lhuc9x3XzJ1ahspaYnSAwAAAACAse2dd8o2x+bNTfExbNq0ZNmy8mzV1Ve3l5PWKT0AAAAAABhbjh1LHn+8bHO8+GLn/NJLyzbH8uXJzJnt5GTMUXoAAAAAANC+jz9OBgebkmPDhuSLL8qspydZtKgUHbfemlRVa1EZu5QeAAAAAACceXWd7NxZtjmeeqr52LB585K1a5uSY82a5Nxz28vKuKH0AAAAAADgzNi/P9m0qbnNMTCQfPhh5/yOO8o2xz33JJMmtRKT8UvpAQAAAADA6Nm1q2xzPPxwcvRomc2cmaxa1ZQc69Yll1zSXk66gtIDAAAAAICRc/hw8tBDTckxMJD88ped82uvLdscDzyQTJvWTk66ktIDAAAAAIAf5v33m4Kjvz958MHkwIEymzIlWbKkFB3XXddeTrqe0gMAAAAAgFNz/Hjy5JPl2arnn++cX3RR0tvblBwrVyazZrWTkwlH6QEAAAAAwK/32WfJ4GBTcgwONt8fVlXJvfeWbY477mg+BmeY0gMAAAAAgF9V18kLL5RtjscfT4aGyvycc5K1a5uNjrVrk/PPby0qDFN6AAAAAADQOHAg2by5HCF/773O+a23lm2On/wkmey3mBlb/DcSAAAAAGAie/31coR827bk8OEyO+usZMWKZpujtze54orWYsL3ofQAAAAAAJhIjhxJHn20PFv16qud8yuvLNscS5c2xQeME0oPAAAAAIBut3t3s80xMJBs3Jjs319mkycnixeXouPGGx0hZ9xSegAAAAAAdJuhoWTHjrLN8cwznfP588uTVatXJ3PmtJMTRpjSAwAAAACgG3zxRbPFMTCQrF+f7NnTOb/77rLNcdddSU9PKzFhNCk9AAAAAADGo7pOXnmlbHM8+mhy/HiZz57dbHH09ibr1iUXXtheVjhDlB4AAAAAAOPFV18lW7c2JcfAQPLWW53zG28s2xz33ZdMndpKTGiL0gMAAAAAYCx7++1ScmzZ0hQfw6ZNS5Yta7Y5+vqSq69uLyeMAUoPAAAAAICx5NixZPv28mzVSy91zi+9tGxzLF+ezJzZTk4Yg5QeAAAAAABt+/jj5vh4f3+yYUOyd2+Z9fQkixaVouPWW5Oqai8rjGFKDwAAAACAM21oKNm5s2xzPP10c5h82Lx5zfHxvr7mGPm557aXFcYRpQcAAAAAwJmwb1/y4IPlPsfu3Z3zO+8stznuuSeZNKmdnDCOKT0AAAAAAEZDXSe7dpVtjkceSY4eLfOZM5NVq5qSo7c3ufji9rJCl1B6AAAAAACMlEOHkoceKtscr7/eOb/uunKb4/77k2nT2skJXUrpAQAAAADwQ7z3XlNw9Pc3z1cdPFhmU6cmS5aUZ6uuu669nDABKD0AAAAAAE7F8ePJE0+UZ6t+/vPO+cUXl5Jj5crk7LPbyQkTkNIDAAAAAODX+fTTZHCw2egYHEw++6zMqipZuLA8W3X77c3HgDNO6QEAAAAA8HV13WxwDG9zPPFEMjRU5nPnJmvXNhsda9cm553XXlbgryk9AAAAAACS5Msvk82bm22OgYHmVsfJfvSjss2xcGEy2W+vwljjqxIAAAAAmLhef71sc2zblhw5UmZnndXc5Ojtbf66/PLWYgLfj9IDAAAAAJg4jhxJHnmkFB27dnXOr7qqbHMsXZpMn95KTOD0KD0AAAAAgO724YflyapNm5L9+8ts8uTk/vubTY6+vuTGGx0hh3FM6QEAAAAAdJehoeTpp8s2x7PPds7nzy8lx6pVyZw57eQERpzSAwAAAAAY/774ItmwoSk5BgeTjz/unC9Y0JQcvb3JXXclPT2txARGl9IDAAAAABh/6jp56aXmyar+/uSxx5Ljx8t89uxk9eqm6Fi3LrnggvayAmeM0gMAAAAAGB8OHky2bm1KjoGB5O23O+c33VSOkN93XzJlSjs5gdYoPQAAAACAseutt8ptjq1bk0OHymzatGT58vJs1VVXtRYTGBuUHgAAAADA2HH0aLJ9eyk6Xn65c37ZZWWbY/nyZMaMdnICY5LSAwAAAABo1549yfr1TcmxcWOyd2+ZTZqULFpUio5bbkmqqr2swJim9AAAAAAAzqyhoeTZZ8s2x44dzWHyYeed1xwf7+1N1qxJ5s5tLyswrig9AAAAAIDRt29fsmlTOUL+0Ued8x//uCk5+vqSBQuaDQ+AU6T0AAAAAABGXl0nr75atjkeeSQ5dqzMzz47WbWqKTnWrUsuvri9rEDXUHoAAAAAACPj0KFk27ayzfHGG53z668v2xz3359Mm9ZKTKB7KT0AAAAAgNP37rtNwdHfn2zenBw8WGZTpyZLl5ai49prW4sJTAxKDwAAAADg+zt2LHniifJs1QsvdM4vvrgpOPr6khUrmmesAM4QpQcAAAAA8N0++STZsKEpOQYHk88/L7OenmThwrLNcfvtSVW1lxWY0JQeAAAAAECnuk6ef75sczz5ZDI0VOZz5yZr1zYlx5o1yXnntZcV4CRKDwAAAAAg+fLL5MEHyxHyDz7onN92W1Ny9PY2mx2T/dYiMPb4XyYAAAAAmKh++cuyzfHQQ8mRI2U2Y0Zzk2O46LjssvZyAnxPSg8AAAAAmCiOHEkefrhsc+za1Tm/+upyhHzJkmT69HZyApwmpQcAAAAAdLMPPkjWr2+Kjk2bmmeshk2enNx/fyk6brjBEXJgXFN6AAAAAEA3OX48efrp8mzVzp2d8wsuaJ6r6utLVq1KZs9uJyfAKFB6AAAAAMB49/nnyYYNTckxOJh88kmZVVWyYEG5zfHjHyc9Pe1lBRhFSg8AAAAAGG/qOnnppbLNsX17s+ExbM6cZM2apuRYty6ZP7+9rABnkNIDAAAAAMaDgweTLVvKEfJ33umc33xzuc2xaFEyZUo7OQFapPQAAAAAgLHqzTdLybF1a3LoUJlNn54sX16erbryytZiAowVSg8AAAAAGCuOHk0efbQpOfr7k1de6ZxffnnZ5li2LJkxo52cAGOU0gMAAAAA2vTRR8n69U3JsXFjsm9fmU2alNx3Xyk6br65OUwOwDdSegAAAADAmTQ0lDzzTDlCvmNH5/z885vj4319yapVydy57eQEGIeUHgAAAAAw2vbuTTZtakqO9eub7Y6T3XVXc5ejry9ZsCDp6WknJ8A4p/QAAAAAgJFW18kvflG2OR59NDl2rMxnzWq2OPr6mq2Oiy5qLytAF1F6AAAAAMBIOHQo2bq1HCF/883O+Q03NCVHb29y//3J1Knt5AToYkoPAAAAADhd775btjk2b06++qrMpk5Nli4tR8ivuaa1mAATxSmVHlVV/TTJ30pyY5KvkmxP8vt1Xb960udUSf4wyW8nmZvkySR/v67rl0YqNAAAAAC04tix5PHHyzbHCy90zi+5pGxzrFiRnH12OzkBJqhT3fRYkuTPkjx94u/9Z0k2VlV1c13XB058zu8l+YdJ/l6SXUn+SZJNVVXdUNf1/hFJDQAAAABnyiefJIODTcmxYUPy+edl1tOTLFxYtjluuy2pqvayAkxwVV3Xp/83V9X5SfYkWVLX9cMntjw+SPIndV3/yxOfMy3JR2k2Qv7t9/hnzk6yd+/evZk9e/ZpZwMAAACA01LXyXPPNSXHwEDyxBPNx4ade26ydm1TcqxZk8yb11pUgIlg3759mTNnTpLMqet633d97g+96THnxLefnfj2qiQXJtk4/Al1XR+uquqhJIuS/ErpcaIUmXbSh2b9wEwAAAAAcGr2729ucgwXHR980Dm/7bayzXHvvclkp3IBxqLT/l/nE1sdP0vyaF3XL5748IUnvv3oa5/+UZIrvuUf9dM0N0AAAAAA4Mx57bVyhPzhh5MjR8psxoxk5cpyn+PSS9vLCcD39kMq6X+d5LYki79h9vU3s6pv+NiwP05TngybleS9H5ALAAAAAH7V4cNNuTG8zfHaa53za64pJceSJcn06e3kBOC0nVbpUVXVnyb5G0keqOv65IJi94lvL0zy4Ukfn59f3f5I0jx/leTwSf/s04kEAAAAAL/q/feT9eubouPBB5MvvyyzyZOTBx4oz1Zdf70j5ADj3CmVHieetPrTJL+RZGld129+7VPeTFN8rEqy88TfMzXJkiS//4PTAgAAAMB3OX48eeqp8mzVc891zi+8sNnk6Otrnq+aPbuVmACMjlPd9PizJH8nyd9Msr+qquEbHnvruv6qruu6qqo/SfIHVVW9luS1JH+Q5GCSfzdCmQEAAACg+OyzZMOGpuQYHEw+/bTMqiq5557ybNWddyY9Pe1lBWBUnWrp8Tsnvt32tY//VpI/P/Gf/1WSs5L8myRzkzyZZHVd1/tPLyIAAAAAnKSukxdeaO5y9Pcn27cnQ0NlPmdOsmZNU3SsXZvMn99eVgDOqKquv+2+eDuqqpqdZO/evXsz23ohAAAAAEly4ECyZUs5Qv7uu53zW24ptzkWLWrudQDQFfbt25c5c+YkyZy6rvd91+f6X38AAAAAxqY33ii3ObZtSw4fLrPp05MVK8qzVVdc0VpMAMYOpQcAAAAAY8ORI8ljj5Wi4xe/6JxfcUXZ5li2LDnrrHZyAjBmKT0AAAAAaM/u3cn69U3JsXFjsv+ks7CTJiWLF5ei46abmsPkAPAtlB4AAAAAnDlDQ8mOHeU2x44dnfPzz2+eq+rtTVavTs45p5WYAIxPSg8AAAAARtcXXySbNjVFx/r1yZ49nfO77irbHHffnfT0tBITgPFP6QEAAADAyKrr5JVXym2Oxx5Ljh0r81mzmi2Ovr5k3brkwgvbywpAV1F6AAAAAPDDffVVsm1bKTreeqtzfsMNZZtj8eJk6tQ2UgLQ5ZQeAAAAAJyed94pJceWLU3xMWzatGTp0qbk6O1NrrmmtZgATBxKDwAAAAC+n2PHku3byxHyF1/snF96aSk5VqxIZs5sJycAE5bSAwAAAIBv9/HHyeBgU3Rs2NAcJR/W05P85Cfl2aof/SipqtaiAoDSAwAAAICirpOdO8s2x5NPNh8bdu65zfHxvr7mGPm8ee1lBYCvUXoAAAAATHT79ycPPliKjg8/7JzffnvZ5rj33mTSpHZyAsCvofQAAAAAmIh27SpHyB9+ODl6tMxmzkxWrmxKjnXrmlsdADAOKD0AAAAAJoLDh5tyY7jo+OUvO+fXXFO2OZYsSaZNaycnAPwASg8AAACAbvX++81zVf39zfNVBw6U2ZQpyQMPlKLj+uvbywkAI0TpAQAAANAtjh9vDo8PFx3PPdc5v+iipLe3KTlWrkxmzWolJgCMFqUHAAAAwHj22WfJhg1NyTE4mHz6aZlVVXLPPWWb4447kp6e1qICwGhTegAAAACMJ3WdvPBCuc3x+OPJ0FCZn3NOsmZNU3KsXZucf35rUQHgTFN6AAAAAIx1Bw4kmzc3JcfAQPLee53zW29tSo7e3mTRomSy3/IBYGLyb0AAAACAsej118ttjm3bksOHy+yss5Lly0vRccUVrcUEgLFE6QEAAAAwFhw5kjz6aHm26tVXO+dXXllucyxd2hQfAEAHpQcAAABAW3bvbrY5BgaSjRuT/fvLbPLkZPHiZpOjry+56abmMDkA8K2UHgAAAABnytBQ8vTT5dmqZ57pnM+fn6xb15Qcq1cnc+a0kxMAximlBwAAAMBo+uKLZoujvz9Zvz75+OPO+d13l2er7ror6elpJSYAdAOlBwAAAMBIquvk5ZfLbY7HHkuOHy/z2bObLY7e3mar48IL28sKAF1G6QEAAADwQ331VbJ1ayk63n67c37jjWWb4777kqlT28kJAF1O6QEAAABwOt5+u5QcW7Ykhw6V2bRpybJl5Qj51Ve3lxMAJhClBwAAAMD3cexYsn17KTpeeqlzfumlZZtj+fJk5sx2cgLABKb0AAAAAPg2H3/cHB/v7082bEj27i2znp5k0aJSdNx6a1JV7WUFAJQeAAAAAH9taCjZuTMZGGiKjqeeag6TD5s3L1m7tik51qxJzj23vawAwK9QegAAAAAT2/79yaZNTckxMJDs3t05v+OOss1xzz3JpEmtxAQAfj2lBwAAADCx1HWya1fZ5nj44eTo0TKfOTNZtaopOdatSy65pL2sAMApUXoAAAAA3e/w4eShh8oR8tdf75xfd11TcvT2Jg88kEyb1k5OAOAHUXoAAAAA3en998uTVQ8+mBw4UGZTpiRLlpRnq667rr2cAMCIUXoAAAAA3eH48eTJJ8s2x/PPd84vvrjZ5OjrS1asSGbNaicnADBqlB4AAADA+PXZZ8ngYFNyDA423x9WVcnCheXZqjvuaD4GAHQtpQcAAAAwftR18vOfl2erHn88GRoq83POSdaubYqOtWuT885rLSoAcOYpPQAAAICx7cCBZPPmUnS8917n/Ec/Krc5Fi5MJvvtDgCYqPwsAAAAABh7Xn+93ObYti05cqTMzjorWbmyebKqtze5/PLWYgIAY4vSAwAAAGjfkSPJI4+UomPXrs75VVeVbY6lS5Pp01uJCQCMbUoPAAAAoB0ffpisX9+UHJs2Jfv3l9nkycn99zebHH19yY03OkIOAPxaSg8AAADgzBgaSp5+umxzPPts53z+/FJyrFqVzJnTTk4AYNxSegAAAACj54svkg0bmpJjcDD5+OPO+YIFTcnR25vcdVfS09NKTACgOyg9AAAAgJFT18nLL5dtjsceS44fL/PZs5PVq5uiY9265IIL2ssKAHQdpQcAAADwwxw8mGzd2pQcAwPJ2293zm+6qRwhv+++ZMqUdnICAF1P6QEAAACcurfeagqO/v5ky5bk0KEymzYtWbasFB1XXdVaTABgYlF6AAAAAL/e0aPJ9u3l2aqXX+6cX3ZZKTmWL09mzGgnJwAwoSk9AAAAgG+2Z0+yfn2z0bFhQ7J3b5lNmpQsWlSOkN96a1JV7WUFAIjSAwAAABg2NJTs3Fm2OZ5+ujlMPmzevOb4eF9fsmZNMndue1kBAL6B0gMAAAAmsn37kk2bmpJj/fpk9+7O+Z13lmerFixoNjwAAMYopQcAAABMJHWd7NpVtjkeeaS51zHs7LOTVauaJ6t6e5OLL24vKwDAKVJ6AAAAQLc7dCh56KFSdLzxRuf8uuvKNsf99yfTprWTEwDgB1J6AAAAQDd6772m4BgYSB58MDl4sMymTk2WLClHyK+7rr2cAAAjSOkBAAAA3eDYseTJJ8s2x89/3jm/+OKyzbFiRfOMFQBAl1F6AAAAwHj16afJ4GBTcmzYkHz2WZn19CQLFzabHH19ye23J1XVXlYAgDNA6QEAAADjRV0nzz/fPFnV35888UQyNFTmc+cma9c2JceaNcl557WXFQCgBUoPAAAAGMu+/DLZvLnc53j//c75bbeV2xwLFyaT/VIfAJi4/EwIAAAAxppf/rLc5njooeTIkTKbMaO5yTFcdFx2WXs5AQDGGKUHAAAAtO3IkeThh8uzVbt2dc6vvrocIV+yJJk+vZ2cAABjnNIDAAAA2vDhh6Xk2LSpecZq2OTJyQMPlG2OG25whBwA4HtQegAAAMCZcPx48vTT5TbHs892zi+4oCk4+vqSVauS2bPbyQkAMI4pPQAAAGC0fP55snFjU3SsX5988kmZVVWyYEHZ5vjxj5OenvayAgB0AaUHAAAAjJS6Tl56qRwh37692fAYNnt2smZNU3SsW5fMn99eVgCALqT0AAAAgB/i4MFky5Zyn+OddzrnN99cjpAvWpRMmdJOTgCACUDpAQAAAKfqrbfKNsfWrcmhQ2U2fXqyfHm5z3HllW2lBACYcJQeAAAA8OscPZo89lg5Qv7yy53zyy8v2xzLliUzZrSTEwBgglN6AAAAwDfZs6c5Pt7f3xwj37u3zCZNSu67r2xz3HJLc5gcAIBWKT0AAAAgSYaGkmefLc9W7djRHCYfdt55zfHxvr5k9epk7tz2sgIA8I2UHgAAAExc+/Y1WxwDA81fH33UOf/xj8uzVXff3Wx4AAAwZik9AAAAmDjqOnn11bLN8cgjybFjZX722cmqVU3JsW5dcvHF7WUFAOCUKT0AAADobocOJdu2lSPkb7zROb/++rLNcf/9ydSprcQEAOCHO+XSo6qqB5L8oyR3JbkoyW/Udf1XJ83/PMl//rW/7cm6rheefkwAAAA4Be++2xQc/f3J5s3JwYNlNnVqsnRpU3L09ibXXttaTAAARtbpbHrMTPJ8kv82yV9+y+cMJvmtk75/5DR+HAAAAPh+jh1LnniiPFv1wgud80suaQqOvr5kxYrmGSsAALrOKZcedV2vT7I+Saqq+rZPO1zX9e4fkAsAAAC+2yefJIODTcmxYUPy+edl1tOTLFxYnq267bbk238NCwBAlxitmx5Lq6rak+SLJA8l+cd1Xe/5pk+sqmpakmknfWjWKGUCAABgPKvr5Lnnym2OJ55oPjbs3HOTtWubjY61a5N581qLCgBAO0aj9Fif5D8keTvJVUn+aZItVVXdVdf14W/4/J8m+cNRyAEAAMB49+WXyYMPlqLjgw8657fdVrY57r03mTxaf7YPAIDxYMR/NljX9f940ndfrKpqR5oCpC/Jf/yGv+WPk/zspO/PSvLeSOcCAABgnHjttXKb4+GHkyMnnYmcMSNZubIcIb/00vZyAgAw5oz6H4Gp6/rDqqreTnLdt8wPJ/nrDZDvuBMCAABANzp8uCk3hrc5Xnutc3711WWbY8mSZPr0dnICADDmjXrpUVXVvCSXJflwtH8sAAAAxokPPmgKjv7+5vmqL78ss8mTkwceKEXH9dc7Qg4AwPdyyqVHVVVnJ7n2pA9dVVXVHUk+O/HXHyX5yzQlx5VJ/nmST5L8Tz8sKgAAAOPW8ePJU0+VbY6dOzvnF17YPFfV19c8XzV7djs5AQAY105n0+PuJFtP+v7wPY6/SPI7SX6U5O8mOSdN8bE1yW/Wdb3/9GMCAAAw7nz+ebJhQ1N0DA4mn3xSZlWVLFhQtjnuvDPp6WkvKwAAXeGUS4+6rrcl+a694jWnnQYAAIDxq66TF18sR8i3b0+Ghsp8zpxkzZqm5Fi7Npk/v72sAAB0pVG/6QEAAEAXO3gw2by53Od4993O+S23lGerFi1KpkxpJycAABOC0gMAAIBT8+abZZtj69bk8OEymz49Wb68KTl6e5Mrr2wtJgAAE4/SAwAAgO929Gjy6KNlm+OVVzrnV1xRbnMsXZrMmNFKTAAAUHoAAADwqz76KFm/vik5Nm5M9u0rs0mTkvvuK0XHzTc3h8kBAKBlSg8AAACag+PPPFOerdqxo3N+/vnJunXNk1WrVydz57aTEwAAvoPSAwAAYKLau7fZ4hgYaLY6Pvqoc37XXeU2x4IFSU9POzkBAOB7UnoAAABMFHXd3OMYvs3x6KPJsWNlPmtWsmpVU3SsW5dcdFF7WQEA4DQoPQAAALrZV18l27aVZ6veeqtzfsMN5TbH4sXJ1KltpAQAgBGh9AAAAOg277zTFBwDA8nmzU3xMWzq1GTZsvJs1TXXtJcTAABGmNIDAABgvDt2LHn88bLN8eKLnfNLL20Kjr6+ZMWKZObMdnICAMAoU3oAAACMRx9/nAwONiXHhg3JF1+UWU9P8pOflGerfvSjpKpaiwoAAGeK0gMAAGA8qOtk587ybNWTTzYfG3buuc3x8b6+ZPXqZN689rICAEBLlB4AAABj1f79yYMPlqLjww8757ffXrY57r03mTSpnZwAADBGKD0AAADGkl27Ssnx0EPJ0aNlNnNmsnJlOUJ+ySXt5QQAgDFI6QEAANCmw4eThx8uR8h/+cvO+bXXlpJjyZJk2rR2cgIAwDig9AAAADjT3n+/2eTo72+erzpwoMymTEkeeKA8W3X99e3lBACAcUbpAQAAMNqOH0+eeqpsczz3XOf8oouaTY6+vub5qlmzWokJAADjndIDAABgNHz2WbJhQ1NyDA4mn35aZlWV3HNP2ea4446kp6e1qAAA0C2UHgAAACOhrpMXXijPVm3fngwNlfk55yRr1jQlx9q1yfnntxYVAAC6ldIDAADgdB04kGzZ0pQcAwPJu+92zm+9tTxbtWhRMtkvwQAAYDT5GTcAAMCpeOONcptj27bk8OEyO+usZPnypuTo7U2uuKK1mAAAMBEpPQAAAL7LkSPJY4+VouMXv+icX3llKTmWLWuKDwAAoBVKDwAAgK/bvTtZv74pOTZuTPbvL7NJk5LFi8sR8ptuag6TAwAArVN6AAAADA0lO3aUbY5nnumcz5+frFvXlByrVjVHyQEAgDFH6QEAAExMX3zRbHEMDDRbHXv2dM7vvrs8W3X33UlPTysxAQCA70/pAQAATAx1nbzyStnmePTR5PjxMp81K1m9uik61q1LLrywvawAAMBpUXoAAADd66uvkq1bm5JjYCB5663O+Y03ltsc992XTJ3aSkwAAGBkKD0AAIDu8vbbpeTYsqUpPoZNm5YsW9Y8WdXXl1x9dXs5AQCAEaf0AAAAxrdjx5Lt28uzVS+91Dm/9NKyzbF8eTJzZjs5AQCAUaf0AAAAxp+PP26Oj/f3Jxs2JHv3lllPT7JoUSk6br01qar2sgIAAGeM0gMAABj7hoaSnTubJ6v6+5OnnmoOkw+bNy9Zu7YpOdasSc49t72sAABAa5QeAADA2LR/f7JpU7nPsXt35/yOO8o2xz33JJMmtRITAAAYO5QeAADA2FDXya5dZZvj4YeTo0fLfObMZNWqpuRYty655JL2sgIAAGOS0gMAAGjP4cPJQw+VI+Svv945v/bass3xwAPJtGnt5AQAAMYFpQcAAHBmvfde2ebYvDk5cKDMpkxJlixpSo7e3uT669vLCQAAjDtKDwAAYHQdP548+WTZ5nj++c75RRc1BUdfX7JyZTJrVjs5AQCAcU/pAQAAjLzPPksGB5uSY3Cw+f6wqkruvbc8W3XHHc3HAAAAfiClBwAA8MPVdfLznzclx8BA8vjjydBQmZ9zTrJmTVNyrF2bnH9+a1EBAIDupfQAAABOz4EDzU2O4aLjvfc657feWrY5fvKTZLJffgAAAKPLrzoAAIDv7/XXy22ObduSI0fK7KyzkhUryhHyyy9vLSYAADAxKT0AAIBvd+RI8sgjZZvj1Vc751ddVUqOpUub4gMAAKAlSg8AAKDThx82BcfAQLJpU7J/f5lNnpwsXlyerbrxRkfIAQCAMUPpAQAAE93QUPL00+XZqmef7ZzPn99scvT1JatWJXPmtJMTAADg11B6AADARPTFF8mGDU3JMTiYfPxx53zBgvJs1V13JT09rcQEAAA4FUoPAACYCOo6eeml5smq/v7ksceS48fLfPbsZPXqpuhYty654IL2sgIAAJwmpQcAAHSrgweTrVvLEfK33+6c33RTuc1x333JlCnt5AQAABghSg8AAOgmb71VSo4tW5JDh8ps2rRk+fJyn+Oqq1qLCQAAMBqUHgAAMJ4dPZps316OkL/8cuf8ssvKNsfy5cmMGe3kBAAAOAOUHgAAMN7s2ZOsX9+UHBs3Jnv3ltmkScmiRaXouOWWpKraywoAAHAGKT0AAGCsGxpKdu4s2xxPP90cJh82b15zfLyvL1mzJpk7t72sAAAALVJ6AADAWLRvX7JpU1NyrF+f7N7dOb/zzrLNsWBBs+EBAAAwwSk9AABgLKjrZNeuss3xyCPNvY5hZ5+drFrVHCHv7U0uvri9rAAAAGOU0gMAANpy6FDy0EOl6Hjjjc75ddeVbY7770+mTWsnJwAAwDih9AAAgDPpvfeSgYGm5HjwweTgwTKbOjVZsqQpOXp7m9IDAACA703pAQAAo+n48eSJJ8o2x89/3jm/+OKm4OjrS1aubJ6xAgAA4LQoPQAAYKR9+mkyONhsdAwOJp99VmZVlSxcWJ6tuv325mMAAAD8YEoPAAD4oeq62eAY3uZ44olkaKjM585N1q5tSo41a5LzzmsvKwAAQBdTegAAwOn48stk8+Zmm2NgoLnVcbLbbivPVi1cmEz2U28AAIDR5ldeAADwfb3+etnm2LYtOXKkzGbMSFasKEfIL7ustZgAAAATldIDAAC+zZEjySOPlKJj167O+dVXl5Jj6dJk+vRWYgIAANBQegAAwMk+/DBZv74pOTZtSvbvL7PJk5P77y9HyG+4wRFyAACAMUTpAQDAxDY0lDz9dNnmePbZzvkFF5TbHKtWJbNnt5MTAACAX0vpAQDAxPPFF8mGDU3JMTiYfPxxmVVVsmBB2ea4886kp6e1qAAAAHx/Sg8AALpfXScvvZQMDDRFx2OPJcePl/mcOcmaNc1Gx7p1yfz57WUFAADgtCk9AADoTgcPJlu3NiXHwEDy9tud85tvLtscixYlU6a0kxMAAIARo/QAAKB7vPVWuc2xdWty6FCZTZ+eLF9e7nNceWVbKQEAABglSg8AAMavo0eT7dtL0fHyy53zyy8v2xzLliUzZrSTEwAAgDNC6QEAwPiyZ0+yfn1TcmzcmOzdW2aTJiX33deUHL29yS23NIfJAQAAmBCUHgAAjG1DQ8mzz5Ztjh07msPkw847rzk+3teXrF6dzJ3bXlYAAABapfQAAGDs2bcv2bSpHCH/6KPO+Y9/XLY5FixoNjwAAACY8JQeAAC0r66TV18t2xyPPJIcO1bmZ5+drFrVFB3r1iUXX9xeVgAAAMasUy49qqp6IMk/SnJXkouS/EZd13910rxK8odJfjvJ3CRPJvn7dV2/NBKBAQDoEocOJdu2lW2ON97onF9/fbPJ0deX3H9/Mm1aKzEBAAAYP05n02NmkueT/LdJ/vIb5r+X5B8m+XtJdiX5J0k2VVV1Q13X+08zJwAA3eDdd5uCo78/2bw5OXiwzKZOTZYuLUXHtde2FhMAAIDx6ZRLj7qu1ydZnyTNUkdxYsvjHyT5Z3Vd/8cTH/vPk3yU5O8k+bc/LC4AAOPKsWPJE0+UZ6teeKFzfvHFTcHR15esWNE8YwUAAACnaaRvelyV5MIkG4c/UNf14aqqHkqyKN9QelRVNS3JyW8VzBrhTAAAnEmffJJs2NCUHIODyeefl1lPT7JwYdnmuP325Gt/kAYAAABO10iXHhee+Pajr338oyRXfMvf89M0N0AAABiP6jp5/vmyzfHkk8nQUJnPnZusXduUHGvWJOed115WAAAAutpIlx7D6q99v/qGjw374yQ/O+n7s5K8NxqhAAAYIV9+mTz4YDlC/sEHnfPbbmtKjt7eZrNj8mj9tBMAAACKkf7V5+4T316Y5MOTPj4/v7r9kaR5/irJ4eHvf/1OCAAAY8Qvf1m2OR56KDlypMxmzGhucgwXHZdd1l5OAAAAJqyRLj3eTFN8rEqyM0mqqpqaZEmS3x/hHwsAgNF05Ejy8MNlm2PXrs751VeXI+RLliTTp7eTEwAAAE445dKjqqqzk1x70oeuqqrqjiSf1XX9TlVVf5LkD6qqei3Ja0n+IMnBJP/uh8cFAGBUffBBsn59U3Rs2tQ8YzVs8uTk/vtL0XHDDY6QAwAAMKaczqbH3Um2nvT94Xscf5Hk7yX5V0nOSvJvksxN8mSS1XVd7z/9mAAAjIrjx5Onny7PVu3c2Tm/4ILmuaq+vmTVqmT27HZyAgAAwPdQ1fW33RdvR1VVs5Ps3bt3b2b7RTUAwMj7/PNkw4am5BgcTD75pMyqKlmwoNzm+PGPk56e9rICAAAw4e3bty9z5sxJkjl1Xe/7rs8d6ZseAACMNXWdvPRS2ebYvr3Z8Bg2Z06yZk1Tcqxbl8yf315WAAAA+AGUHgAA3ejgwWTLlnKE/J13Ouc331xucyxalEyZ0k5OAAAAGEFKDwCAbvHmm6Xk2Lo1OXSozKZPT5YvL89WXXllazEBAABgtCg9AADGq6NHk0cfbUqO/v7klVc655dfXrY5li1LZsxoJycAAACcIUoPAIDx5KOPkvXrm5Jj48Zk30n32yZNSu67rxQdN9/cHCYHAACACULpAQAwlg0NJc88U7Y5nn66c37++c3x8b6+ZNWqZO7cdnICAADAGKD0AAAYa/buTTZtakqO9eub7Y6T3XVXc5ejry9ZsCDp6WknJwAAAIwxSg8AgLbVdfKLXzQlR39/c6fj2LEynzWr2eLo62u2Oi66qL2sAAAAMIYpPQAA2nDoULJ1a3m26s03O+c33NCUHL29yf33J1OntpMTAAAAxhGlBwDAmfLuu2WbY/Pm5Kuvymzq1GTp0nKE/JprWosJAAAA45XSAwBgtBw7ljz+eNnmeOGFzvkll5SSY8WKZObMdnICAABAl1B6AACMpE8+SQYHm5Jjw4bk88/LrKcnWbiwFB233ZZUVXtZAQAAoMsoPQAAfoi6Tp57rik5BgaSJ55oPjbs3HOTtWubkmPNmmTevNaiAgAAQLdTegAAnKr9+5ubHMNFxwcfdM5vv70cIV+4MJk0qZ2cAAAAMMEoPQAAvo/XXitHyB9+ODlypMxmzEhWrixFx6WXtpcTAAAAJjClBwDANzl8uCk3hrc5Xnutc37NNeU2xwMPJNOnt5MTAAAA+GtKDwCAYe+/n6xf3xQdDz6YfPllmU2Z0pQbvb1N0XH99Y6QAwAAwBij9AAAJq7jx5OnnirPVj33XOf8wgtLybFyZTJ7disxAQAAgO9H6QEATCyffZZs2NCUHIODyaeflllVJffcU56tuuOOpKentagAAADAqVF6AADdra6TF15o7nL09yfbtydDQ2U+Z06yZk1Tcqxdm8yf315WAAAA4AdRegAA3efAgWTLlnKE/N13O+e33FK2ORYtSib7KREAAAB0A7/CBwC6wxtvlNsc27Ylhw+X2fTpyYoVTcnR25tccUVrMQEAAIDRo/QAAManI0eSxx4rRccvftE5v+KKss2xbFly1lnt5AQAAADOGKUHADB+7N6drF/flBwbNyb795fZpEnJ4sWl6LjppuYwOQAAADBhKD0AgLFraCjZsaPc5tixo3N+/vnNc1W9vcnq1ck557QSEwAAABgblB4AwNiyd2+zxdHf32x17NnTOb/rrrLNcffdSU9POzkBAACAMUfpAQC0q66TV14p2xyPPpocO1bms2Y1Wxx9fcm6dcmFF7aXFQAAABjTlB4AwJn31VfJtm3lCPlbb3XOb7ihbHMsXpxMndpGSgAAAGCcUXoAAGfGO++UkmPLlqb4GDZtWrJ0aVNy9PYm11zTWkwAAABg/FJ6AACj49ix5PHHS9Hx4oud80svLSXHihXJzJnt5AQAAAC6htIDABg5H3+cDA42JceGDckXX5RZT0/yk5+UZ6t+9KOkqlqLCgAAAHQfpQcAcPrqOtm5s2xzPPVU87Fh556brF3blBxr1iTz5rWXFQAAAOh6Sg8A4NTs359s2pQMDDR/ffhh5/yOO5onq/r6knvvTSZNaiUmAAAAMPEoPQCAX2/XrrLN8fDDydGjZTZzZrJyZbnPcckl7eUEAAAAJjSlBwDwqw4fTh56qCk5BgaSX/6yc37tteU2xwMPJNOmtZMTAAAA4CRKDwCg8d575cmqBx9MDhwosylTmnJjuOi4/vr2cgIAAAB8C6UHAExUx48nTz5Znq16/vnO+UUXldscK1cms2a1kxMAAADge1J6AMBE8tlnyeBgU3IMDjbfH1ZVzeHx4W2OO+5oPgYAAAAwTig9AKCb1XXy85+X2xyPP54MDZX5Oecka9c2Gx1r1ybnn99aVAAAAIAfSukBAN3mwIFk8+ZSdLz3Xuf81lvLNsdPfpJM9tMBAAAAoDv4XQ4A6Aavv15Kjm3bksOHy+yss5IVK5qSY9265IorWosJAAAAMJqUHgAwHh05kjz6aDlC/uqrnfOrrmpKjt7eZOnSpvgAAAAA6HJKDwAYL3bvbjY5+vuTTZuS/fvLbPLkZPHi8mzVjTc6Qg4AAABMOEoPABirhoaSHTvKNsczz3TO589vNjl6e5PVq5M5c9rJCQAAADBGKD0AYCz54otk48am5Fi/Pvn448753XeXbY677kp6elqJCQAAADAWKT0AoE11nbzyStnmePTR5PjxMp89u9ni6OtL1q5NLrywvawAAAAAY5zSAwDOtK++SrZubUqOgYHkrbc65zfeWLY5Fi9OpkxpJSYAAADAeKP0AIAz4e23S8mxZUtTfAybNi1ZtqwpOXp7k6uvbi8nAAAAwDim9ACA0XD0aPL44+XZqpde6pxfemnZ5li+PJk5s52cAAAAAF1E6QEAI+Xjj5vj4/39yYYNyd69ZdbTkyxaVIqOW29Nqqq9rAAAAABdSOkBAKdraCjZubNsczz9dHOYfNi8ecm6dU3JsXp1cu657WUFAAAAmACUHgBwKvbtSx58sNzn2L27c37nneU2xz33JJMmtZMTAAAAYAJSegDAd6nrZNeuss3xyCPNvY5hM2cmq1aVouPii9vLCgAAADDBKT0A4OsOHUoeeqhsc7z+euf8uuvKbY7770+mTWsnJwAAAAAdlB4AkCTvvdcUHP39zfNVBw+W2dSpyZIlzSZHX19TegAAAAAw5ig9AJiYjh9PnniiPFv18593zi++uJQcK1cmZ5/dTk4AAAAAvjelBwATx6efJoODzUbH4GDy2WdlVlXJwoXl2arbb28+BgAAAMC4ofQAoHvVdbPBMbzN8cQTydBQmc+dm6xd22x0rF2bnHdee1kBAAAA+MGUHgB0ly+/TDZvLkfI33+/c/6jH5VtjoULk8n+VQgAAADQLfxODwDj3+uvl22ObduSI0fKbMaMZMWKpuRYty65/PLWYgIAAAAwupQeAIw/R44kjzxSio5duzrnV11VtjmWLk2mT28lJgAAAABnltIDgPHhww+b56oGBpJNm5L9+8ts8uTk/vubkqO3N7nxRkfIAQAAACYgpQcAY9PQUPL002Wb49lnO+cXXNA8V9XXl6xalcyZ005OAAAAAMYMpQcAY8cXXyQbNjTbHOvXJx9/3DlfsKA8W/XjHyc9Pa3EBAAAAGBsUnoA0J66Tl56qSk5+vuTxx5Ljh8v89mzk9WryxHyCy5oLysAAAAAY57SA4Az6+DBZOvWpuQYGEjefrtzftNN5TbH4sXJlCnt5AQAAABg3FF6ADD63nqr3ObYujU5dKjMpk1Lli8vRcdVV7UWEwAAAIDxTekBwMg7erR5qmr42aqXX+6cX3ZZuc2xfHkyY0Y7OQEAAADoKkoPAEbGnj3N8fH+/mTjxmTv3jKbNClZtKgUHbfcklRVe1kBAAAA6EpKDwBOz9BQ8uyz5dmqHTuaw+TDzjuvOT7e19ccI587t72sAAAAAEwISg8Avr99+5otjoGB5q+PPuqc//jHzV2Ovr5kwYJmwwMAAAAAzpARLz2qqvqjJH/4tQ9/VNf1hSP9YwEwyuo6efXVss3xyCPJsWNlfvbZyapVTcmxbl1y8cXtZQUAAABgwhutTY+Xkqw86fvHR+nHAWCkHTqUbNvWlBwDA8kbb3TOr7++3OZYvDiZNq2VmAAAAADwdaNVehyr63r3KP2zARhp777bFBz9/cnmzcnBg2U2dWqyZEkpOq69tr2cAAAAAPAdRqv0uK6qqg+SHE7yZJI/qOv6jW/6xKqqpiU5+Y8JzxqlTAAMO3YseeKJ8mzVCy90zi+5pNzmWLGiecYKAAAAAMa40Sg9nkzyd5PsSnJBkn+SZHtVVbfUdf3pN3z+T/OrN0AAGGmffJIMDjYlx4YNyeefl1lPT7JwYVNy9PYmt9+eVFV7WQEAAADgNFR1XY/uD1BVM5O8nuRf1XX9s2+Yf9Omx3t79+7N7NmzRzUbQFer6+T558s2x5NPJkNDZT53brJ2bVN0rF2bzJvXXlYAAAAA+Bb79u3LnDlzkmROXdf7vutzR+t5q79W1/WBqqpeSHLdt8wPp3kGK0lS+ZPFAKfvyy+TBx8sR8g/+KBzfttt5TbHvfcmk0f9XwMAAAAAcMaM+u92ndjkuCnJI6P9YwFMSK+9Vo6QP/RQcuRImc2Y0dzkGH626rLL2ssJAAAAAKNsxEuPqqr+6yT/Kck7SeanuekxO8lfjPSPBTAhHTmSPPxwebbqtdc651dfXbY5lixJpk9vJycAAAAAnGGjselxaZJ/n+S8JB8neSLJwrqu3x6FHwtgYvjgg2T9+qbk2LSpecZq2OTJyQMPlG2OG25whBwAAACACWnES4+6rv/2SP8zASac48eTp58u2xw7d3bOL7igKTj6+pJVq5LZs9vJCQAAAABjiAu2AGPF558nGzY09znWr08++aTMqipZsKBsc/z4x0lPT3tZAQAAAGAMUnoAtKWuk5deKtsc27c3Gx7D5sxJ1qxpSo5165L589vLCgAAAADjgNID4Ew6eDDZsqUpOQYGknfe6ZzffHM5Qr5oUTJlSjs5AQAAAGAcUnoAjLa33irbHFu3JocOldn06cny5eXZqiuvbCslAAAAAIx7Sg+AkXb0aPLYY2Wb4+WXO+eXX162OZYtS2bMaCcnAAAAAHQZpQfASNizpzk+3t+fbNyY7N1bZpMmJffdV4qOm29uDpMDAAAAACNK6QFwOoaGkmefLc9W7djRHCYfdv75zfHxvr5k9erknHNaiwoAAAAAE4XSA+D72rev2eIYGGj++uijzvldd5XbHAsWJD097eQEAAAAgAlK6QHwbeo6efXVss3xyCPJsWNlPmtWsmpVU3SsW5dcdFF7WQEAAAAApQdAh0OHkm3byhHyN97onN9wQ7nNsXhxMnVqKzEBAAAAgF+l9AB4992m4OjvTzZvTg4eLLOpU5OlS0vRcc01rcUEAAAAAL6b0gOYeI4dS554ojxb9cILnfNLLiklx4oVycyZ7eQEAAAAAE6J0gOYGD75JBkcbEqODRuSzz8vs56eZOHCUnTcdltSVe1lBQAAAABOi9ID6E51nTz3XLnN8cQTzceGnXtusnZtU3KsWZPMm9daVAAAAABgZCg9gO7x5ZfJgw+WouODDzrnt9/elBy9vc1mx6RJ7eQEAAAAAEaF0gMY3157rdzmePjh5MiRMpsxI1m5shQdl17aXk4AAAAAYNQpPYDx5fDhptwY3uZ47bXO+dVXl9scS5Yk06e3kxMAAAAAOOOUHsDY98EHTcHR3988X/Xll2U2eXLywAOl6Lj+ekfIAQAAAGCCUnoAY8/x48lTT5Vtjp07O+cXXtg8V9XX1zxfNXt2OzkBAAAAgDFF6QGMDZ9/nmzY0BQdg4PJJ5+UWVUlCxaUbY4770x6etrLCgAAAACMSUoPoB11nbz4YjlCvn17MjRU5nPmJGvWNCXH2rXJ/PntZQUAAAAAxgWlB3DmHDyYbN5c7nO8+27n/JZbyrNVixYlU6a0kxMAAAAAGJeUHsDoevPNss2xdWty+HCZTZ+eLF/elBy9vcmVV7YWEwAAAAAY/5QewMg6ejR59NGyzfHKK53zK64otzmWLk1mzGglJgAAAADQfZQewA/30UfJ+vVNybFxY7JvX5lNmpTcd18pOm6+uTlMDgAAAAAwwpQewKkbGkqeeaY8W7VjR+f8/POTdeuaJ6tWr07mzm0nJwAAAAAwoSg9gO9n795mi6O/v9nq2LOnc37XXeU2x4IFSU9POzkBAAAAgAlL6QF8s7pu7nEM3+Z49NHk2LEynzUrWbWqKTrWrUsuuqi9rAAAAAAAUXoAJ/vqq2TbtvJs1Vtvdc5vuKHc5li8OJk6tY2UAAAAAADfSOkBE9077zQFx8BAsnlzU3wMmzo1WbasPFt1zTXt5QQAAAAA+DWUHjDRHDuWbN9enq168cXO+aWXNgVHX1+yYkUyc2Y7OQEAAAAATpHSAyaCjz9OBgebkmPDhuSLL8qspyf5yU/Ks1U/+lFSVa1FBQAAAAA4XUoP6EZ1nezcWW5zPPVU87Fh557bHB/v60tWr07mzWsvKwAAAADACFF6QLfYvz958MFyn+PDDzvnd9xRnq26995k0qRWYgIAAAAAjBalB4xnu3aVbY6HH06OHi2zmTOTlSvLEfJLLmkvJwAAAADAGaD0gPHk8OGm3BguOn75y875NdeU2xxLliTTprWTEwAAAACgBUoPGOvef795rqq/v3m+6sCBMpsyJXnggVJ0XH99ezkBAAAAAFqm9ICx5vjx5MknS9Hx3HOd84suKrc5Vq5MZs1qJSYAAAAAwFij9ICx4LPPkg0bmpJjcDD59NMyq6rknnvKNscddyQ9Pa1FBQAAAAAYq5Qe0Ia6Tl54oWxzbN+eDA2V+TnnJGvWNCXH2rXJ+ee3FhUAAAAAYLxQesCZcuBAsmVLU3IMDCTvvts5v/XW8mzVokXJZF+eAAAAAACnwu+qwmh6442m5OjvT7ZtSw4fLrOzzkqWL29Kjt7e5IorWosJAAAAANANlB4wko4cSR57rBQdv/hF5/zKK0vJsWxZU3wAAAAAADAilB7wQ+3enaxf35QcGzcm+/eX2aRJyeLF5Qj5TTc1h8kBAAAAABhxSg84VUNDyY4dZZvjmWc65/PnJ+vWNSXHqlXNUXIAAAAAAEad0gO+jy++aLY4BgaarY49ezrnd99dnq26++6kp6eVmAAAAAAAE5nSA75JXSevvFK2OR59NDl+vMxnz05Wr25KjnXrkgsvbC8rAAAAAABJlB5QfPVVsnVrU3IMDCRvvdU5v/HGcpvjvvuSqVNbiQkAAAAAwDdTejCxvf12KTm2bGmKj2HTpiXLlpVnq66+ur2cAAAAAAD8WkoPJpZjx5Lt28uzVS+91Dm/9NKyzbF8eTJzZjs5AQAAAAA4ZUoPut/HHzfHx/v7kw0bkr17y6ynJ1m0qBQdt96aVFV7WQEAAAAAOG1KD7rP0FCyc2fzZFV/f/LUU81h8mHz5jXHx/v6mmPk557bXlYAAAAAAEaM0oPusH9/smlTuc+xe3fn/I47yjbHPfckkya1EhMAAAAAgNGj9GB8qutk166yzfHww8nRo2U+c2ayalVTcqxbl1xySXtZAQAAAAA4I5QejB+HDycPPVSOkL/+euf82mvLNscDDyTTprWTEwAAAACAVig9GNvee69sc2zenBw4UGZTpiRLlpSi47rr2ssJAAAAAEDrlB6MLcePJ08+WbY5nn++c37RRUlvb1NyrFyZzJrVTk4AAAAAAMYcpQft++yzZHCwKTkGB5vvD6uq5N57yzbHHXc0HwMAAAAAgK9RenDm1XXy8583JcfAQPL448nQUJmfc06yZk1Tcqxdm5x/fmtRAQAAAAAYP5QenBkHDjQ3OYaLjvfe65zfemvZ5vjJT5LJ/qsJAAAAAMCp8TvLjJ7XXy+3ObZtS44cKbOzzkpWrGhKjt7e5PLLW4sJAAAAAEB3UHowco4cSR55pGxzvPpq5/yqq0rJsXRpU3wAAAAAAMAIUXrww3z4YVNwDAwkmzYl+/eX2eTJyeLF5dmqG290hBwAAAAAgFGj9ODUDA0lTz9dnq169tnO+fz5zSZHX1+yalUyZ047OQEAAAAAmHCUHvx6X3yRbNjQlByDg8nHH3fOFywoz1bddVfS09NKTAAAAAAAJjalB7+qrpOXXmqerOrvTx57LDl+vMxnz05Wr26KjnXrkgsuaC8rAAAAAACcoPSgcfBgsnVrOUL+9tud85tuKrc57rsvmTKlnZwAAAAAAPAtlB4T2VtvlZJjy5bk0KEymzYtWb683Oe46qrWYgIAAAAAwPeh9JhIjh5Ntm8vR8hffrlzftllZZtj+fJkxox2cgIAAAAAwGlQenS7PXuS9eubkmPjxmTv3jKbNClZtKgUHbfcklRVe1kBAAAAAOAHUHp0m6GhZOfOss3x9NPNYfJh8+Y1x8f7+pI1a5K5c9vLCgAAAAAAI2jUSo+qqn43yT9KclGSl5L8g7quHxmtH29C27cv2bSpKTnWr0927+6c33ln2eZYsKDZ8AAAAAAAgC4zKqVHVVW/meRPkvxukseS/BdJ1ldVdXNd1++Mxo85odR1smtX2eZ45JHmXsewmTOTVauakqO3N7n44vayAgAAAADAGVLVJz99NFL/0Kp6MsmzdV3/zkkfeyXJX9V1/dNf8/fOTrJ37969mT179ohnG7cOH062bStFxxtvdM6vu65sc9x/fzJtWisxAQAAAABgJO3bty9z5sxJkjl1Xe/7rs8d8U2PqqqmJrkryb/42mhjkkXf8PnTkpz8O/SzRjpTV9i5M1m7tnx/6tRkyZJmk6Ovryk9AAAAAABgAhuN563OSzIpyUdf+/hHSS78hs//aZI/HIUc3WXBguT225tv+/qSlSuTs89uOxUAAAAAAIwZo3bIPMnX382qvuFjSfLHSX520vdnJXlvtEKNW5MmJc8913YKAAAAAAAYs0aj9PgkyfH86lbH/Pzq9kfquj6c5PDw96uqGoVIAAAAAABAt+sZ6X9gXddHkjyTZNXXRquSbB/pHw8AAAAAACAZveetfpbkv6+qakeSx5P8dpLLk/w3o/TjAQAAAAAAE9yolB51Xf+PVVXNS/JfJbkoyYtJeuu6fns0fjwAAAAAAIBRO2Re1/W/SfJvRuufDwAAAAAAcLIRv+kBAAAAAADQBqUHAAAAAADQFZQeAAAAAABAV1B6AAAAAAAAXUHpAQAAAAAAdAWlBwAAAAAA0BWUHgAAAAAAQFdQegAAAAAAAF1B6QEAAAAAAHQFpQcAAAAAANAVlB4AAAAAAEBXUHoAAAAAAABdYXLbAb7Nvn372o4AAAAAAAC07FT6gqqu61GMcuqqqrokyXtt5wAAAAAAAMaUS+u6fv+7PmEslh5VkouT7G87yxg0K00hdGn83wfONF9/0B5ff9AuX4PQHl9/0B5ff9AuX4N8k1lJPqh/Takx5p63OhH4O5uaiarpg5Ik++u69v4XnEG+/qA9vv6gXb4GoT2+/qA9vv6gXb4G+Rbf678LDpkDAAAAAABdQekBAAAAAAB0BaXH+HI4yf/hxLfAmeXrD9rj6w/a5WsQ2uPrD9rj6w/a5WuQ0zbmDpkDAAAAAACcDpseAAAAAABAV1B6AAAAAAAAXUHpAQAAAAAAdAWlBwAAAAAA0BWUHgAAAAAAQFdQeowTVVX9blVVb1ZVdaiqqmeqqrq/7UwwEVRV9dOqqp6uqmp/VVV7qqr6q6qqbmg7F0xEJ74e66qq/qTtLDARVFV1SVVV/8+qqj6tqupgVVXPVVV1V9u5YCKoqmpyVVX/pxO/Bvyqqqo3qqr6r6qq8mt4GGFVVT1QVdV/qqrqgxM/1/yffW1eVVX1RyfmX1VVta2qqltaigtd5bu+/qqqmlJV1b+squqFqqoOnPic/66qqotbjMw44SdM40BVVb+Z5E+S/LMkdyZ5JMn6qqoubzMXTBBLkvxZkoVJViWZnGRjVVUzW00FE0xVVQuS/HaSn7edBSaCqqrmJnksydEk65LcnOR/n+SLFmPBRPL7Sf43Sf7LJDcl+b0k/yjJ/7bNUNClZiZ5Ps3X2zf5vST/8MR8QZLdSTZVVTXrzMSDrvZdX38zkvw4yT898e3fSnJ9kv/vGUvHuFXVdd12Bn6NqqqeTPJsXde/c9LHXknyV3Vd/7S9ZDDxVFV1fpI9SZbUdf1w23lgIqiq6uwkzyb53ST/JMlzdV3/g1ZDQZerqupfJLmvrmvbxdCCqqr+f0k+quv6f3XSx/4yycG6rv+X7SWD7lZVVZ3kN+q6/qsT36+SfJDkT+q6/pcnPjYtyUdJfr+u63/bVlboNl//+vuWz1mQ5KkkV9R1/c6Zysb4Y9NjjKuqamqSu5Js/NpoY5JFZz4RTHhzTnz7WaspYGL5syT9dV0/2HYQmED+RpIdVVX9hxPPO+6squp/3XYomEAeTbKiqqrrk6SqqtuTLE4y0GoqmHiuSnJhTvo9mbquDyd5KH5PBtowJ0kd28f8GpPbDsCvdV6SSWn+FMHJPkrzL17gDDnxp3x+luTRuq5fbDsPTARVVf3tNKvMC9rOAhPM1Ul+J82/9/55knuS/N+qqjpc1/V/12oymBj+ZZrf2PlFVVXH0/ya8B/Xdf3v240FE87w77t80+/JXHGGs8CEVlXV9CT/Ism/q+t6X9t5GNuUHuPH198hq77hY8Do+tdJbkvzp+yAUVZV1WVJ/q9JVtd1fajtPDDB9CTZUdf1H5z4/s4TR1t/J4nSA0bfbyb5XyT5O0leSnJHkj+pquqDuq7/os1gMEH5PRloUVVVU5L8D2l+jvq7LcdhHFB6jH2fJDmeX93qmJ9f/ZMGwCipqupP0zz18UBd1++1nQcmiLvS/PvumWbRKknzJ10fqKrqv0wyra7r422Fgy73YZKXv/axV5L8z1vIAhPR/znJv6jr+n848f0Xqqq6IslPkyg94MzZfeLbC9P8u3GY35OBM+RE4fH/TvPc3HJbHnwfbnqMcXVdH0nyTJJVXxutSrL9zCeCiaVq/OskfyvNv1zfbDsTTCCbk/wozZ9uHf5rR5L/V5I7FB4wqh5LcsPXPnZ9krdbyAIT0YwkQ1/72PH4NTycaW+mKT7++vdkTtxeXRK/JwOj7qTC47okK+u6/rTlSIwTNj3Gh58l+e+rqtqR5PEkv53k8iT/TaupYGL4szTPCvzNJPurqhreutpb1/VX7cWC7lfX9f4kHfdzqqo6kORTd3Vg1P1fkmyvquoP0vxC8540Pwf97VZTwcTxn5L846qq3knzvNWdSf5hkv9Hq6mgC1VVdXaSa0/60FVVVd2R5LO6rt+pqupPkvxBVVWvJXktyR8kOZjk353prNBtvuvrL8kHSf4/aW48/mdJJp30ezKfnfiD4vCNqrr2BOF4UFXV7yb5vSQXpfkNoP9dXdcPt5sKul9VVd/2P5K/Vdf1n5/JLEBSVdW2JM/Vdf0PWo4CXa+qqv8syR+n+ZN1byb5WV3X//d2U8HEUFXVrCT/NMlvpHlG54Mk/z7J/9Fv8sDIqqpqaZKt3zD6i7qu/17VvLP6h0n+iyRzkzyZ5O/7Qzjww33X11+SP0rzc9Bvsqyu622jEoquoPQAAAAAAAC6gvdAAQAAAACArqD0AAAAAAAAuoLSAwAAAAAA6ApKDwAAAAAAoCsoPQAAAAAAgK6g9AAAAAAAALqC0gMAAAAAAOgKSg8AAAAAAKArKD0AAAAAAICuoPQAAAAAAAC6gtIDAAAAAADoCv9/1MXeuKac8eEAAAAASUVORK5CYII=\n", + "text/plain": [ + "<Figure size 2000x600 with 1 Axes>" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "do_plot(is_sin = False)" + ] +},' +# rubocop:enable Layout/LineLength + +base = '{ + "cells": [ + <<>>{ + "cell_type": "markdown", + "id": "1", + "metadata": { + "tags": [ + "hello", + "world" + ] + }, + "source": [ + "# A\n", + "\n", + "B" + ] + } + ], + "metadata": { + } +}' + +SMALL_NOTEBOOK = base.gsub('<<>>', large_cell) +LARGE_NOTEBOOK = base.gsub('<<>>', Array.new(100, large_cell).join("\n")) + +puts "Small Notebook: #{SMALL_NOTEBOOK.bytesize}" +puts "Large Notebook: #{LARGE_NOTEBOOK.bytesize}" + +def cases(benchmark_runner) + benchmark_runner.report('small_notebook') { IpynbDiff.transform(SMALL_NOTEBOOK) } + benchmark_runner.report('large_notebook') { IpynbDiff.transform(LARGE_NOTEBOOK) } +end + +Benchmark.benchmark { |x| cases(x) } +Benchmark.memory { |x| cases(x) } diff --git a/gems/ipynbdiff/spec/ipynb_diff/symbol_map_spec.rb b/gems/ipynbdiff/spec/ipynb_diff/symbol_map_spec.rb new file mode 100644 index 00000000000..94cb10772aa --- /dev/null +++ b/gems/ipynbdiff/spec/ipynb_diff/symbol_map_spec.rb @@ -0,0 +1,55 @@ +# frozen_string_literal: true + +require_relative '../test_helper' + +describe IpynbDiff::SymbolMap do + def res(*cases) + cases&.to_h || [] + end + + describe '.parse' do + subject(:symbol_map) { described_class.parse(JSON.pretty_generate(source)) } + + context 'when object has blank key' do + let(:source) { { "": { "": 5 } } } + + it { is_expected.to match_array(res([".", 2], ["..", 3])) } + end + + context 'when object is empty' do + let(:source) { {} } + + it { is_expected.to be_empty } + end + + context 'when object is empty array' do + let(:source) { [] } + + it { is_expected.to be_empty } + end + + context 'when object has inner object and number' do + let(:source) { { obj1: { obj2: 1 } } } + + it { is_expected.to match_array(res(['.obj1', 2], ['.obj1.obj2', 3])) } + end + + context 'when object has inner object and number, string and array with object' do + let(:source) { { obj1: { obj2: [123, 2, true], obj3: "hel\nlo", obj4: true, obj5: 123, obj6: 'a' } } } + + specify do + expect(symbol_map).to match_array( + res(['.obj1', 2], + ['.obj1.obj2', 3], + ['.obj1.obj2.0', 4], + ['.obj1.obj2.1', 5], + ['.obj1.obj2.2', 6], + ['.obj1.obj3', 8], + ['.obj1.obj4', 9], + ['.obj1.obj5', 10], + ['.obj1.obj6', 11]) + ) + end + end + end +end diff --git a/gems/ipynbdiff/spec/ipynb_diff/transformer_spec.rb b/gems/ipynbdiff/spec/ipynb_diff/transformer_spec.rb new file mode 100644 index 00000000000..214a8192542 --- /dev/null +++ b/gems/ipynbdiff/spec/ipynb_diff/transformer_spec.rb @@ -0,0 +1,94 @@ +# frozen_string_literal: true + +require_relative '../test_helper' + +describe IpynbDiff::Transformer do + describe '.transform' do + using RSpec::Parameterized::TableSyntax + + let!(:default_config) { { include_frontmatter: false, hide_images: false } } + + let(:test_case) { read_test_case(test_case_name) } + let(:notebook) { test_case[:input] || FROM_IPYNB } + let(:config) { {} } + + subject { described_class.new(**default_config.merge(config)).transform(notebook) } + + where(:ctx, :test_case_name, :config) do + 'renders metadata' | 'no_cells' | { include_frontmatter: true } + 'is empty for no cells, but metadata is false' | 'no_cells_no_metadata' | {} + 'adds markdown cell' | 'only_md' | {} + 'adds block with only one line of markdown' | 'single_line_md' | {} + 'adds raw block' | 'only_raw' | {} + 'code cell, but no output' | 'only_code' | {} + 'code cell, but no language' | 'only_code_no_language' | {} + 'code cell, but no kernelspec' | 'only_code_no_kernelspec' | {} + 'code cell, but no nb metadata' | 'only_code_no_metadata' | {} + 'text output' | 'text_output' | {} + 'ignores html output' | 'ignore_html_output' | {} + 'extracts png output along with text' | 'text_png_output' | {} + 'embeds svg as image' | 'svg' | {} + 'extracts latex output' | 'latex_output' | {} + 'extracts error output' | 'error_output' | {} + 'does not fetch tags if there is no cell metadata' | 'no_metadata_on_cell' | {} + 'generates :percent decorator' | 'percent_decorator' | {} + 'parses stream output' | 'stream_text' | {} + 'ignores unknown output type' | 'unknown_output_type' | {} + 'handles backslash correctly' | 'backslash_as_last_char' | {} + 'multiline png output' | 'multiline_png_output' | {} + 'hides images when option passed' | 'hide_images' | { hide_images: true } + '\n within source lines' | 'source_with_linebreak' | { hide_images: true } + end + + with_them do + it 'generates the expected markdown' do + expect(subject.as_text).to eq test_case[:expected_markdown] + end + + it 'marks the lines correctly' do + blocks = subject.blocks.map { |b| b[:source_symbol] }.join("\n") + + expect(blocks).to eq test_case[:expected_symbols] + end + end + + describe 'Source line map' do + let(:config) { { include_frontmatter: false } } + let(:test_case_name) { 'text_png_output' } + + it 'generates the correct transformed to source line map' do + line_numbers = subject.blocks.map { |b| b[:source_line] }.join("\n") + + expect(line_numbers).to eq test_case[:expected_line_numbers] + end + end + + context 'when json is invalid' do + let(:notebook) { 'a' } + + it 'raises error' do + expect { subject }.to raise_error(IpynbDiff::InvalidNotebookError) + end + end + + context 'when it does not have the cell tag' do + let(:notebook) { '{"metadata":[]}' } + + it 'raises error' do + expect { subject }.to raise_error(IpynbDiff::InvalidNotebookError) + end + end + + context 'when notebook can not be parsed' do + let(:notebook) { '{"cells":[]}' } + + before do + allow(Oj::Parser.usual).to receive(:parse).and_return(nil) + end + + it 'raises error' do + expect { subject }.to raise_error(IpynbDiff::InvalidNotebookError) + end + end + end +end diff --git a/gems/ipynbdiff/spec/ipynb_diff_spec.rb b/gems/ipynbdiff/spec/ipynb_diff_spec.rb new file mode 100644 index 00000000000..3ca092aaf5c --- /dev/null +++ b/gems/ipynbdiff/spec/ipynb_diff_spec.rb @@ -0,0 +1,127 @@ +# frozen_string_literal: true + +require_relative 'test_helper' + +describe IpynbDiff do + def diff_signs(diff) + diff.to_s(:text).scan(/.*\n/).map { |l| l[0] }.join('') + end + + describe '.diff' do + let(:from_path) { FROM_PATH } + let(:to_path) { TO_PATH } + let(:from) { File.read(from_path) } + let(:to) { File.read(to_path) } + let(:include_frontmatter) { false } + let(:hide_images) { false } + + subject { described_class.diff(from, to, include_frontmatter: include_frontmatter, hide_images: hide_images) } + + context 'if preprocessing is active' do + it { is_expected.not_to include('<td>') } + end + + context 'when to is nil' do + let(:to) { nil } + let(:from_path) { test_case_input_path('only_md') } + + it 'all lines are removals' do + expect(diff_signs(subject)).to eq('-----') + end + end + + context 'when from is nil' do + let(:from) { nil } + let(:to_path) { test_case_input_path('only_md') } + + it 'all lines are additions' do + expect(diff_signs(subject)).to eq('+++++') + end + end + + context 'when include_frontmatter is true' do + let(:include_frontmatter) { true } + + it 'shows changes metadata in the metadata' do + expect(subject.to_s(:text)).to include('+ display_name: New Python 3 (ipykernel)') + end + end + + context 'when hide_images is true' do + let(:hide_images) { true } + + it 'hides images' do + expect(subject.to_s(:text)).to include(' [Hidden Image Output]') + end + end + + context 'when include_frontmatter is false' do + it 'drops metadata from the diff' do + expect(subject.to_s(:text)).not_to include('+ display_name: New Python 3 (ipykernel)') + end + end + + context 'when either notebook can not be processed' do + using RSpec::Parameterized::TableSyntax + + where(:ctx, :from, :to) do + 'because from is invalid' | 'a' | nil + 'because from does not have the cell tag' | '{"metadata":[]}' | nil + 'because to is invalid' | nil | 'a' + 'because to does not have the cell tag' | nil | '{"metadata":[]}' + end + + with_them do + it { is_expected.to be_nil } + end + end + end + + describe '.transform' do + let(:notebook) { FROM_IPYNB } + let(:include_frontmatter) { false } + let(:hide_images) { false } + + subject do + described_class.transform(notebook, + include_frontmatter: include_frontmatter, + hide_images: hide_images) + end + + describe 'error cases' do + using RSpec::Parameterized::TableSyntax + + where(:ctx, :notebook) do + 'notebook is nil' | nil + 'notebook is invalid' | 'a' + 'notebook does not have cell' | '{"metadata":[]}' + end + + with_them do + it { is_expected.to be_nil } + end + end + + describe 'options' do + context 'when include_frontmatter is false' do + it { is_expected.not_to include('display_name: Python 3 (ipykernel)') } + end + + context 'when include_frontmatter is true' do + let(:include_frontmatter) { true } + + it { is_expected.to include('display_name: Python 3 (ipykernel)') } + end + + context 'when hide_images is false' do + it { is_expected.not_to include('[Hidden Image Output]') } + end + + context 'when hide_images is true' do + let(:hide_images) { true } + + it { is_expected.to include(' [Hidden Image Output]') } + end + end + end +end diff --git a/gems/ipynbdiff/spec/test_helper.rb b/gems/ipynbdiff/spec/test_helper.rb new file mode 100644 index 00000000000..626b72b99f0 --- /dev/null +++ b/gems/ipynbdiff/spec/test_helper.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +require 'simplecov' +SimpleCov.start + +require 'ipynb_diff' +require 'rspec' +require 'rspec-parameterized' +require 'json' + +BASE_PATH = File.join(__dir__ || '', 'testdata') + +FROM_PATH = File.join(BASE_PATH, 'from.ipynb') +TO_PATH = File.join(BASE_PATH, 'to.ipynb') + +FROM_IPYNB = File.read(FROM_PATH) +TO_IPYNB = File.read(TO_PATH) + +def test_case_input_path(test_case) + File.join(BASE_PATH, test_case, 'input.ipynb') +end + +def test_case_symbols_path(test_case) + File.join(BASE_PATH, test_case, 'expected_symbols.txt') +end + +def test_case_md_path(test_case) + File.join(BASE_PATH, test_case, 'expected.md') +end + +def test_case_line_numbers_path(test_case) + File.join(BASE_PATH, test_case, 'expected_line_numbers.txt') +end + +def read_file_if_exists(path) + File.read(path) if File.file?(path) +end + +def read_test_case(test_case_name) + { + input: read_file_if_exists(test_case_input_path(test_case_name)), + expected_markdown: read_file_if_exists(test_case_md_path(test_case_name)), + expected_symbols: read_file_if_exists(test_case_symbols_path(test_case_name)), + expected_line_numbers: read_file_if_exists(test_case_line_numbers_path(test_case_name)) + } +end diff --git a/gems/ipynbdiff/spec/testdata/backslash_as_last_char/expected.md b/gems/ipynbdiff/spec/testdata/backslash_as_last_char/expected.md new file mode 100644 index 00000000000..299e286c679 --- /dev/null +++ b/gems/ipynbdiff/spec/testdata/backslash_as_last_char/expected.md @@ -0,0 +1,7 @@ +%% Cell type:markdown id: tags: + +\ + +%% Cell type:markdown id: tags: + +a diff --git a/gems/ipynbdiff/spec/testdata/backslash_as_last_char/expected_symbols.txt b/gems/ipynbdiff/spec/testdata/backslash_as_last_char/expected_symbols.txt new file mode 100644 index 00000000000..6fa29ae28de --- /dev/null +++ b/gems/ipynbdiff/spec/testdata/backslash_as_last_char/expected_symbols.txt @@ -0,0 +1,7 @@ +.cells.0 + +.cells.0.source.0 + +.cells.1 + +.cells.1.source.0 diff --git a/gems/ipynbdiff/spec/testdata/backslash_as_last_char/input.ipynb b/gems/ipynbdiff/spec/testdata/backslash_as_last_char/input.ipynb new file mode 100644 index 00000000000..0714044e3ae --- /dev/null +++ b/gems/ipynbdiff/spec/testdata/backslash_as_last_char/input.ipynb @@ -0,0 +1,16 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "source": [ + "\\" + ] + }, + { + "cell_type": "markdown", + "source": [ + "a" + ] + } + ] +} diff --git a/gems/ipynbdiff/spec/testdata/error_output/expected.md b/gems/ipynbdiff/spec/testdata/error_output/expected.md new file mode 100644 index 00000000000..e6e8a075598 --- /dev/null +++ b/gems/ipynbdiff/spec/testdata/error_output/expected.md @@ -0,0 +1,16 @@ +%% Cell type:code id:5 tags: + +``` python +# A cell that has an error +y = sin(x) +``` + +%% Output + + --------------------------------------------------------------------------- + NameError Traceback (most recent call last) + /var/folders/cq/l637k4x13gx6y9p_gfs4c_gc0000gn/T/ipykernel_72857/3962062127.py in <module> + 1 # A cell that has an error + ----> 2 y = sin(x) + + NameError: name 'sin' is not defined diff --git a/gems/ipynbdiff/spec/testdata/error_output/expected_symbols.txt b/gems/ipynbdiff/spec/testdata/error_output/expected_symbols.txt new file mode 100644 index 00000000000..5d2f248135d --- /dev/null +++ b/gems/ipynbdiff/spec/testdata/error_output/expected_symbols.txt @@ -0,0 +1,16 @@ +.cells.0 + +.cells.0.source +.cells.0.source.0 +.cells.0.source.1 + + +.cells.0.outputs + +.cells.0.outputs.0.traceback.0 +.cells.0.outputs.0.traceback.1 +.cells.0.outputs.0.traceback.2 +.cells.0.outputs.0.traceback.2 +.cells.0.outputs.0.traceback.2 +.cells.0.outputs.0.traceback.2 +.cells.0.outputs.0.traceback.3 diff --git a/gems/ipynbdiff/spec/testdata/error_output/input.ipynb b/gems/ipynbdiff/spec/testdata/error_output/input.ipynb new file mode 100644 index 00000000000..45ee81a0e2d --- /dev/null +++ b/gems/ipynbdiff/spec/testdata/error_output/input.ipynb @@ -0,0 +1,32 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 7, + "id": "5", + "metadata": {}, + "outputs": [ + { + "ename": "NameError", + "evalue": "name 'sin' is not defined", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m/var/folders/cq/l637k4x13gx6y9p_gfs4c_gc0000gn/T/ipykernel_72857/3962062127.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[0;31m# A cell that has an error\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0my\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", + "\u001b[0;31mNameError\u001b[0m: name 'sin' is not defined" + ] + } + ], + "source": [ + "# A cell that has an error\n", + "y = sin(x)" + ] + } + ], + "metadata": { + "kernelspec": { + "language": "python" + } + } +} diff --git a/gems/ipynbdiff/spec/testdata/from.ipynb b/gems/ipynbdiff/spec/testdata/from.ipynb new file mode 100644 index 00000000000..68a4b11cbbc --- /dev/null +++ b/gems/ipynbdiff/spec/testdata/from.ipynb @@ -0,0 +1,197 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "0aac5da7-745c-4eda-847a-3d0d07a1bb9b", + "metadata": { + "tags": [] + }, + "source": [ + "# This is a markdown cell\n", + "\n", + "This paragraph has\n", + "With\n", + "Many\n", + "Lines. How we will he handle MR notes?\n", + "\n", + "But I can add another paragraph" + ] + }, + { + "cell_type": "raw", + "id": "faecea5b-de0a-49fa-9a3a-61c2add652da", + "metadata": {}, + "source": [ + "This is a raw cell\n", + "With\n", + "Multiple lines" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "893ca2c0-ab75-4276-9dad-be1c40e16e8a", + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import numpy as np\n", + "import matplotlib.pyplot as plt" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "0d707fb5-226f-46d6-80bd-489ebfb8905c", + "metadata": {}, + "outputs": [], + "source": [ + "np.random.seed(42)" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "35467fcf-28b1-4c7b-bb09-4cb192c35293", + "metadata": { + "tags": [ + "senoid" + ] + }, "outputs": [ + { + "data": { + "text/plain": [ + "[<matplotlib.lines.Line2D at 0x123e39370>]" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "some_invalid_base64_image_here\n", + "text/plain": [ + "<Figure size 432x288 with 1 Axes>" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "x = np.linspace(0, 4*np.pi,50)\n", + "y = np.sin(x)\n", + "\n", + "plt.plot(x, y)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "dc1178cd-c46d-4da3-9ab5-08f000699884", + "metadata": {}, + "outputs": [], + "source": [ + "df = pd.DataFrame({\"x\": x, \"y\": y})" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "6e749b4f-b409-4700-870f-f68c39462490", + "metadata": { + "tags": [ + "some-table" + ] + }, + "outputs": [ + { + "data": { + "text/html": [ + "<div>\n", + "<style scoped>\n", + " .dataframe tbody tr th:only-of-type {\n", + " vertical-align: middle;\n", + " }\n", + "\n", + " .dataframe tbody tr th {\n", + " vertical-align: top;\n", + " }\n", + "\n", + " .dataframe thead th {\n", + " text-align: right;\n", + " }\n", + "</style>\n", + "<table border=\"1\" class=\"dataframe\">\n", + " <thead>\n", + " <tr style=\"text-align: right;\">\n", + " <th></th>\n", + " <th>x</th>\n", + " <th>y</th>\n", + " </tr>\n", + " </thead>\n", + " <tbody>\n", + " <tr>\n", + " <th>0</th>\n", + " <td>0.000000</td>\n", + " <td>0.000000</td>\n", + " </tr>\n", + " <tr>\n", + " <th>1</th>\n", + " <td>0.256457</td>\n", + " <td>0.253655</td>\n", + " </tr>\n", + " </tbody>\n", + "</table>\n", + "</div>" + ], + "text/plain": [ + " x y\n", + "0 0.000000 0.000000\n", + "1 0.256457 0.253655" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df[:2]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0ddef5ef-94a3-4afd-9c70-ddee9694f512", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.7" + }, + "toc-showtags": true + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/gems/ipynbdiff/spec/testdata/hide_images/expected.md b/gems/ipynbdiff/spec/testdata/hide_images/expected.md new file mode 100644 index 00000000000..ff63d351a3b --- /dev/null +++ b/gems/ipynbdiff/spec/testdata/hide_images/expected.md @@ -0,0 +1,10 @@ +%% Cell type:code id:5 tags:senoid + +``` python +``` + +%% Output + + [Hidden Image Output] + + [Hidden Image Output] diff --git a/gems/ipynbdiff/spec/testdata/hide_images/expected_symbols.txt b/gems/ipynbdiff/spec/testdata/hide_images/expected_symbols.txt new file mode 100644 index 00000000000..b8f24f9fba5 --- /dev/null +++ b/gems/ipynbdiff/spec/testdata/hide_images/expected_symbols.txt @@ -0,0 +1,10 @@ +.cells.0 + +.cells.0.source + + +.cells.0.outputs + + + + diff --git a/gems/ipynbdiff/spec/testdata/hide_images/input.ipynb b/gems/ipynbdiff/spec/testdata/hide_images/input.ipynb new file mode 100644 index 00000000000..dab0e5bb9cf --- /dev/null +++ b/gems/ipynbdiff/spec/testdata/hide_images/input.ipynb @@ -0,0 +1,45 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 3, + "id": "5", + "metadata": { + "tags": [ + "senoid" + ] + }, + "outputs": [ + { + "data": { + "image/png": "this_is_an_invalid_hash_for_testing_purposes\n", + "text/plain": [ + "<Figure size 432x288 with 1 Axes>" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + }, + { + "data": { + "image/svg+xml": "<svg xmlns=\"http://www.w3.org/2000/svg\" viewBox=\"0 0 100 100\"><circle cx=\"50\" cy=\"50\" r=\"50\"/></svg>", + "text/plain": [ + "<IPython.core.display.SVG object>" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + ] + } + ], + "metadata": { + "kernelspec": { + "language": "python" + } + } +} diff --git a/gems/ipynbdiff/spec/testdata/ignore_html_output/expected.md b/gems/ipynbdiff/spec/testdata/ignore_html_output/expected.md new file mode 100644 index 00000000000..3085da739ed --- /dev/null +++ b/gems/ipynbdiff/spec/testdata/ignore_html_output/expected.md @@ -0,0 +1,11 @@ +%% Cell type:code id:5 tags:some-table + +``` python +df[:2] +``` + +%% Output + + x y + 0 0.000000 0.000000 + 1 0.256457 0.507309 diff --git a/gems/ipynbdiff/spec/testdata/ignore_html_output/expected_symbols.txt b/gems/ipynbdiff/spec/testdata/ignore_html_output/expected_symbols.txt new file mode 100644 index 00000000000..3bf319d1fa6 --- /dev/null +++ b/gems/ipynbdiff/spec/testdata/ignore_html_output/expected_symbols.txt @@ -0,0 +1,11 @@ +.cells.0 + +.cells.0.source +.cells.0.source.0 + + +.cells.0.outputs + +.cells.0.outputs.0.data.text/plain.0 +.cells.0.outputs.0.data.text/plain.1 +.cells.0.outputs.0.data.text/plain.2 diff --git a/gems/ipynbdiff/spec/testdata/ignore_html_output/input.ipynb b/gems/ipynbdiff/spec/testdata/ignore_html_output/input.ipynb new file mode 100644 index 00000000000..26117a78934 --- /dev/null +++ b/gems/ipynbdiff/spec/testdata/ignore_html_output/input.ipynb @@ -0,0 +1,74 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 5, + "id": "5", + "metadata": { + "tags": [ + "some-table" + ] + }, + "outputs": [ + { + "data": { + "text/html": [ + "<div>\n", + "<style scoped>\n", + " .dataframe tbody tr th:only-of-type {\n", + " vertical-align: middle;\n", + " }\n", + "\n", + " .dataframe tbody tr th {\n", + " vertical-align: top;\n", + " }\n", + "\n", + " .dataframe thead th {\n", + " text-align: right;\n", + " }\n", + "</style>\n", + "<table border=\"1\" class=\"dataframe\">\n", + " <thead>\n", + " <tr style=\"text-align: right;\">\n", + " <th></th>\n", + " <th>x</th>\n", + " <th>y</th>\n", + " </tr>\n", + " </thead>\n", + " <tbody>\n", + " <tr>\n", + " <th>0</th>\n", + " <td>0.000000</td>\n", + " <td>0.000000</td>\n", + " </tr>\n", + " <tr>\n", + " <th>1</th>\n", + " <td>0.256457</td>\n", + " <td>0.507309</td>\n", + " </tr>\n", + " </tbody>\n", + "</table>\n", + "</div>" + ], + "text/plain": [ + " x y\n", + "0 0.000000 0.000000\n", + "1 0.256457 0.507309" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df[:2]" + ] + } + ], + "metadata": { + "kernelspec": { + "language": "python" + } + } +} diff --git a/gems/ipynbdiff/spec/testdata/latex_output/expected.md b/gems/ipynbdiff/spec/testdata/latex_output/expected.md new file mode 100644 index 00000000000..194c1f43c42 --- /dev/null +++ b/gems/ipynbdiff/spec/testdata/latex_output/expected.md @@ -0,0 +1,10 @@ +%% Cell type:code id:5 tags: + +``` python +from IPython.display import display, Math +display(Math(r'Dims: {}x{}m \\ Area: {}m^2 \\ Volume: {}m^3'.format(1, round(2,2), 3, 4))) +``` + +%% Output + + $\displaystyle Dims: 1x2m \\ Area: 3m^2 \\ Volume: 4m^3$ diff --git a/gems/ipynbdiff/spec/testdata/latex_output/expected_symbols.txt b/gems/ipynbdiff/spec/testdata/latex_output/expected_symbols.txt new file mode 100644 index 00000000000..868adca2712 --- /dev/null +++ b/gems/ipynbdiff/spec/testdata/latex_output/expected_symbols.txt @@ -0,0 +1,10 @@ +.cells.0 + +.cells.0.source +.cells.0.source.0 +.cells.0.source.1 + + +.cells.0.outputs + +.cells.0.outputs.0.data.text/latex.0 diff --git a/gems/ipynbdiff/spec/testdata/latex_output/input.ipynb b/gems/ipynbdiff/spec/testdata/latex_output/input.ipynb new file mode 100644 index 00000000000..f8ff3e72beb --- /dev/null +++ b/gems/ipynbdiff/spec/testdata/latex_output/input.ipynb @@ -0,0 +1,34 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 4, + "id": "5", + "outputs": [ + { + "data": { + "text/latex": [ + "$\\displaystyle Dims: 1x2m \\\\ Area: 3m^2 \\\\ Volume: 4m^3$" + ], + "text/plain": [ + "<IPython.core.display.Math object>" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from IPython.display import display, Math\n", + "display(Math(r'Dims: {}x{}m \\\\ Area: {}m^2 \\\\ Volume: {}m^3'.format(1, round(2,2), 3, 4)))" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + } + } +} diff --git a/gems/ipynbdiff/spec/testdata/multiline_png_output/expected.md b/gems/ipynbdiff/spec/testdata/multiline_png_output/expected.md new file mode 100644 index 00000000000..0a69c8370e7 --- /dev/null +++ b/gems/ipynbdiff/spec/testdata/multiline_png_output/expected.md @@ -0,0 +1,9 @@ +%% Cell type:code id:5 tags: + +``` +Some Image +``` + +%% Output + +  diff --git a/gems/ipynbdiff/spec/testdata/multiline_png_output/expected_symbols.txt b/gems/ipynbdiff/spec/testdata/multiline_png_output/expected_symbols.txt new file mode 100644 index 00000000000..1b66012ef20 --- /dev/null +++ b/gems/ipynbdiff/spec/testdata/multiline_png_output/expected_symbols.txt @@ -0,0 +1,9 @@ +.cells.0 + +.cells.0.source +.cells.0.source.0 + + +.cells.0.outputs + +.cells.0.outputs.0.data.image/png diff --git a/gems/ipynbdiff/spec/testdata/multiline_png_output/input.ipynb b/gems/ipynbdiff/spec/testdata/multiline_png_output/input.ipynb new file mode 100644 index 00000000000..4d19a504553 --- /dev/null +++ b/gems/ipynbdiff/spec/testdata/multiline_png_output/input.ipynb @@ -0,0 +1,25 @@ +{ + "cells": [ + { + "cell_type": "code", + "id": "5", + "metadata": { + }, + "outputs": [ + { + "data": { + "image/png": [ + "this_is_an_invalid_hash_for_testing_purposes" + ] + }, + "output_type": "display_data" + } + ], + "source": [ + "Some Image" + ] + } + ], + "metadata": { + } +} diff --git a/gems/ipynbdiff/spec/testdata/no_cells/expected.md b/gems/ipynbdiff/spec/testdata/no_cells/expected.md new file mode 100644 index 00000000000..b7c09c51fb8 --- /dev/null +++ b/gems/ipynbdiff/spec/testdata/no_cells/expected.md @@ -0,0 +1,19 @@ +--- +jupyter: + kernelspec: + display_name: Python 3 (ipykernel) + language: python + name: python3 + language_info: + codemirror_mode: + name: ipython + version: 3 + file_extension: ".py" + mimetype: text/x-python + name: python + nbconvert_exporter: python + pygments_lexer: ipython3 + version: 3.9.7 + nbformat: 4 + nbformat_minor: 5 +--- diff --git a/gems/ipynbdiff/spec/testdata/no_cells/expected_symbols.txt b/gems/ipynbdiff/spec/testdata/no_cells/expected_symbols.txt new file mode 100644 index 00000000000..a60f3032882 --- /dev/null +++ b/gems/ipynbdiff/spec/testdata/no_cells/expected_symbols.txt @@ -0,0 +1,19 @@ + + + + + + + + + + + + + + + + + + + diff --git a/gems/ipynbdiff/spec/testdata/no_cells/input.ipynb b/gems/ipynbdiff/spec/testdata/no_cells/input.ipynb new file mode 100644 index 00000000000..c2ba0ebf50a --- /dev/null +++ b/gems/ipynbdiff/spec/testdata/no_cells/input.ipynb @@ -0,0 +1,25 @@ +{ + "cells": [], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.7" + }, + "toc-showtags": true + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/gems/ipynbdiff/spec/testdata/no_cells_no_metadata/expected.md b/gems/ipynbdiff/spec/testdata/no_cells_no_metadata/expected.md new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/gems/ipynbdiff/spec/testdata/no_cells_no_metadata/expected.md diff --git a/gems/ipynbdiff/spec/testdata/no_cells_no_metadata/expected_symbols.txt b/gems/ipynbdiff/spec/testdata/no_cells_no_metadata/expected_symbols.txt new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/gems/ipynbdiff/spec/testdata/no_cells_no_metadata/expected_symbols.txt diff --git a/gems/ipynbdiff/spec/testdata/no_cells_no_metadata/input.ipynb b/gems/ipynbdiff/spec/testdata/no_cells_no_metadata/input.ipynb new file mode 100644 index 00000000000..c2ba0ebf50a --- /dev/null +++ b/gems/ipynbdiff/spec/testdata/no_cells_no_metadata/input.ipynb @@ -0,0 +1,25 @@ +{ + "cells": [], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.7" + }, + "toc-showtags": true + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/gems/ipynbdiff/spec/testdata/no_metadata_on_cell/expected.md b/gems/ipynbdiff/spec/testdata/no_metadata_on_cell/expected.md new file mode 100644 index 00000000000..d9d72bf8f76 --- /dev/null +++ b/gems/ipynbdiff/spec/testdata/no_metadata_on_cell/expected.md @@ -0,0 +1,13 @@ +%% Cell type:markdown id:1 tags: + +# A + +B + +%% Cell type:code id:3 tags: + +``` python +import pandas as pd +import numpy as np +import matplotlib.pyplot as plt +``` diff --git a/gems/ipynbdiff/spec/testdata/no_metadata_on_cell/expected_symbols.txt b/gems/ipynbdiff/spec/testdata/no_metadata_on_cell/expected_symbols.txt new file mode 100644 index 00000000000..a7000494a1b --- /dev/null +++ b/gems/ipynbdiff/spec/testdata/no_metadata_on_cell/expected_symbols.txt @@ -0,0 +1,13 @@ +.cells.0 + +.cells.0.source.0 +.cells.0.source.1 +.cells.0.source.2 + +.cells.1 + +.cells.1.source +.cells.1.source.0 +.cells.1.source.1 +.cells.1.source.2 + diff --git a/gems/ipynbdiff/spec/testdata/no_metadata_on_cell/input.ipynb b/gems/ipynbdiff/spec/testdata/no_metadata_on_cell/input.ipynb new file mode 100644 index 00000000000..62060124a2a --- /dev/null +++ b/gems/ipynbdiff/spec/testdata/no_metadata_on_cell/input.ipynb @@ -0,0 +1,29 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "1", + "source": [ + "# A\n", + "\n", + "B" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "3", + "outputs": [], + "source": [ + "import pandas as pd\n", + "import numpy as np\n", + "import matplotlib.pyplot as plt" + ] + } + ], + "metadata": { + "kernelspec": { + "language": "python" + } + } +} diff --git a/gems/ipynbdiff/spec/testdata/only_code/expected.md b/gems/ipynbdiff/spec/testdata/only_code/expected.md new file mode 100644 index 00000000000..124b8217a6a --- /dev/null +++ b/gems/ipynbdiff/spec/testdata/only_code/expected.md @@ -0,0 +1,7 @@ +%% Cell type:code id:3 tags: + +``` python +import pandas as pd +import numpy as np +import matplotlib.pyplot as plt +``` diff --git a/gems/ipynbdiff/spec/testdata/only_code/expected_symbols.txt b/gems/ipynbdiff/spec/testdata/only_code/expected_symbols.txt new file mode 100644 index 00000000000..59b11103195 --- /dev/null +++ b/gems/ipynbdiff/spec/testdata/only_code/expected_symbols.txt @@ -0,0 +1,7 @@ +.cells.0 + +.cells.0.source +.cells.0.source.0 +.cells.0.source.1 +.cells.0.source.2 + diff --git a/gems/ipynbdiff/spec/testdata/only_code/input.ipynb b/gems/ipynbdiff/spec/testdata/only_code/input.ipynb new file mode 100644 index 00000000000..a93108dccb8 --- /dev/null +++ b/gems/ipynbdiff/spec/testdata/only_code/input.ipynb @@ -0,0 +1,21 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "3", + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import numpy as np\n", + "import matplotlib.pyplot as plt" + ] + } + ], + "metadata": { + "kernelspec": { + "language": "python" + } + } +} diff --git a/gems/ipynbdiff/spec/testdata/only_code_no_kernelspec/expected.md b/gems/ipynbdiff/spec/testdata/only_code_no_kernelspec/expected.md new file mode 100644 index 00000000000..c6d8e13fc3a --- /dev/null +++ b/gems/ipynbdiff/spec/testdata/only_code_no_kernelspec/expected.md @@ -0,0 +1,4 @@ +%% Cell type:code id:3 tags: + +``` +``` diff --git a/gems/ipynbdiff/spec/testdata/only_code_no_kernelspec/expected_symbols.txt b/gems/ipynbdiff/spec/testdata/only_code_no_kernelspec/expected_symbols.txt new file mode 100644 index 00000000000..2e902582e14 --- /dev/null +++ b/gems/ipynbdiff/spec/testdata/only_code_no_kernelspec/expected_symbols.txt @@ -0,0 +1,4 @@ +.cells.0 + +.cells.0.source + diff --git a/gems/ipynbdiff/spec/testdata/only_code_no_kernelspec/input.ipynb b/gems/ipynbdiff/spec/testdata/only_code_no_kernelspec/input.ipynb new file mode 100644 index 00000000000..c3ff71057a6 --- /dev/null +++ b/gems/ipynbdiff/spec/testdata/only_code_no_kernelspec/input.ipynb @@ -0,0 +1,12 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "3", + "source": "", + "outputs": [] + } + ], + "metadata": {} +} diff --git a/gems/ipynbdiff/spec/testdata/only_code_no_language/expected.md b/gems/ipynbdiff/spec/testdata/only_code_no_language/expected.md new file mode 100644 index 00000000000..c6d8e13fc3a --- /dev/null +++ b/gems/ipynbdiff/spec/testdata/only_code_no_language/expected.md @@ -0,0 +1,4 @@ +%% Cell type:code id:3 tags: + +``` +``` diff --git a/gems/ipynbdiff/spec/testdata/only_code_no_language/expected_symbols.txt b/gems/ipynbdiff/spec/testdata/only_code_no_language/expected_symbols.txt new file mode 100644 index 00000000000..2e902582e14 --- /dev/null +++ b/gems/ipynbdiff/spec/testdata/only_code_no_language/expected_symbols.txt @@ -0,0 +1,4 @@ +.cells.0 + +.cells.0.source + diff --git a/gems/ipynbdiff/spec/testdata/only_code_no_language/input.ipynb b/gems/ipynbdiff/spec/testdata/only_code_no_language/input.ipynb new file mode 100644 index 00000000000..fb16b106cbe --- /dev/null +++ b/gems/ipynbdiff/spec/testdata/only_code_no_language/input.ipynb @@ -0,0 +1,14 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "3", + "source": "", + "outputs": [] + } + ], + "metadata": { + "kernelspec": {} + } +} diff --git a/gems/ipynbdiff/spec/testdata/only_code_no_metadata/expected.md b/gems/ipynbdiff/spec/testdata/only_code_no_metadata/expected.md new file mode 100644 index 00000000000..c6d8e13fc3a --- /dev/null +++ b/gems/ipynbdiff/spec/testdata/only_code_no_metadata/expected.md @@ -0,0 +1,4 @@ +%% Cell type:code id:3 tags: + +``` +``` diff --git a/gems/ipynbdiff/spec/testdata/only_code_no_metadata/expected_symbols.txt b/gems/ipynbdiff/spec/testdata/only_code_no_metadata/expected_symbols.txt new file mode 100644 index 00000000000..2e902582e14 --- /dev/null +++ b/gems/ipynbdiff/spec/testdata/only_code_no_metadata/expected_symbols.txt @@ -0,0 +1,4 @@ +.cells.0 + +.cells.0.source + diff --git a/gems/ipynbdiff/spec/testdata/only_code_no_metadata/input.ipynb b/gems/ipynbdiff/spec/testdata/only_code_no_metadata/input.ipynb new file mode 100644 index 00000000000..364c080168b --- /dev/null +++ b/gems/ipynbdiff/spec/testdata/only_code_no_metadata/input.ipynb @@ -0,0 +1,11 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "3", + "source": "", + "outputs": [] + } + ] +} diff --git a/gems/ipynbdiff/spec/testdata/only_md/expected.md b/gems/ipynbdiff/spec/testdata/only_md/expected.md new file mode 100644 index 00000000000..bdf4db5aea5 --- /dev/null +++ b/gems/ipynbdiff/spec/testdata/only_md/expected.md @@ -0,0 +1,5 @@ +%% Cell type:markdown id:1 tags:hello,world + +# A + +B diff --git a/gems/ipynbdiff/spec/testdata/only_md/expected_symbols.txt b/gems/ipynbdiff/spec/testdata/only_md/expected_symbols.txt new file mode 100644 index 00000000000..d3d6d526fc3 --- /dev/null +++ b/gems/ipynbdiff/spec/testdata/only_md/expected_symbols.txt @@ -0,0 +1,5 @@ +.cells.0 + +.cells.0.source.0 +.cells.0.source.1 +.cells.0.source.2 diff --git a/gems/ipynbdiff/spec/testdata/only_md/input.ipynb b/gems/ipynbdiff/spec/testdata/only_md/input.ipynb new file mode 100644 index 00000000000..9d6b550af31 --- /dev/null +++ b/gems/ipynbdiff/spec/testdata/only_md/input.ipynb @@ -0,0 +1,21 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "1", + "metadata": { + "tags": [ + "hello", + "world" + ] + }, + "source": [ + "# A\n", + "\n", + "B" + ] + } + ], + "metadata": { + } +} diff --git a/gems/ipynbdiff/spec/testdata/only_raw/expected.md b/gems/ipynbdiff/spec/testdata/only_raw/expected.md new file mode 100644 index 00000000000..91c476e843b --- /dev/null +++ b/gems/ipynbdiff/spec/testdata/only_raw/expected.md @@ -0,0 +1,4 @@ +%% Cell type:raw id:2 tags: + +A +B diff --git a/gems/ipynbdiff/spec/testdata/only_raw/expected_symbols.txt b/gems/ipynbdiff/spec/testdata/only_raw/expected_symbols.txt new file mode 100644 index 00000000000..bceaf355c2f --- /dev/null +++ b/gems/ipynbdiff/spec/testdata/only_raw/expected_symbols.txt @@ -0,0 +1,4 @@ +.cells.0 + +.cells.0.source.0 +.cells.0.source.1 diff --git a/gems/ipynbdiff/spec/testdata/only_raw/input.ipynb b/gems/ipynbdiff/spec/testdata/only_raw/input.ipynb new file mode 100644 index 00000000000..750e1bba615 --- /dev/null +++ b/gems/ipynbdiff/spec/testdata/only_raw/input.ipynb @@ -0,0 +1,15 @@ +{ + "cells": [ + { + "cell_type": "raw", + "id": "2", + "metadata": {}, + "source": [ + "A\n", + "B" + ] + } + ], + "metadata": { + } +} diff --git a/gems/ipynbdiff/spec/testdata/percent_decorator/expected.md b/gems/ipynbdiff/spec/testdata/percent_decorator/expected.md new file mode 100644 index 00000000000..1ece1f2fd06 --- /dev/null +++ b/gems/ipynbdiff/spec/testdata/percent_decorator/expected.md @@ -0,0 +1,68 @@ +%% Cell type:markdown id:0aac5da7-745c-4eda-847a-3d0d07a1bb9b tags: + +# This is a markdown cell + +This paragraph has +With +Many +Lines. How we will he handle MR notes? + +But I can add another paragraph + +%% Cell type:raw id:faecea5b-de0a-49fa-9a3a-61c2add652da tags: + +This is a raw cell +With +Multiple lines + +%% Cell type:code id:893ca2c0-ab75-4276-9dad-be1c40e16e8a tags: + +``` python +import pandas as pd +import numpy as np +import matplotlib.pyplot as plt +``` + +%% Cell type:code id:0d707fb5-226f-46d6-80bd-489ebfb8905c tags: + +``` python +np.random.seed(42) +``` + +%% Cell type:code id:35467fcf-28b1-4c7b-bb09-4cb192c35293 tags:senoid + +``` python +x = np.linspace(0, 4*np.pi,50) +y = np.sin(x) + +plt.plot(x, y) +``` + +%% Output + + [<matplotlib.lines.Line2D at 0x123e39370>] + +  + +%% Cell type:code id:dc1178cd-c46d-4da3-9ab5-08f000699884 tags: + +``` python +df = pd.DataFrame({"x": x, "y": y}) +``` + +%% Cell type:code id:6e749b4f-b409-4700-870f-f68c39462490 tags:some-table + +``` python +df[:2] +``` + +%% Output + + x y + 0 0.000000 0.000000 + 1 0.256457 0.253655 + +%% Cell type:code id:0ddef5ef-94a3-4afd-9c70-ddee9694f512 tags: + +``` python +``` diff --git a/gems/ipynbdiff/spec/testdata/percent_decorator/expected_symbols.txt b/gems/ipynbdiff/spec/testdata/percent_decorator/expected_symbols.txt new file mode 100644 index 00000000000..c95665d1903 --- /dev/null +++ b/gems/ipynbdiff/spec/testdata/percent_decorator/expected_symbols.txt @@ -0,0 +1,68 @@ +.cells.0 + +.cells.0.source.0 +.cells.0.source.1 +.cells.0.source.2 +.cells.0.source.3 +.cells.0.source.4 +.cells.0.source.5 +.cells.0.source.6 +.cells.0.source.7 + +.cells.1 + +.cells.1.source.0 +.cells.1.source.1 +.cells.1.source.2 + +.cells.2 + +.cells.2.source +.cells.2.source.0 +.cells.2.source.1 +.cells.2.source.2 + + +.cells.3 + +.cells.3.source +.cells.3.source.0 + + +.cells.4 + +.cells.4.source +.cells.4.source.0 +.cells.4.source.1 +.cells.4.source.2 +.cells.4.source.3 + + +.cells.4.outputs + +.cells.4.outputs.0.data.text/plain.0 + +.cells.4.outputs.1.data.image/png + +.cells.5 + +.cells.5.source +.cells.5.source.0 + + +.cells.6 + +.cells.6.source +.cells.6.source.0 + + +.cells.6.outputs + +.cells.6.outputs.0.data.text/plain.0 +.cells.6.outputs.0.data.text/plain.1 +.cells.6.outputs.0.data.text/plain.2 + +.cells.7 + +.cells.7.source + diff --git a/gems/ipynbdiff/spec/testdata/single_line_md/expected.md b/gems/ipynbdiff/spec/testdata/single_line_md/expected.md new file mode 100644 index 00000000000..392a5048f59 --- /dev/null +++ b/gems/ipynbdiff/spec/testdata/single_line_md/expected.md @@ -0,0 +1,3 @@ +%% Cell type:markdown id:1 tags:hello,world + +A diff --git a/gems/ipynbdiff/spec/testdata/single_line_md/expected_symbols.txt b/gems/ipynbdiff/spec/testdata/single_line_md/expected_symbols.txt new file mode 100644 index 00000000000..86a7f6b3960 --- /dev/null +++ b/gems/ipynbdiff/spec/testdata/single_line_md/expected_symbols.txt @@ -0,0 +1,3 @@ +.cells.0 + +.cells.0.source diff --git a/gems/ipynbdiff/spec/testdata/single_line_md/input.ipynb b/gems/ipynbdiff/spec/testdata/single_line_md/input.ipynb new file mode 100644 index 00000000000..5ebd41adbfa --- /dev/null +++ b/gems/ipynbdiff/spec/testdata/single_line_md/input.ipynb @@ -0,0 +1,17 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "1", + "metadata": { + "tags": [ + "hello", + "world" + ] + }, + "source": "A" + } + ], + "metadata": { + } +} diff --git a/gems/ipynbdiff/spec/testdata/source_with_linebreak/expected.md b/gems/ipynbdiff/spec/testdata/source_with_linebreak/expected.md new file mode 100644 index 00000000000..180fffe24ce --- /dev/null +++ b/gems/ipynbdiff/spec/testdata/source_with_linebreak/expected.md @@ -0,0 +1,5 @@ +%% Cell type:markdown id: tags: + +> This is a test +> +> To see if I can duplicate my bug diff --git a/gems/ipynbdiff/spec/testdata/source_with_linebreak/expected_symbols.txt b/gems/ipynbdiff/spec/testdata/source_with_linebreak/expected_symbols.txt new file mode 100644 index 00000000000..1e8bdda4b9a --- /dev/null +++ b/gems/ipynbdiff/spec/testdata/source_with_linebreak/expected_symbols.txt @@ -0,0 +1,5 @@ +.cells.0 + +.cells.0.source +.cells.0.source +.cells.0.source diff --git a/gems/ipynbdiff/spec/testdata/source_with_linebreak/input.ipynb b/gems/ipynbdiff/spec/testdata/source_with_linebreak/input.ipynb new file mode 100644 index 00000000000..faacc703969 --- /dev/null +++ b/gems/ipynbdiff/spec/testdata/source_with_linebreak/input.ipynb @@ -0,0 +1,11 @@ +{ + "metadata": { + }, + "cells": [ + { + "cell_type": "markdown", + "source": "> This is a test\n>\n> To see if I can duplicate my bug", + "metadata": {} + } + ] +} diff --git a/gems/ipynbdiff/spec/testdata/stream_text/expected.md b/gems/ipynbdiff/spec/testdata/stream_text/expected.md new file mode 100644 index 00000000000..0448bf21111 --- /dev/null +++ b/gems/ipynbdiff/spec/testdata/stream_text/expected.md @@ -0,0 +1,9 @@ +%% Cell type:code id:123 tags: + +``` python +print("G'bye") +``` + +%% Output + + G'bye diff --git a/gems/ipynbdiff/spec/testdata/stream_text/expected_symbols.txt b/gems/ipynbdiff/spec/testdata/stream_text/expected_symbols.txt new file mode 100644 index 00000000000..be4e2861ea9 --- /dev/null +++ b/gems/ipynbdiff/spec/testdata/stream_text/expected_symbols.txt @@ -0,0 +1,9 @@ +.cells.0 + +.cells.0.source +.cells.0.source.0 + + +.cells.0.outputs + +.cells.0.outputs.0.text.0 diff --git a/gems/ipynbdiff/spec/testdata/stream_text/input.ipynb b/gems/ipynbdiff/spec/testdata/stream_text/input.ipynb new file mode 100644 index 00000000000..14601fe35e5 --- /dev/null +++ b/gems/ipynbdiff/spec/testdata/stream_text/input.ipynb @@ -0,0 +1,27 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "123", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "G'bye\n" + ] + } + ], + "source": [ + "print(\"G'bye\")" + ] + } + ], + "metadata": { + "kernelspec": { + "language": "python" + } + } +} diff --git a/gems/ipynbdiff/spec/testdata/svg/expected.md b/gems/ipynbdiff/spec/testdata/svg/expected.md new file mode 100644 index 00000000000..a5a167d31c5 --- /dev/null +++ b/gems/ipynbdiff/spec/testdata/svg/expected.md @@ -0,0 +1,17 @@ +%% Cell type:code id:5 tags: + +``` python +from IPython.display import SVG, display + +svg = """<svg viewBox="0 0 100 100" xmlns="http://www.w3.org/2000/svg"> + <circle cx="50" cy="50" r="50"/> +</svg>""" + +display(SVG(svg)) +``` + +%% Output + + ![](data:image/svg+xml;utf8,<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 100 100"><circle cx="50" cy="50" r="50"/></svg>) + + ![](data:image/svg+xml;utf8,<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 100 100"><circle cx="50" cy="50" r="50"/></svg>) diff --git a/gems/ipynbdiff/spec/testdata/svg/expected_symbols.txt b/gems/ipynbdiff/spec/testdata/svg/expected_symbols.txt new file mode 100644 index 00000000000..861198a8c92 --- /dev/null +++ b/gems/ipynbdiff/spec/testdata/svg/expected_symbols.txt @@ -0,0 +1,17 @@ +.cells.0 + +.cells.0.source +.cells.0.source.0 +.cells.0.source.1 +.cells.0.source.2 +.cells.0.source.3 +.cells.0.source.4 +.cells.0.source.5 +.cells.0.source.6 + + +.cells.0.outputs + +.cells.0.outputs.0.data.image/svg+xml + +.cells.0.outputs.1.data.image/svg+xml diff --git a/gems/ipynbdiff/spec/testdata/svg/input.ipynb b/gems/ipynbdiff/spec/testdata/svg/input.ipynb new file mode 100644 index 00000000000..a02d01f7bf2 --- /dev/null +++ b/gems/ipynbdiff/spec/testdata/svg/input.ipynb @@ -0,0 +1,66 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 10, + "id": "5", + "metadata": {}, + "outputs": [ + { + "data": { + "image/svg+xml": [ + "<svg xmlns=\"http://www.w3.org/2000/svg\" viewBox=\"0 0 100 100\">\n", + " <circle cx=\"50\" cy=\"50\" r=\"50\"/>\n", + "</svg>" + ], + "text/plain": [ + "<IPython.core.display.SVG object>" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "image/svg+xml": "<svg xmlns=\"http://www.w3.org/2000/svg\" viewBox=\"0 0 100 100\"><circle cx=\"50\" cy=\"50\" r=\"50\"/></svg>", + "text/plain": [ + "<IPython.core.display.SVG object>" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from IPython.display import SVG, display\n", + "\n", + "svg = \"\"\"<svg viewBox=\"0 0 100 100\" xmlns=\"http://www.w3.org/2000/svg\">\n", + " <circle cx=\"50\" cy=\"50\" r=\"50\"/>\n", + "</svg>\"\"\"\n", + "\n", + "display(SVG(svg))" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.12" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/gems/ipynbdiff/spec/testdata/text_output/expected.md b/gems/ipynbdiff/spec/testdata/text_output/expected.md new file mode 100644 index 00000000000..1b6c086ecd5 --- /dev/null +++ b/gems/ipynbdiff/spec/testdata/text_output/expected.md @@ -0,0 +1,9 @@ +%% Cell type:code id:5 tags:senoid + +``` python +plt.plot(x, y) +``` + +%% Output + + [<matplotlib.lines.Line2D at 0x12a4e43d0>] diff --git a/gems/ipynbdiff/spec/testdata/text_output/expected_symbols.txt b/gems/ipynbdiff/spec/testdata/text_output/expected_symbols.txt new file mode 100644 index 00000000000..a004d852ba4 --- /dev/null +++ b/gems/ipynbdiff/spec/testdata/text_output/expected_symbols.txt @@ -0,0 +1,9 @@ +.cells.0 + +.cells.0.source +.cells.0.source.0 + + +.cells.0.outputs + +.cells.0.outputs.0.data.text/plain.0 diff --git a/gems/ipynbdiff/spec/testdata/text_output/input.ipynb b/gems/ipynbdiff/spec/testdata/text_output/input.ipynb new file mode 100644 index 00000000000..b1b387bb99d --- /dev/null +++ b/gems/ipynbdiff/spec/testdata/text_output/input.ipynb @@ -0,0 +1,31 @@ +{ + "cells": [ + { + "cell_type": "code", + "id": "5", + "metadata": { + "tags": [ + "senoid" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "[<matplotlib.lines.Line2D at 0x12a4e43d0>]" + ] + }, + "output_type": "execute_result" + } + ], + "source": [ + "plt.plot(x, y)" + ] + } + ], + "metadata": { + "kernelspec": { + "language": "python" + } + } +} diff --git a/gems/ipynbdiff/spec/testdata/text_png_output/expected.md b/gems/ipynbdiff/spec/testdata/text_png_output/expected.md new file mode 100644 index 00000000000..c77f109378c --- /dev/null +++ b/gems/ipynbdiff/spec/testdata/text_png_output/expected.md @@ -0,0 +1,14 @@ +%% Cell type:code id:5 tags:senoid + +``` python +x = np.linspace(0, 4*np.pi,50) +y = 2 * np.sin(x) + +plt.plot(x, y) +``` + +%% Output + + [<matplotlib.lines.Line2D at 0x12a4e43d0>] + +  diff --git a/gems/ipynbdiff/spec/testdata/text_png_output/expected_line_numbers.txt b/gems/ipynbdiff/spec/testdata/text_png_output/expected_line_numbers.txt new file mode 100644 index 00000000000..62e35deb96d --- /dev/null +++ b/gems/ipynbdiff/spec/testdata/text_png_output/expected_line_numbers.txt @@ -0,0 +1,14 @@ +3 + +36 +37 +38 +39 +40 + + +12 + +16 + +25 diff --git a/gems/ipynbdiff/spec/testdata/text_png_output/expected_symbols.txt b/gems/ipynbdiff/spec/testdata/text_png_output/expected_symbols.txt new file mode 100644 index 00000000000..49f2d7149d8 --- /dev/null +++ b/gems/ipynbdiff/spec/testdata/text_png_output/expected_symbols.txt @@ -0,0 +1,14 @@ +.cells.0 + +.cells.0.source +.cells.0.source.0 +.cells.0.source.1 +.cells.0.source.2 +.cells.0.source.3 + + +.cells.0.outputs + +.cells.0.outputs.0.data.text/plain.0 + +.cells.0.outputs.1.data.image/png diff --git a/gems/ipynbdiff/spec/testdata/text_png_output/input.ipynb b/gems/ipynbdiff/spec/testdata/text_png_output/input.ipynb new file mode 100644 index 00000000000..3728b129d26 --- /dev/null +++ b/gems/ipynbdiff/spec/testdata/text_png_output/input.ipynb @@ -0,0 +1,49 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 3, + "id": "5", + "metadata": { + "tags": [ + "senoid" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "[<matplotlib.lines.Line2D at 0x12a4e43d0>]" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "this_is_an_invalid_hash_for_testing_purposes\n", + "text/plain": [ + "<Figure size 432x288 with 1 Axes>" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "x = np.linspace(0, 4*np.pi,50)\n", + "y = 2 * np.sin(x)\n", + "\n", + "plt.plot(x, y)" + ] + } + ], + "metadata": { + "kernelspec": { + "language": "python" + } + } +} diff --git a/gems/ipynbdiff/spec/testdata/to.ipynb b/gems/ipynbdiff/spec/testdata/to.ipynb new file mode 100644 index 00000000000..99b51f3b857 --- /dev/null +++ b/gems/ipynbdiff/spec/testdata/to.ipynb @@ -0,0 +1,200 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "0aac5da7-745c-4eda-847a-3d0d07a1bb9b", + "metadata": { + "tags": [] + }, + "source": [ + "# This is a markdown cell\n", + "\n", + "This paragraph has\n", + "With\n", + "Many\n", + "Lines. How we will he handle MR notes?\n", + "\n", + "But I can add another paragraph\n", + "\n", + "Another paragraph added" + ] + }, + { + "cell_type": "raw", + "id": "faecea5b-de0a-49fa-9a3a-61c2add652da", + "metadata": {}, + "source": [ + "This is a raw cell\n", + "With\n", + "Multiple lines" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "893ca2c0-ab75-4276-9dad-be1c40e16e8a", + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import numpy as np\n", + "import matplotlib.pyplot as plt" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "0d707fb5-226f-46d6-80bd-489ebfb8905c", + "metadata": {}, + "outputs": [], + "source": [ + "np.random.seed(42)" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "35467fcf-28b1-4c7b-bb09-4cb192c35293", + "metadata": { + "tags": [ + "senoid" + ] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "[<matplotlib.lines.Line2D at 0x12a4e43d0>]" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "another_invalid_base64_image_here\n", + "text/plain": [ + "<Figure size 432x288 with 1 Axes>" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "x = np.linspace(0, 4*np.pi,50)\n", + "y = 2 * np.sin(x)\n", + "\n", + "plt.plot(x, y)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "dc1178cd-c46d-4da3-9ab5-08f000699884", + "metadata": {}, + "outputs": [], + "source": [ + "df = pd.DataFrame({\"x\": x, \"y\": y})" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "6e749b4f-b409-4700-870f-f68c39462490", + "metadata": { + "tags": [ + "some-table" + ] + }, + "outputs": [ + { + "data": { + "text/html": [ + "<div>\n", + "<style scoped>\n", + " .dataframe tbody tr th:only-of-type {\n", + " vertical-align: middle;\n", + " }\n", + "\n", + " .dataframe tbody tr th {\n", + " vertical-align: top;\n", + " }\n", + "\n", + " .dataframe thead th {\n", + " text-align: right;\n", + " }\n", + "</style>\n", + "<table border=\"1\" class=\"dataframe\">\n", + " <thead>\n", + " <tr style=\"text-align: right;\">\n", + " <th></th>\n", + " <th>x</th>\n", + " <th>y</th>\n", + " </tr>\n", + " </thead>\n", + " <tbody>\n", + " <tr>\n", + " <th>0</th>\n", + " <td>0.000000</td>\n", + " <td>0.000000</td>\n", + " </tr>\n", + " <tr>\n", + " <th>1</th>\n", + " <td>0.256457</td>\n", + " <td>0.507309</td>\n", + " </tr>\n", + " </tbody>\n", + "</table>\n", + "</div>" + ], + "text/plain": [ + " x y\n", + "0 0.000000 0.000000\n", + "1 0.256457 0.507309" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df[:2]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0ddef5ef-94a3-4afd-9c70-ddee9694f512", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "New Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.7" + }, + "toc-showtags": true + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/gems/ipynbdiff/spec/testdata/unknown_output_type/expected.md b/gems/ipynbdiff/spec/testdata/unknown_output_type/expected.md new file mode 100644 index 00000000000..af34d6eb8c3 --- /dev/null +++ b/gems/ipynbdiff/spec/testdata/unknown_output_type/expected.md @@ -0,0 +1,5 @@ +%% Cell type:code id:123 tags: + +``` python +print("G'bye") +``` diff --git a/gems/ipynbdiff/spec/testdata/unknown_output_type/expected_symbols.txt b/gems/ipynbdiff/spec/testdata/unknown_output_type/expected_symbols.txt new file mode 100644 index 00000000000..cb35f88c897 --- /dev/null +++ b/gems/ipynbdiff/spec/testdata/unknown_output_type/expected_symbols.txt @@ -0,0 +1,5 @@ +.cells.0 + +.cells.0.source +.cells.0.source.0 + diff --git a/gems/ipynbdiff/spec/testdata/unknown_output_type/input.ipynb b/gems/ipynbdiff/spec/testdata/unknown_output_type/input.ipynb new file mode 100644 index 00000000000..42f4b39b365 --- /dev/null +++ b/gems/ipynbdiff/spec/testdata/unknown_output_type/input.ipynb @@ -0,0 +1,27 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "123", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "unknown_output", + "text": [ + "G'bye\n" + ] + } + ], + "source": [ + "print(\"G'bye\")" + ] + } + ], + "metadata": { + "kernelspec": { + "language": "python" + } + } +} diff --git a/gems/rspec_flaky/.gitignore b/gems/rspec_flaky/.gitignore new file mode 100644 index 00000000000..b04a8c840df --- /dev/null +++ b/gems/rspec_flaky/.gitignore @@ -0,0 +1,11 @@ +/.bundle/ +/.yardoc +/_yardoc/ +/coverage/ +/doc/ +/pkg/ +/spec/reports/ +/tmp/ + +# rspec failure tracking +.rspec_status diff --git a/gems/rspec_flaky/.gitlab-ci.yml b/gems/rspec_flaky/.gitlab-ci.yml new file mode 100644 index 00000000000..41fac86e7a5 --- /dev/null +++ b/gems/rspec_flaky/.gitlab-ci.yml @@ -0,0 +1,4 @@ +include: + - local: gems/gem.gitlab-ci.yml + inputs: + gem_name: "rspec_flaky" diff --git a/gems/rspec_flaky/.rspec b/gems/rspec_flaky/.rspec new file mode 100644 index 00000000000..34c5164d9b5 --- /dev/null +++ b/gems/rspec_flaky/.rspec @@ -0,0 +1,3 @@ +--format documentation +--color +--require spec_helper diff --git a/gems/rspec_flaky/.rubocop.yml b/gems/rspec_flaky/.rubocop.yml new file mode 100644 index 00000000000..62cb8a982c5 --- /dev/null +++ b/gems/rspec_flaky/.rubocop.yml @@ -0,0 +1,13 @@ +inherit_from: + - ../config/rubocop.yml + +# FIXME once Gitlab::Json is in a gem +Gitlab/Json: + Enabled: false + +# FIXME +RSpec/MultipleMemoizedHelpers: + Enabled: false + +RSpec/VerifiedDoubles: + Enabled: false diff --git a/gems/rspec_flaky/Gemfile b/gems/rspec_flaky/Gemfile new file mode 100644 index 00000000000..90bf29fb647 --- /dev/null +++ b/gems/rspec_flaky/Gemfile @@ -0,0 +1,8 @@ +# frozen_string_literal: true + +source "https://rubygems.org" + +# Specify your gem's dependencies in rspec_flaky.gemspec +gemspec + +gem "gitlab-rspec", "~> 0.1", path: "../gitlab-rspec" diff --git a/gems/rspec_flaky/Gemfile.lock b/gems/rspec_flaky/Gemfile.lock new file mode 100644 index 00000000000..547dc24e375 --- /dev/null +++ b/gems/rspec_flaky/Gemfile.lock @@ -0,0 +1,126 @@ +PATH + remote: ../gitlab-rspec + specs: + gitlab-rspec (0.1.0) + activesupport (>= 6.1, < 7.1) + rspec (~> 3.0) + +PATH + remote: . + specs: + rspec_flaky (0.1.0) + activesupport (>= 6.1, < 7.1) + rspec (~> 3.0) + +GEM + remote: https://rubygems.org/ + specs: + activesupport (7.0.6) + concurrent-ruby (~> 1.0, >= 1.0.2) + i18n (>= 1.6, < 2) + minitest (>= 5.1) + tzinfo (~> 2.0) + ast (2.4.2) + binding_of_caller (1.0.0) + debug_inspector (>= 0.0.1) + coderay (1.1.3) + concurrent-ruby (1.2.2) + debug_inspector (1.1.0) + diff-lcs (1.5.0) + gitlab-styles (10.1.0) + rubocop (~> 1.50.2) + rubocop-graphql (~> 0.18) + rubocop-performance (~> 1.15) + rubocop-rails (~> 2.17) + rubocop-rspec (~> 2.22) + i18n (1.14.1) + concurrent-ruby (~> 1.0) + json (2.6.3) + minitest (5.18.1) + parallel (1.23.0) + parser (3.2.2.3) + ast (~> 2.4.1) + racc + proc_to_ast (0.1.0) + coderay + parser + unparser + racc (1.7.1) + rack (3.0.8) + rainbow (3.1.1) + regexp_parser (2.8.1) + rexml (3.2.5) + rspec (3.12.0) + rspec-core (~> 3.12.0) + rspec-expectations (~> 3.12.0) + rspec-mocks (~> 3.12.0) + rspec-core (3.12.2) + rspec-support (~> 3.12.0) + rspec-expectations (3.12.3) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.12.0) + rspec-mocks (3.12.5) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.12.0) + rspec-parameterized (1.0.0) + rspec-parameterized-core (< 2) + rspec-parameterized-table_syntax (< 2) + rspec-parameterized-core (1.0.0) + parser + proc_to_ast + rspec (>= 2.13, < 4) + unparser + rspec-parameterized-table_syntax (1.0.0) + binding_of_caller + rspec-parameterized-core (< 2) + rspec-support (3.12.1) + rubocop (1.50.2) + json (~> 2.3) + parallel (~> 1.10) + parser (>= 3.2.0.0) + rainbow (>= 2.2.2, < 4.0) + regexp_parser (>= 1.8, < 3.0) + rexml (>= 3.2.5, < 4.0) + rubocop-ast (>= 1.28.0, < 2.0) + ruby-progressbar (~> 1.7) + unicode-display_width (>= 2.4.0, < 3.0) + rubocop-ast (1.29.0) + parser (>= 3.2.1.0) + rubocop-capybara (2.18.0) + rubocop (~> 1.41) + rubocop-factory_bot (2.23.1) + rubocop (~> 1.33) + rubocop-graphql (0.19.0) + rubocop (>= 0.87, < 2) + rubocop-performance (1.18.0) + rubocop (>= 1.7.0, < 2.0) + rubocop-ast (>= 0.4.0) + rubocop-rails (2.20.2) + activesupport (>= 4.2.0) + rack (>= 1.1) + rubocop (>= 1.33.0, < 2.0) + rubocop-rspec (2.22.0) + rubocop (~> 1.33) + rubocop-capybara (~> 2.17) + rubocop-factory_bot (~> 2.22) + ruby-progressbar (1.13.0) + tzinfo (2.0.6) + concurrent-ruby (~> 1.0) + unicode-display_width (2.4.2) + unparser (0.6.8) + diff-lcs (~> 1.3) + parser (>= 3.2.0) + +PLATFORMS + ruby + +DEPENDENCIES + gitlab-rspec (~> 0.1)! + gitlab-styles (~> 10.1.0) + rspec-parameterized (~> 1.0) + rspec_flaky! + rubocop (~> 1.50) + rubocop-rspec (~> 2.22) + +BUNDLED WITH + 2.4.14 diff --git a/gems/rspec_flaky/lib/rspec_flaky.rb b/gems/rspec_flaky/lib/rspec_flaky.rb new file mode 100644 index 00000000000..90fc6b1dc49 --- /dev/null +++ b/gems/rspec_flaky/lib/rspec_flaky.rb @@ -0,0 +1,6 @@ +# frozen_string_literal: true + +require "rspec" +require_relative "rspec_flaky/config" +require_relative "rspec_flaky/listener" +require_relative "rspec_flaky/version" diff --git a/gems/rspec_flaky/lib/rspec_flaky/config.rb b/gems/rspec_flaky/lib/rspec_flaky/config.rb new file mode 100644 index 00000000000..ca57d1e08be --- /dev/null +++ b/gems/rspec_flaky/lib/rspec_flaky/config.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module RspecFlaky + class Config + def self.generate_report? + !!(ENV['FLAKY_RSPEC_GENERATE_REPORT'] =~ /1|true/) + end + + def self.suite_flaky_examples_report_path + ENV['FLAKY_RSPEC_SUITE_REPORT_PATH'] || "rspec/flaky/suite-report.json" + end + + def self.flaky_examples_report_path + ENV['FLAKY_RSPEC_REPORT_PATH'] || "rspec/flaky/report.json" + end + + def self.new_flaky_examples_report_path + ENV['NEW_FLAKY_RSPEC_REPORT_PATH'] || "rspec/flaky/new-report.json" + end + end +end diff --git a/gems/rspec_flaky/lib/rspec_flaky/example.rb b/gems/rspec_flaky/lib/rspec_flaky/example.rb new file mode 100644 index 00000000000..4a128a151dc --- /dev/null +++ b/gems/rspec_flaky/lib/rspec_flaky/example.rb @@ -0,0 +1,68 @@ +# frozen_string_literal: true + +require 'forwardable' +require 'digest' + +module RspecFlaky + # This is a wrapper class for RSpec::Core::Example + class Example + extend Forwardable + + def_delegators :execution_result, :status, :exception + + def initialize(rspec_example) + @rspec_example = rspec_example.respond_to?(:example) ? rspec_example.example : rspec_example + end + + def uid + @uid ||= Digest::MD5.hexdigest("#{description}-#{file}") # rubocop:disable Fips/MD5 + end + + def example_id + rspec_example.id + end + + def file + metadata[:file_path] + end + + def line + metadata[:line_number] + end + + def description + metadata[:full_description] + end + + def attempts + rspec_example.respond_to?(:attempts) ? rspec_example.attempts : 1 + end + + def feature_category + metadata[:feature_category] + end + + def to_h + { + example_id: example_id, + file: file, + line: line, + description: description, + last_attempts_count: attempts, + feature_category: feature_category + } + end + + private + + attr_reader :rspec_example + + def metadata + rspec_example.metadata + end + + def execution_result + rspec_example.execution_result + end + end +end diff --git a/gems/rspec_flaky/lib/rspec_flaky/flaky_example.rb b/gems/rspec_flaky/lib/rspec_flaky/flaky_example.rb new file mode 100644 index 00000000000..35d1f34d2a2 --- /dev/null +++ b/gems/rspec_flaky/lib/rspec_flaky/flaky_example.rb @@ -0,0 +1,59 @@ +# frozen_string_literal: true + +require 'ostruct' + +module RspecFlaky + # This represents a flaky RSpec example and is mainly meant to be saved in a JSON file + class FlakyExample + ALLOWED_ATTRIBUTES = %i[ + example_id + file + line + description + first_flaky_at + last_flaky_at + last_flaky_job + last_attempts_count + flaky_reports + feature_category + ].freeze + + def initialize(example_hash) + @attributes = { + first_flaky_at: Time.now, + last_flaky_at: Time.now, + last_flaky_job: nil, + last_attempts_count: example_hash[:attempts], + flaky_reports: 0, + feature_category: example_hash[:feature_category] + }.merge(example_hash.slice(*ALLOWED_ATTRIBUTES)) + + %i[first_flaky_at last_flaky_at].each do |attr| + attributes[attr] = Time.parse(attributes[attr]) if attributes[attr].is_a?(String) + end + end + + def update!(example_hash) + attributes[:file] = example_hash[:file] + attributes[:line] = example_hash[:line] + attributes[:description] = example_hash[:description] + attributes[:first_flaky_at] ||= Time.now + attributes[:last_flaky_at] = Time.now + attributes[:flaky_reports] += 1 + attributes[:feature_category] = example_hash[:feature_category] + attributes[:last_attempts_count] = example_hash[:last_attempts_count] if example_hash[:last_attempts_count] + + return unless ENV['CI_JOB_URL'] + + attributes[:last_flaky_job] = (ENV['CI_JOB_URL']).to_s + end + + def to_h + attributes.dup + end + + private + + attr_reader :attributes + end +end diff --git a/gems/rspec_flaky/lib/rspec_flaky/flaky_examples_collection.rb b/gems/rspec_flaky/lib/rspec_flaky/flaky_examples_collection.rb new file mode 100644 index 00000000000..f03fe63d11b --- /dev/null +++ b/gems/rspec_flaky/lib/rspec_flaky/flaky_examples_collection.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +require 'active_support/hash_with_indifferent_access' +require 'delegate' + +require_relative 'flaky_example' + +module RspecFlaky + class FlakyExamplesCollection < SimpleDelegator + def initialize(collection = {}) + raise ArgumentError, "`collection` must be a Hash, #{collection.class} given!" unless collection.is_a?(Hash) + + collection_of_flaky_examples = + collection.map do |uid, example| + [ + uid, + FlakyExample.new(example.to_h.symbolize_keys) + ] + end + + super(Hash[collection_of_flaky_examples]) + end + + def to_h + transform_values(&:to_h).deep_symbolize_keys + end + + def -(other) + raise ArgumentError, "`other` must respond to `#key?`, #{other.class} does not!" unless other.respond_to?(:key) + + self.class.new(reject { |uid, _| other.key?(uid) }) + end + end +end diff --git a/gems/rspec_flaky/lib/rspec_flaky/listener.rb b/gems/rspec_flaky/lib/rspec_flaky/listener.rb new file mode 100644 index 00000000000..c2deb1c327a --- /dev/null +++ b/gems/rspec_flaky/lib/rspec_flaky/listener.rb @@ -0,0 +1,70 @@ +# frozen_string_literal: true + +require 'json' + +require_relative 'config' +require_relative 'example' +require_relative 'flaky_example' +require_relative 'flaky_examples_collection' +require_relative 'report' + +module RspecFlaky + class Listener + # - suite_flaky_examples: contains all the currently tracked flacky example + # for the whole RSpec suite + # - flaky_examples: contains the examples detected as flaky during the + # current RSpec run + attr_reader :suite_flaky_examples, :flaky_examples + + def initialize(suite_flaky_examples_json = nil) + @flaky_examples = FlakyExamplesCollection.new + @suite_flaky_examples = init_suite_flaky_examples(suite_flaky_examples_json) + end + + def example_passed(notification) + current_example = Example.new(notification.example) + + return unless current_example.attempts > 1 + + flaky_example = suite_flaky_examples.fetch(current_example.uid) do + FlakyExample.new(current_example.to_h) + end + flaky_example.update!(current_example.to_h) + + flaky_examples[current_example.uid] = flaky_example + end + + def dump_summary(_) + Report.new(flaky_examples).write(Config.flaky_examples_report_path) + + return unless new_flaky_examples.any? + + rails_logger_warn("\nNew flaky examples detected:\n") + rails_logger_warn(JSON.pretty_generate(new_flaky_examples.to_h)) + + Report.new(new_flaky_examples).write(Config.new_flaky_examples_report_path) + end + + private + + def new_flaky_examples + @new_flaky_examples ||= flaky_examples - suite_flaky_examples + end + + def init_suite_flaky_examples(suite_flaky_examples_json = nil) + if suite_flaky_examples_json + Report.load_json(suite_flaky_examples_json).flaky_examples + else + return {} unless File.exist?(Config.suite_flaky_examples_report_path) + + Report.load(Config.suite_flaky_examples_report_path).flaky_examples + end + end + + def rails_logger_warn(text) + target = defined?(Rails) ? Rails.logger : Kernel + + target.warn(text) + end + end +end diff --git a/gems/rspec_flaky/lib/rspec_flaky/report.rb b/gems/rspec_flaky/lib/rspec_flaky/report.rb new file mode 100644 index 00000000000..cc213d336ae --- /dev/null +++ b/gems/rspec_flaky/lib/rspec_flaky/report.rb @@ -0,0 +1,59 @@ +# frozen_string_literal: true + +require 'json' +require 'time' +require 'fileutils' + +require_relative 'config' +require_relative 'flaky_examples_collection' + +module RspecFlaky + # This class is responsible for loading/saving JSON reports, and pruning + # outdated examples. + class Report < SimpleDelegator + OUTDATED_DAYS_THRESHOLD = 7 + + attr_reader :flaky_examples + + def self.load(file_path) + load_json(File.read(file_path)) + end + + def self.load_json(json) + new(FlakyExamplesCollection.new(JSON.parse(json))) + end + + def initialize(flaky_examples) + unless flaky_examples.is_a?(FlakyExamplesCollection) + raise ArgumentError, + "`flaky_examples` must be a RspecFlaky::FlakyExamplesCollection, #{flaky_examples.class} given!" + end + + @flaky_examples = flaky_examples + super(flaky_examples) + end + + def write(file_path) + unless Config.generate_report? + Kernel.warn "! Generating reports is disabled. To enable it, please set the `FLAKY_RSPEC_GENERATE_REPORT=1` !" + return + end + + report_path_dir = File.dirname(file_path) + FileUtils.mkdir_p(report_path_dir) + + File.write(file_path, JSON.pretty_generate(flaky_examples.to_h)) + end + + def prune_outdated(days: OUTDATED_DAYS_THRESHOLD) + outdated_date_threshold = Time.now - (3600 * 24 * days) + recent_flaky_examples = flaky_examples.dup + .delete_if do |_uid, flaky_example| + last_flaky_at = flaky_example.to_h[:last_flaky_at] + last_flaky_at && last_flaky_at.to_i < outdated_date_threshold.to_i + end + + self.class.new(FlakyExamplesCollection.new(recent_flaky_examples)) + end + end +end diff --git a/gems/rspec_flaky/lib/rspec_flaky/version.rb b/gems/rspec_flaky/lib/rspec_flaky/version.rb new file mode 100644 index 00000000000..ec507d734c8 --- /dev/null +++ b/gems/rspec_flaky/lib/rspec_flaky/version.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +module RspecFlaky + module Version + VERSION = "0.1.0" + end +end diff --git a/gems/rspec_flaky/rspec_flaky.gemspec b/gems/rspec_flaky/rspec_flaky.gemspec new file mode 100644 index 00000000000..5c0a434218f --- /dev/null +++ b/gems/rspec_flaky/rspec_flaky.gemspec @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +require_relative "lib/rspec_flaky/version" + +Gem::Specification.new do |spec| + spec.name = "rspec_flaky" + spec.version = RspecFlaky::Version::VERSION + spec.authors = ["Engineering Productivity"] + spec.email = ["quality@gitlab.com"] + + spec.summary = "GitLab's RSpec Flaky test detector" + spec.description = + "This gem provide an RSpec listener that allows to detect flaky examples. See " \ + "https://docs.gitlab.com/ee/development/testing_guide/flaky_tests.html#automatic-retries-and-flaky-tests-detection." + spec.homepage = "https://gitlab.com/gitlab-org/gitlab/-/tree/master/gems/rspec_flaky" + spec.license = "MIT" + spec.required_ruby_version = ">= 3.0" + spec.metadata["rubygems_mfa_required"] = "true" + + spec.files = Dir["lib/**/*.rb"] + spec.require_paths = ["lib"] + + spec.add_runtime_dependency "activesupport", ">= 6.1", "< 7.1" + spec.add_runtime_dependency "rspec", "~> 3.0" + + spec.add_development_dependency "gitlab-styles", "~> 10.1.0" + spec.add_development_dependency "rspec-parameterized", "~> 1.0" + spec.add_development_dependency "rubocop", "~> 1.50" + spec.add_development_dependency "rubocop-rspec", "~> 2.22" +end diff --git a/gems/rspec_flaky/spec/rspec_flaky/config_spec.rb b/gems/rspec_flaky/spec/rspec_flaky/config_spec.rb new file mode 100644 index 00000000000..827249efefa --- /dev/null +++ b/gems/rspec_flaky/spec/rspec_flaky/config_spec.rb @@ -0,0 +1,99 @@ +# frozen_string_literal: true + +require 'rspec_flaky/config' + +RSpec.describe RspecFlaky::Config, :aggregate_failures do + include StubENV + + before do + # Stub these env variables otherwise specs don't behave the same on the CI + stub_env('FLAKY_RSPEC_GENERATE_REPORT', nil) + stub_env('FLAKY_RSPEC_SUITE_REPORT_PATH', nil) + stub_env('FLAKY_RSPEC_REPORT_PATH', nil) + stub_env('NEW_FLAKY_RSPEC_REPORT_PATH', nil) + end + + describe '.generate_report?' do + context "when ENV['FLAKY_RSPEC_GENERATE_REPORT'] is not set" do + it 'returns false' do + expect(described_class).not_to be_generate_report + end + end + + context "when ENV['FLAKY_RSPEC_GENERATE_REPORT'] is set" do + using RSpec::Parameterized::TableSyntax + + where(:env_value, :result) do + '1' | true + 'true' | true + 'foo' | false + '0' | false + 'false' | false + end + + with_them do + before do + stub_env('FLAKY_RSPEC_GENERATE_REPORT', env_value) + end + + it 'returns false' do + expect(described_class.generate_report?).to be(result) + end + end + end + end + + describe '.suite_flaky_examples_report_path' do + context "when ENV['FLAKY_RSPEC_SUITE_REPORT_PATH'] is not set" do + it 'returns the default path' do + expect(described_class.suite_flaky_examples_report_path).to eq('rspec/flaky/suite-report.json') + end + end + + context "when ENV['FLAKY_RSPEC_SUITE_REPORT_PATH'] is set" do + before do + stub_env('FLAKY_RSPEC_SUITE_REPORT_PATH', 'foo/suite-report.json') + end + + it 'returns the value of the env variable' do + expect(described_class.suite_flaky_examples_report_path).to eq('foo/suite-report.json') + end + end + end + + describe '.flaky_examples_report_path' do + context "when ENV['FLAKY_RSPEC_REPORT_PATH'] is not set" do + it 'returns the default path' do + expect(described_class.flaky_examples_report_path).to eq('rspec/flaky/report.json') + end + end + + context "when ENV['FLAKY_RSPEC_REPORT_PATH'] is set" do + before do + stub_env('FLAKY_RSPEC_REPORT_PATH', 'foo/report.json') + end + + it 'returns the value of the env variable' do + expect(described_class.flaky_examples_report_path).to eq('foo/report.json') + end + end + end + + describe '.new_flaky_examples_report_path' do + context "when ENV['NEW_FLAKY_RSPEC_REPORT_PATH'] is not set" do + it 'returns the default path' do + expect(described_class.new_flaky_examples_report_path).to eq('rspec/flaky/new-report.json') + end + end + + context "when ENV['NEW_FLAKY_RSPEC_REPORT_PATH'] is set" do + before do + stub_env('NEW_FLAKY_RSPEC_REPORT_PATH', 'foo/new-report.json') + end + + it 'returns the value of the env variable' do + expect(described_class.new_flaky_examples_report_path).to eq('foo/new-report.json') + end + end + end +end diff --git a/gems/rspec_flaky/spec/rspec_flaky/example_spec.rb b/gems/rspec_flaky/spec/rspec_flaky/example_spec.rb new file mode 100644 index 00000000000..64d65c0e170 --- /dev/null +++ b/gems/rspec_flaky/spec/rspec_flaky/example_spec.rb @@ -0,0 +1,99 @@ +# frozen_string_literal: true + +require 'rspec_flaky/example' + +RSpec.describe RspecFlaky::Example do + let(:example_attrs) do + { + id: 'spec/foo/bar_spec.rb:2', + metadata: { + file_path: 'spec/foo/bar_spec.rb', + line_number: 2, + full_description: 'hello world', + feature_category: :feature_category + }, + execution_result: double(status: 'passed', exception: 'BOOM!'), + attempts: 1 + } + end + + let(:rspec_example) { double(example_attrs) } + + describe '#initialize' do + shared_examples 'a valid Example instance' do + it 'returns valid attributes' do + example = described_class.new(args) + + expect(example.example_id).to eq(example_attrs[:id]) + end + end + + context 'when given an Rspec::Core::Example that responds to #example' do + let(:args) { double(example: rspec_example) } + + it_behaves_like 'a valid Example instance' + end + + context 'when given an Rspec::Core::Example that does not respond to #example' do + let(:args) { rspec_example } + + it_behaves_like 'a valid Example instance' + end + end + + subject { described_class.new(rspec_example) } + + describe '#uid' do + it 'returns a hash of the full description' do + expect(subject.uid).to eq(Digest::MD5.hexdigest("#{subject.description}-#{subject.file}")) # rubocop:disable Fips/MD5 + end + end + + describe '#example_id' do + it 'returns the ID of the RSpec::Core::Example' do + expect(subject.example_id).to eq(rspec_example.id) + end + end + + describe '#attempts' do + it 'returns the attempts of the RSpec::Core::Example' do + expect(subject.attempts).to eq(rspec_example.attempts) + end + end + + describe '#file' do + it 'returns the metadata[:file_path] of the RSpec::Core::Example' do + expect(subject.file).to eq(rspec_example.metadata[:file_path]) + end + end + + describe '#line' do + it 'returns the metadata[:line_number] of the RSpec::Core::Example' do + expect(subject.line).to eq(rspec_example.metadata[:line_number]) + end + end + + describe '#description' do + it 'returns the metadata[:full_description] of the RSpec::Core::Example' do + expect(subject.description).to eq(rspec_example.metadata[:full_description]) + end + end + + describe '#status' do + it 'returns the execution_result.status of the RSpec::Core::Example' do + expect(subject.status).to eq(rspec_example.execution_result.status) + end + end + + describe '#exception' do + it 'returns the execution_result.exception of the RSpec::Core::Example' do + expect(subject.exception).to eq(rspec_example.execution_result.exception) + end + end + + describe '#feature_category' do + it 'returns the metadata[:feature_category] of the RSpec::Core::Example' do + expect(subject.feature_category).to eq(rspec_example.metadata[:feature_category]) + end + end +end diff --git a/gems/rspec_flaky/spec/rspec_flaky/flaky_example_spec.rb b/gems/rspec_flaky/spec/rspec_flaky/flaky_example_spec.rb new file mode 100644 index 00000000000..244ca275f14 --- /dev/null +++ b/gems/rspec_flaky/spec/rspec_flaky/flaky_example_spec.rb @@ -0,0 +1,144 @@ +# frozen_string_literal: true + +require 'rspec_flaky/flaky_example' + +RSpec.describe RspecFlaky::FlakyExample, :aggregate_failures do + include StubENV + + let(:example_attrs) do + { + example_id: 'spec/foo/bar_spec.rb:2', + file: 'spec/foo/bar_spec.rb', + line: 2, + description: 'hello world', + last_attempts_count: 2, + feature_category: :feature_category + } + end + + before do + # Stub these env variables otherwise specs don't behave the same on the CI + stub_env('CI_JOB_URL', nil) + end + + describe '#initialize', :freeze_time do + shared_examples 'a valid FlakyExample instance' do + let(:flaky_example) { described_class.new(args) } + + it 'returns valid attributes' do + attrs = flaky_example.to_h + + expect(attrs[:uid]).to eq(example_attrs[:uid]) + expect(attrs[:file]).to eq(example_attrs[:file]) + expect(attrs[:line]).to eq(example_attrs[:line]) + expect(attrs[:description]).to eq(example_attrs[:description]) + expect(attrs[:feature_category]).to eq(example_attrs[:feature_category]) + expect(attrs[:first_flaky_at]).to eq(expected_first_flaky_at) + expect(attrs[:last_flaky_at]).to eq(expected_last_flaky_at) + expect(attrs[:last_attempts_count]).to eq(example_attrs[:last_attempts_count]) + expect(attrs[:flaky_reports]).to eq(expected_flaky_reports) + end + end + + context 'when given an Example.to_h' do + it_behaves_like 'a valid FlakyExample instance' do + let(:args) { example_attrs } + let(:expected_first_flaky_at) { Time.now } + let(:expected_last_flaky_at) { Time.now } + let(:expected_flaky_reports) { 0 } + end + end + end + + describe '#update!' do + shared_examples 'an up-to-date FlakyExample instance' do + let(:flaky_example) { described_class.new(args) } + + it 'sets the first_flaky_at if none exists' do + args[:first_flaky_at] = nil + + freeze_time do + flaky_example.update!(example_attrs) + + expect(flaky_example.to_h[:first_flaky_at]).to eq(Time.now) + end + end + + it 'maintains the first_flaky_at if exists' do + flaky_example.update!(example_attrs) + expected_first_flaky_at = flaky_example.to_h[:first_flaky_at] + + travel_to(Time.now + 42) do + flaky_example.update!(example_attrs) + expect(flaky_example.to_h[:first_flaky_at]).to eq(expected_first_flaky_at) + end + end + + it 'updates the last_flaky_at' do + travel_to(Time.now + 42) do + the_future = Time.now + flaky_example.update!(example_attrs) + + expect(flaky_example.to_h[:last_flaky_at]).to eq(the_future) + end + end + + it 'updates the flaky_reports' do + expected_flaky_reports = flaky_example.to_h[:first_flaky_at] ? flaky_example.to_h[:flaky_reports] + 1 : 1 + + expect { flaky_example.update!(example_attrs) }.to change { flaky_example.to_h[:flaky_reports] }.by(1) + expect(flaky_example.to_h[:flaky_reports]).to eq(expected_flaky_reports) + end + + it 'updates the last_attempts_count' do + example_attrs[:last_attempts_count] = 42 + flaky_example.update!(example_attrs) + + expect(flaky_example.to_h[:last_attempts_count]).to eq(42) + end + + context 'when run on the CI' do + let(:job_url) { 'https://gitlab.com/gitlab-org/gitlab-foss/-/jobs/42' } + + before do + stub_env('CI_JOB_URL', job_url) + end + + it 'updates the last_flaky_job' do + flaky_example.update!(example_attrs) + + expect(flaky_example.to_h[:last_flaky_job]).to eq(job_url) + end + end + end + + context 'when given an Example hash' do + it_behaves_like 'an up-to-date FlakyExample instance' do + let(:args) { example_attrs } + end + end + end + + describe '#to_h', :freeze_time do + shared_examples 'a valid FlakyExample hash' do + let(:additional_attrs) { {} } + + it 'returns a valid hash' do + flaky_example = described_class.new(args) + final_hash = example_attrs.merge(additional_attrs) + + expect(flaky_example.to_h).to eq(final_hash) + end + end + + context 'when given an Example hash' do + let(:args) { example_attrs } + + it_behaves_like 'a valid FlakyExample hash' do + let(:additional_attrs) do + { first_flaky_at: Time.now, last_flaky_at: Time.now, last_flaky_job: nil, flaky_reports: 0 } + end + end + end + end +end diff --git a/gems/rspec_flaky/spec/rspec_flaky/flaky_examples_collection_spec.rb b/gems/rspec_flaky/spec/rspec_flaky/flaky_examples_collection_spec.rb new file mode 100644 index 00000000000..260ebc72192 --- /dev/null +++ b/gems/rspec_flaky/spec/rspec_flaky/flaky_examples_collection_spec.rb @@ -0,0 +1,87 @@ +# frozen_string_literal: true + +require 'rspec_flaky/flaky_examples_collection' + +RSpec.describe RspecFlaky::FlakyExamplesCollection, :aggregate_failures, :freeze_time do + let(:collection_hash) do + { + a: { example_id: 'spec/foo/bar_spec.rb:2' }, + b: { example_id: 'spec/foo/baz_spec.rb:3' } + } + end + + let(:collection_report) do + { + a: { + example_id: 'spec/foo/bar_spec.rb:2', + first_flaky_at: Time.now, + last_flaky_at: Time.now, + last_flaky_job: nil, + flaky_reports: 0, + feature_category: nil, + last_attempts_count: nil + }, + b: { + example_id: 'spec/foo/baz_spec.rb:3', + first_flaky_at: Time.now, + last_flaky_at: Time.now, + last_flaky_job: nil, + flaky_reports: 0, + feature_category: nil, + last_attempts_count: nil + } + } + end + + describe '#initialize' do + it 'accepts no argument' do + expect { described_class.new }.not_to raise_error + end + + it 'accepts a hash' do + expect { described_class.new(collection_hash) }.not_to raise_error + end + + it 'does not accept anything else' do + expect do + described_class.new([1, 2, 3]) + end.to raise_error(ArgumentError, "`collection` must be a Hash, Array given!") + end + end + + describe '#to_h' do + it 'calls #to_h on the values' do + collection = described_class.new(collection_hash) + + expect(collection.to_h).to eq(collection_report) + end + end + + describe '#-' do + it 'returns only examples that are not present in the given collection' do + collection1 = described_class.new(collection_hash) + collection2 = described_class.new( + a: { example_id: 'spec/foo/bar_spec.rb:2' }, + c: { example_id: 'spec/bar/baz_spec.rb:4' }) + + expect((collection2 - collection1).to_h).to eq( + c: { + example_id: 'spec/bar/baz_spec.rb:4', + first_flaky_at: Time.now, + last_flaky_at: Time.now, + last_flaky_job: nil, + flaky_reports: 0, + feature_category: nil, + last_attempts_count: nil + }) + end + + it 'fails if the given collection does not respond to `#key?`' do + collection = described_class.new(collection_hash) + + expect do + collection - [1, 2, 3] + end.to raise_error(ArgumentError, "`other` must respond to `#key?`, Array does not!") + end + end +end diff --git a/gems/rspec_flaky/spec/rspec_flaky/listener_spec.rb b/gems/rspec_flaky/spec/rspec_flaky/listener_spec.rb new file mode 100644 index 00000000000..cbc5422a763 --- /dev/null +++ b/gems/rspec_flaky/spec/rspec_flaky/listener_spec.rb @@ -0,0 +1,229 @@ +# frozen_string_literal: true + +require 'rspec_flaky/listener' + +RSpec.describe RspecFlaky::Listener, :aggregate_failures do + include StubENV + + let(:already_flaky_example_uid) { '6e869794f4cfd2badd93eb68719371d1' } + let(:suite_flaky_example_report) do + { + "#{already_flaky_example_uid}": { + example_id: 'spec/foo/bar_spec.rb:2', + file: 'spec/foo/bar_spec.rb', + line: 2, + description: 'hello world', + first_flaky_at: 1234, + last_flaky_at: 4321, + last_attempts_count: 3, + flaky_reports: 1, + last_flaky_job: nil + } + } + end + + let(:already_flaky_example_attrs) do + { + id: 'spec/foo/bar_spec.rb:2', + metadata: { + file_path: 'spec/foo/bar_spec.rb', + line_number: 2, + full_description: 'hello world' + }, + execution_result: double(status: 'passed', exception: nil) + } + end + + let(:already_flaky_example) do + RspecFlaky::FlakyExample.new(suite_flaky_example_report[already_flaky_example_uid]) + end + + let(:new_example_attrs) do + { + id: 'spec/foo/baz_spec.rb:3', + metadata: { + file_path: 'spec/foo/baz_spec.rb', + line_number: 3, + full_description: 'hello GitLab' + }, + execution_result: double(status: 'passed', exception: nil) + } + end + + before do + # Stub these env variables otherwise specs don't behave the same on the CI + stub_env('CI_JOB_URL', nil) + stub_env('FLAKY_RSPEC_SUITE_REPORT_PATH', nil) + end + + describe '#initialize' do + shared_examples 'a valid Listener instance' do + let(:expected_suite_flaky_examples) { {} } + + it 'returns a valid Listener instance' do + listener = described_class.new + + expect(listener.suite_flaky_examples.to_h).to eq(expected_suite_flaky_examples) + expect(listener.flaky_examples).to eq({}) + end + end + + context 'when no report file exists' do + it_behaves_like 'a valid Listener instance' + end + + context 'when FLAKY_RSPEC_SUITE_REPORT_PATH is set' do + let(:report_file_path) { 'foo/report.json' } + + before do + stub_env('FLAKY_RSPEC_SUITE_REPORT_PATH', report_file_path) + end + + context 'when report file exists' do + before do + allow(File).to receive(:exist?).with(report_file_path).and_return(true) + end + + it 'delegates the load to RspecFlaky::Report' do + report = RspecFlaky::Report + .new(RspecFlaky::FlakyExamplesCollection.new(suite_flaky_example_report)) + + expect(RspecFlaky::Report).to receive(:load).with(report_file_path).and_return(report) + expect(described_class.new.suite_flaky_examples.to_h).to eq(report.flaky_examples.to_h) + end + end + + context 'when report file does not exist' do + before do + allow(File).to receive(:exist?).with(report_file_path).and_return(false) + end + + it 'return an empty hash' do + expect(RspecFlaky::Report).not_to receive(:load) + expect(described_class.new.suite_flaky_examples.to_h).to eq({}) + end + end + end + end + + describe '#example_passed' do + let(:rspec_example) { double(new_example_attrs) } + let(:notification) { double(example: rspec_example) } + let(:listener) { described_class.new(suite_flaky_example_report.to_json) } + + shared_examples 'a non-flaky example' do + it 'does not change the flaky examples hash' do + expect { listener.example_passed(notification) } + .not_to change { listener.flaky_examples } + end + end + + shared_examples 'an existing flaky example' do + let(:expected_flaky_example) do + { + example_id: 'spec/foo/bar_spec.rb:2', + file: 'spec/foo/bar_spec.rb', + line: 2, + description: 'hello world', + first_flaky_at: 1234, + last_attempts_count: 2, + flaky_reports: 2, + feature_category: nil, + last_flaky_job: nil + } + end + + it 'changes the flaky examples hash' do + new_example = RspecFlaky::Example.new(rspec_example) + + travel_to(Time.now + 42) do + the_future = Time.now + expect { listener.example_passed(notification) } + .to change { listener.flaky_examples[new_example.uid].to_h } + expect(listener.flaky_examples[new_example.uid].to_h) + .to eq(expected_flaky_example.merge(last_flaky_at: the_future)) + end + end + end + + shared_examples 'a new flaky example' do + let(:expected_flaky_example) do + { + example_id: 'spec/foo/baz_spec.rb:3', + file: 'spec/foo/baz_spec.rb', + line: 3, + description: 'hello GitLab', + last_attempts_count: 2, + flaky_reports: 1, + feature_category: nil, + last_flaky_job: nil + } + end + + it 'changes the all flaky examples hash' do + new_example = RspecFlaky::Example.new(rspec_example) + + travel_to(Time.now + 42) do + the_future = Time.now + expect { listener.example_passed(notification) } + .to change { listener.flaky_examples[new_example.uid].to_h } + expect(listener.flaky_examples[new_example.uid].to_h) + .to eq(expected_flaky_example.merge(first_flaky_at: the_future, last_flaky_at: the_future)) + end + end + end + + describe 'when the RSpec example does not respond to attempts' do + it_behaves_like 'a non-flaky example' + end + + describe 'when the RSpec example has 1 attempt' do + let(:rspec_example) { double(new_example_attrs.merge(attempts: 1)) } + + it_behaves_like 'a non-flaky example' + end + + describe 'when the RSpec example has 2 attempts' do + let(:rspec_example) { double(new_example_attrs.merge(attempts: 2)) } + + it_behaves_like 'a new flaky example' + + context 'with an existing flaky example' do + let(:rspec_example) { double(already_flaky_example_attrs.merge(attempts: 2)) } + + it_behaves_like 'an existing flaky example' + end + end + end + + describe '#dump_summary' do + let(:listener) { described_class.new(suite_flaky_example_report.to_json) } + let(:new_flaky_rspec_example) { double(new_example_attrs.merge(attempts: 2)) } + let(:already_flaky_rspec_example) { double(already_flaky_example_attrs.merge(attempts: 2)) } + let(:notification_new_flaky_rspec_example) { double(example: new_flaky_rspec_example) } + let(:notification_already_flaky_rspec_example) { double(example: already_flaky_rspec_example) } + + before do + allow(Kernel).to receive(:warn) + end + + context 'when a report file path is set by FLAKY_RSPEC_REPORT_PATH' do + it 'delegates the writes to RspecFlaky::Report' do + listener.example_passed(notification_new_flaky_rspec_example) + listener.example_passed(notification_already_flaky_rspec_example) + + report1 = double + report2 = double + + expect(RspecFlaky::Report).to receive(:new).with(listener.flaky_examples).and_return(report1) + expect(report1).to receive(:write).with(RspecFlaky::Config.flaky_examples_report_path) + + expect(RspecFlaky::Report) + .to receive(:new).with(listener.__send__(:new_flaky_examples)).and_return(report2) + expect(report2).to receive(:write).with(RspecFlaky::Config.new_flaky_examples_report_path) + + listener.dump_summary(nil) + end + end + end +end diff --git a/gems/rspec_flaky/spec/rspec_flaky/report_spec.rb b/gems/rspec_flaky/spec/rspec_flaky/report_spec.rb new file mode 100644 index 00000000000..e1e9bd6a7b1 --- /dev/null +++ b/gems/rspec_flaky/spec/rspec_flaky/report_spec.rb @@ -0,0 +1,141 @@ +# frozen_string_literal: true + +require 'tempfile' + +require 'rspec_flaky/report' + +RSpec.describe RspecFlaky::Report, :aggregate_failures, :freeze_time do + let(:thirty_one_days) { 3600 * 24 * 31 } + let(:collection_hash) do + { + a: { example_id: 'spec/foo/bar_spec.rb:2' }, + b: { example_id: 'spec/foo/baz_spec.rb:3', first_flaky_at: (Time.now - thirty_one_days).to_s, + last_flaky_at: (Time.now - thirty_one_days).to_s } + } + end + + let(:suite_flaky_example_report) do + { + '6e869794f4cfd2badd93eb68719371d1': { + example_id: 'spec/foo/bar_spec.rb:2', + file: 'spec/foo/bar_spec.rb', + line: 2, + description: 'hello world', + first_flaky_at: 1234, + last_flaky_at: 4321, + last_attempts_count: 3, + flaky_reports: 1, + feature_category: 'feature_category', + last_flaky_job: nil + } + } + end + + let(:flaky_examples) { RspecFlaky::FlakyExamplesCollection.new(collection_hash) } + let(:report) { described_class.new(flaky_examples) } + + before do + allow(Kernel).to receive(:warn) + end + + describe '.load' do + let!(:report_file) do + Tempfile.new(%w[rspec_flaky_report .json]).tap do |f| + f.write(JSON.pretty_generate(suite_flaky_example_report)) + f.rewind + end + end + + after do + report_file.close + report_file.unlink + end + + it 'loads the report file' do + expect(described_class.load(report_file.path).flaky_examples.to_h).to eq(suite_flaky_example_report) + end + end + + describe '.load_json' do + let(:report_json) do + JSON.pretty_generate(suite_flaky_example_report) + end + + it 'loads the report file' do + expect(described_class.load_json(report_json).flaky_examples.to_h).to eq(suite_flaky_example_report) + end + end + + describe '#initialize' do + it 'accepts a RspecFlaky::FlakyExamplesCollection' do + expect { report }.not_to raise_error + end + + it 'does not accept anything else' do + expect do + described_class.new([1, 2, + 3]) + end.to raise_error(ArgumentError, + "`flaky_examples` must be a RspecFlaky::FlakyExamplesCollection, Array given!") + end + end + + it 'delegates to #flaky_examples using SimpleDelegator' do + expect(report.__getobj__).to eq(flaky_examples) + end + + describe '#write' do + let(:report_file_path) { File.join('tmp', 'rspec_flaky_report.json') } + + before do + FileUtils.rm_f(report_file_path) + end + + after do + FileUtils.rm_f(report_file_path) + end + + context 'when RspecFlaky::Config.generate_report? is false' do + before do + allow(RspecFlaky::Config).to receive(:generate_report?).and_return(false) + end + + it 'does not write any report file' do + report.write(report_file_path) + + expect(File.exist?(report_file_path)).to be(false) + end + end + + context 'when RspecFlaky::Config.generate_report? is true' do + before do + allow(RspecFlaky::Config).to receive(:generate_report?).and_return(true) + end + + it 'delegates the writes to RspecFlaky::Report' do + report.write(report_file_path) + + expect(File.exist?(report_file_path)).to be(true) + expect(File.read(report_file_path)) + .to eq(JSON.pretty_generate(report.flaky_examples.to_h)) + end + end + end + + describe '#prune_outdated' do + it 'returns a new collection without the examples older than 30 days by default' do + new_report = flaky_examples.to_h.dup.tap { |r| r.delete(:b) } + new_flaky_examples = report.prune_outdated + + expect(new_flaky_examples).to be_a(described_class) + expect(new_flaky_examples.to_h).to eq(new_report) + expect(flaky_examples).to have_key(:b) + end + + it 'accepts a given number of days' do + new_flaky_examples = report.prune_outdated(days: 32) + + expect(new_flaky_examples.to_h).to eq(report.to_h) + end + end +end diff --git a/gems/rspec_flaky/spec/spec_helper.rb b/gems/rspec_flaky/spec/spec_helper.rb new file mode 100644 index 00000000000..72d48ee6e63 --- /dev/null +++ b/gems/rspec_flaky/spec/spec_helper.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +require "rspec-parameterized" +require "gitlab/rspec/all" +require "rspec_flaky" + +RSpec.configure do |config| + # Enable flags like --only-failures and --next-failure + config.example_status_persistence_file_path = ".rspec_status" + + # Disable RSpec exposing methods globally on `Module` and `main` + config.disable_monkey_patching! + + config.expect_with :rspec do |c| + c.syntax = :expect + end +end |