Skip to content

Instantly share code, notes, and snippets.

@chrisbloom7

chrisbloom7/README.md

Last active October 20, 2021 22:10
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save chrisbloom7/7aeae0d0c25cb4fbc2f9b2e4117bcb93 to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save chrisbloom7/7aeae0d0c25cb4fbc2f9b2e4117bcb93 to your computer and use it in GitHub Desktop.
Download ZIP

Revisions

  1. Chris Bloom revised this gist Oct 20, 2021. 1 changed file with 1 addition and 1 deletion.
    2 changes: 1 addition & 1 deletion rails-serialize-unicode-test.rb
    View file
    Original file line number Diff line number Diff line change
    @@ -133,7 +133,7 @@ def test_audited_serialized_skipped_blob_column_FAILS
    # changes after saving when the field contains unicode characters, even if
    # the changed fields did not include the blob column.
    #
    # THESE TESTS FAIL
    # THIS TEST FAILS
    def test_audited_serialized_blob_column_when_other_column_is_updated_FAILS
    assert_changes_empty AuditedSerializedReview.create!(blob: { "a" => UNICODE_CHAR }) do |record|
    record.text = "c"
  2. Chris Bloom revised this gist Oct 20, 2021. 1 changed file with 17 additions and 18 deletions.
    35 changes: 17 additions & 18 deletions rails-serialize-unicode-test.rb
    View file
    Original file line number Diff line number Diff line change
    @@ -89,25 +89,25 @@ class ReviewTest < ActiveSupport::TestCase
    #
    # THESE TESTS FAIL
    def test_audited_serialized_tinyblob_column_FAILS
    assert_changes_empty AuditedSerializedReview, { tinyblob: { "a" => UNICODE_CHAR } } do |record|
    assert_changes_empty AuditedSerializedReview.create!(tinyblob: { "a" => UNICODE_CHAR }) do |record|
    record.tinyblob["b"] = "c"
    end
    end

    def test_audited_serialized_blob_column_FAILS
    assert_changes_empty AuditedSerializedReview, { blob: { "a" => UNICODE_CHAR } } do |record|
    assert_changes_empty AuditedSerializedReview.create!(blob: { "a" => UNICODE_CHAR }) do |record|
    record.blob["b"] = "c"
    end
    end

    def test_audited_serialized_mediumblob_column_FAILS
    assert_changes_empty AuditedSerializedReview, { mediumblob: { "a" => UNICODE_CHAR } } do |record|
    assert_changes_empty AuditedSerializedReview.create!(mediumblob: { "a" => UNICODE_CHAR }) do |record|
    record.mediumblob["b"] = "c"
    end
    end

    def test_audited_serialized_longblob_column_FAILS
    assert_changes_empty AuditedSerializedReview, { longblob: { "a" => UNICODE_CHAR } } do |record|
    assert_changes_empty AuditedSerializedReview.create!(longblob: { "a" => UNICODE_CHAR }) do |record|
    record.longblob["b"] = "c"
    end
    end
    @@ -118,13 +118,13 @@ def test_audited_serialized_longblob_column_FAILS
    #
    # THESE TESTS FAIL
    def test_audited_serialized_ignored_blob_column_FAILS
    assert_changes_empty AuditedSerializedReview, { ignored_blob: { "a" => UNICODE_CHAR } } do |record|
    assert_changes_empty AuditedSerializedReview.create!(ignored_blob: { "a" => UNICODE_CHAR }) do |record|
    record.ignored_blob["b"] = "c"
    end
    end

    def test_audited_serialized_skipped_blob_column_FAILS
    assert_changes_empty AuditedSerializedReview, { skipped_blob: { "a" => UNICODE_CHAR } } do |record|
    assert_changes_empty AuditedSerializedReview.create!(skipped_blob: { "a" => UNICODE_CHAR }) do |record|
    record.skipped_blob["b"] = "c"
    end
    end
    @@ -135,7 +135,7 @@ def test_audited_serialized_skipped_blob_column_FAILS
    #
    # THESE TESTS FAIL
    def test_audited_serialized_blob_column_when_other_column_is_updated_FAILS
    assert_changes_empty AuditedSerializedReview, { blob: { "a" => UNICODE_CHAR } } do |record|
    assert_changes_empty AuditedSerializedReview.create!(blob: { "a" => UNICODE_CHAR }) do |record|
    record.text = "c"
    end
    end
    @@ -147,7 +147,7 @@ def test_audited_serialized_blob_column_when_other_column_is_updated_FAILS
    # THIS TEST PASSES
    def test_audited_serialized_blob_column_paper_trail_disabled_PASSES
    PaperTrail.enabled = false
    assert_changes_empty AuditedSerializedReview, { blob: { "a" => UNICODE_CHAR } } do |record|
    assert_changes_empty AuditedSerializedReview.create!(blob: { "a" => UNICODE_CHAR }) do |record|
    record.blob["b"] = "c"
    end
    end
    @@ -158,7 +158,7 @@ def test_audited_serialized_blob_column_paper_trail_disabled_PASSES
    #
    # THIS TEST PASSES
    def test_reloaded_audited_serialized_blob_column_PASSES
    record = assert_changes_empty AuditedSerializedReview, { blob: { "a" => UNICODE_CHAR } }, reload_after_update: true do |record|
    record = assert_changes_empty AuditedSerializedReview.create!(blob: { "a" => UNICODE_CHAR }), reload_after_update: true do |record|
    record.blob["b"] = "c"
    end
    assert_equal record.blob["a"], "\u2022"
    @@ -170,25 +170,25 @@ def test_reloaded_audited_serialized_blob_column_PASSES
    #
    # THESE TESTS PASS
    def test_audited_serialized_tinytext_column_PASSES
    assert_changes_empty AuditedSerializedReview, { tinytext: { "a" => UNICODE_CHAR } } do |record|
    assert_changes_empty AuditedSerializedReview.create!(tinytext: { "a" => UNICODE_CHAR }) do |record|
    record.tinytext["b"] = "c"
    end
    end

    def test_audited_serialized_text_column_PASSES
    assert_changes_empty AuditedSerializedReview, { text: { "a" => UNICODE_CHAR } } do |record|
    assert_changes_empty AuditedSerializedReview.create!(text: { "a" => UNICODE_CHAR }) do |record|
    record.text["b"] = "c"
    end
    end

    def test_audited_serialized_mediumtext_column_PASSES
    assert_changes_empty AuditedSerializedReview, { mediumtext: { "a" => UNICODE_CHAR } } do |record|
    assert_changes_empty AuditedSerializedReview.create!(mediumtext: { "a" => UNICODE_CHAR }) do |record|
    record.mediumtext["b"] = "c"
    end
    end

    def test_audited_serialized_longtext_column_PASSES
    assert_changes_empty AuditedSerializedReview, { longtext: { "a" => UNICODE_CHAR } } do |record|
    assert_changes_empty AuditedSerializedReview.create!(longtext: { "a" => UNICODE_CHAR }) do |record|
    record.longtext["b"] = "c"
    end
    end
    @@ -198,7 +198,7 @@ def test_audited_serialized_longtext_column_PASSES
    #
    # THIS TEST PASSES
    def test_unaudited_serialized_blob_column_PASSES
    assert_changes_empty SerializedReview, { blob: { "a" => UNICODE_CHAR } } do |record|
    assert_changes_empty SerializedReview.create!(blob: { "a" => UNICODE_CHAR }) do |record|
    record.blob["b"] = "c"
    end
    end
    @@ -208,7 +208,7 @@ def test_unaudited_serialized_blob_column_PASSES
    #
    # THIS TEST PASSES
    def test_audited_serialized_blob_column_without_unicode_PASSES
    assert_changes_empty AuditedSerializedReview, { blob: { "a" => "not unicode" } } do |record|
    assert_changes_empty AuditedSerializedReview.create!(blob: { "a" => "not unicode" }) do |record|
    record.blob["b"] = "c"
    end
    end
    @@ -218,16 +218,15 @@ def test_audited_serialized_blob_column_without_unicode_PASSES
    #
    # THIS TEST PASSES
    def test_audited_unserialized_blob_column_PASSES
    assert_changes_empty AuditedReview, { blob: UNICODE_CHAR } do |record|
    assert_changes_empty AuditedReview.create!(blob: UNICODE_CHAR) do |record|
    record.blob = "c"
    end
    end

    def assert_changes_empty(klass, starting_attributes, options = {})
    def assert_changes_empty(record, options = {})
    reload_after_update = options.delete(:reload_after_update) || false

    # Changes are empty after creation
    record = klass.create!(starting_attributes)
    assert_empty record.changes

    # Changes are populated after updating serialized column
  3. Chris Bloom renamed this gist Oct 20, 2021. 1 changed file with 0 additions and 0 deletions.
    0 rails-serialize-unicode-test-output.txt → rails-serialize-unicode-test.rb.txt
    View file
    File renamed without changes.
  4. Chris Bloom revised this gist Oct 20, 2021. 2 changed files with 190 additions and 99 deletions.
    44 changes: 44 additions & 0 deletions rails-serialize-unicode-test-output.txt
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,44 @@
    ReviewTest
    test_audited_serialized_blob_column_FAILS FAIL (0.09s)
    Expected {"blob"=>[{"a"=>"•", "b"=>"c"}, {"a"=>"•", "b"=>"c"}]} to be empty.
    /Users/chrisbloom7/src/sandbox/rails-serialize-unicode-test.rb:238:in `assert_changes_empty'
    /Users/chrisbloom7/src/sandbox/rails-serialize-unicode-test.rb:96:in `test_audited_serialized_blob_column_FAILS'

    test_audited_serialized_tinyblob_column_FAILS FAIL (0.05s)
    Expected {"tinyblob"=>[{"a"=>"•", "b"=>"c"}, {"a"=>"•", "b"=>"c"}]} to be empty.
    /Users/chrisbloom7/src/sandbox/rails-serialize-unicode-test.rb:238:in `assert_changes_empty'
    /Users/chrisbloom7/src/sandbox/rails-serialize-unicode-test.rb:90:in `test_audited_serialized_tinyblob_column_FAILS'

    test_audited_serialized_mediumtext_column_PASSES PASS (0.05s)
    test_audited_serialized_blob_column_without_unicode_PASSES PASS (0.05s)
    test_audited_serialized_longtext_column_PASSES PASS (0.05s)
    test_reloaded_audited_serialized_blob_column_PASSES PASS (0.06s)
    test_audited_serialized_blob_column_when_other_column_is_updated_FAILS FAIL (0.05s)
    Expected {"blob"=>[{"a"=>"•"}, {"a"=>"•"}]} to be empty.
    /Users/chrisbloom7/src/sandbox/rails-serialize-unicode-test.rb:238:in `assert_changes_empty'
    /Users/chrisbloom7/src/sandbox/rails-serialize-unicode-test.rb:136:in `test_audited_serialized_blob_column_when_other_column_is_updated_FAILS'

    test_audited_serialized_text_column_PASSES PASS (0.05s)
    test_audited_serialized_tinytext_column_PASSES PASS (0.05s)
    test_audited_unserialized_blob_column_PASSES PASS (0.06s)
    test_audited_serialized_blob_column_paper_trail_disabled_PASSES PASS (0.05s)
    test_audited_serialized_mediumblob_column_FAILS FAIL (0.05s)
    Expected {"mediumblob"=>[{"a"=>"•", "b"=>"c"}, {"a"=>"•", "b"=>"c"}]} to be empty.
    /Users/chrisbloom7/src/sandbox/rails-serialize-unicode-test.rb:238:in `assert_changes_empty'
    /Users/chrisbloom7/src/sandbox/rails-serialize-unicode-test.rb:102:in `test_audited_serialized_mediumblob_column_FAILS'

    test_audited_serialized_ignored_blob_column_FAILS FAIL (0.05s)
    Expected {"ignored_blob"=>[{"a"=>"•", "b"=>"c"}, {"a"=>"•", "b"=>"c"}]} to be empty.
    /Users/chrisbloom7/src/sandbox/rails-serialize-unicode-test.rb:238:in `assert_changes_empty'
    /Users/chrisbloom7/src/sandbox/rails-serialize-unicode-test.rb:119:in `test_audited_serialized_ignored_blob_column_FAILS'

    test_unaudited_serialized_blob_column_PASSES PASS (0.05s)
    test_audited_serialized_skipped_blob_column_FAILS FAIL (0.05s)
    Expected {"skipped_blob"=>[{"a"=>"•", "b"=>"c"}, {"a"=>"•", "b"=>"c"}]} to be empty.
    /Users/chrisbloom7/src/sandbox/rails-serialize-unicode-test.rb:238:in `assert_changes_empty'
    /Users/chrisbloom7/src/sandbox/rails-serialize-unicode-test.rb:125:in `test_audited_serialized_skipped_blob_column_FAILS'

    test_audited_serialized_longblob_column_FAILS FAIL (0.05s)
    Expected {"longblob"=>[{"a"=>"•", "b"=>"c"}, {"a"=>"•", "b"=>"c"}]} to be empty.
    /Users/chrisbloom7/src/sandbox/rails-serialize-unicode-test.rb:238:in `assert_changes_empty'
    /Users/chrisbloom7/src/sandbox/rails-serialize-unicode-test.rb:108:in `test_audited_serialized_longblob_column_FAILS'
    245 changes: 146 additions & 99 deletions rails-serialize-unicode-test.rb
    View file
    Original file line number Diff line number Diff line change
    @@ -1,4 +1,4 @@
    # Run with `ruby rails-serialize-unicode-test.rb`
    # frozen_string_literal: true

    require "bundler/inline"

    @@ -8,6 +8,7 @@
    gem "mysql2", "~> 0.5.3"
    gem "paper_trail", "~> 12.1.0"
    gem "minitest-reporters"
    gem "pry-byebug"
    end

    require "active_record"
    @@ -19,11 +20,19 @@
    ActiveRecord::Base.logger = nil
    ActiveRecord::Schema.define do
    create_table :reviews, force: true do |t|
    # longblob is not a default Rails column type, but Rails will defer to the
    # blob columns are not a default Rails column type, but Rails will defer to the
    # mysql2 adapter to manage the mapping
    t.tinyblob :tinyblob
    t.blob :blob
    t.mediumblob :mediumblob
    t.longblob :longblob
    t.longtext :longtext
    t.blob :ignored_blob
    t.blob :skipped_blob

    t.text :tinytext
    t.text :text
    t.text :mediumtext
    t.longtext :longtext
    end

    create_table :versions, force: true do |t|
    @@ -44,13 +53,22 @@ class Review < ActiveRecord::Base; end

    class SerializedReview < Review
    # Use default YAML serialization
    serialize :tinyblob
    serialize :blob
    serialize :mediumblob
    serialize :longblob
    serialize :longtext

    serialize :tinytext
    serialize :text
    serialize :mediumtext
    serialize :longtext

    serialize :ignored_blob
    serialize :skipped_blob
    end

    class AuditedReview < Review
    has_paper_trail
    has_paper_trail ignore: [:ignored_blob], skip: [:skipped_blob]
    end

    class AuditedSerializedReview < SerializedReview
    @@ -60,138 +78,167 @@ class AuditedSerializedReview < SerializedReview
    Minitest::Reporters.use! Minitest::Reporters::SpecReporter.new

    class ReviewTest < ActiveSupport::TestCase
    # Serialized longblob columns that use paper trail will not reset changes
    setup do
    PaperTrail.enabled = true
    end

    UNICODE_CHAR = "\u2022"

    # Serialized blob columns that use paper trail will not reset changes
    # after saving when the field contains unicode characters
    #
    # THIS TEST FAILS
    def test_audited_serialized_longblob_column_FAILS
    review = AuditedSerializedReview.create!(longblob: { "a" => "\u2022" })
    # THESE TESTS FAIL
    def test_audited_serialized_tinyblob_column_FAILS
    assert_changes_empty AuditedSerializedReview, { tinyblob: { "a" => UNICODE_CHAR } } do |record|
    record.tinyblob["b"] = "c"
    end
    end

    # Changes are empty after creation
    assert_empty review.changes
    def test_audited_serialized_blob_column_FAILS
    assert_changes_empty AuditedSerializedReview, { blob: { "a" => UNICODE_CHAR } } do |record|
    record.blob["b"] = "c"
    end
    end

    # Changes are populated after updating serialized column
    review.longblob["b"] = "c"
    refute_empty review.changes
    review.save!
    def test_audited_serialized_mediumblob_column_FAILS
    assert_changes_empty AuditedSerializedReview, { mediumblob: { "a" => UNICODE_CHAR } } do |record|
    record.mediumblob["b"] = "c"
    end
    end

    # This should have been reset after `save!` and should be empty
    assert_empty review.changes
    def test_audited_serialized_longblob_column_FAILS
    assert_changes_empty AuditedSerializedReview, { longblob: { "a" => UNICODE_CHAR } } do |record|
    record.longblob["b"] = "c"
    end
    end

    # Serialized longblob columns that use paper trail will not reset changes
    # after saving when the field contains unicode characters, BUT reloading
    # the record clears them and loads updated field content.
    # Serialized blob columns on classes that use paper trail will not reset
    # changes after saving when the field contains unicode characters, even if
    # the attribute is ignored or skipped by paper trail.
    #
    # THIS TEST PASSES
    def test_reloaded_audited_serialized_longblob_column_PASSES
    review = AuditedSerializedReview.create!(longblob: { "a" => "\u2022" })

    # Changes are empty after creation
    assert_empty review.changes
    # THESE TESTS FAIL
    def test_audited_serialized_ignored_blob_column_FAILS
    assert_changes_empty AuditedSerializedReview, { ignored_blob: { "a" => UNICODE_CHAR } } do |record|
    record.ignored_blob["b"] = "c"
    end
    end

    # Changes are populated after updating serialized column
    review.longblob["b"] = "c"
    refute_empty review.changes
    review.save!
    def test_audited_serialized_skipped_blob_column_FAILS
    assert_changes_empty AuditedSerializedReview, { skipped_blob: { "a" => UNICODE_CHAR } } do |record|
    record.skipped_blob["b"] = "c"
    end
    end

    # This should have been reset after `save!` and should be empty, but isn't
    # until after it is reloaded
    refute_empty review.changes
    assert_empty review.reload.changes
    assert_equal review.longblob["a"], "\u2022"
    assert_equal review.longblob["b"], "c"
    # Serialized blob columns on classes that use paper trail will not reset
    # changes after saving when the field contains unicode characters, even if
    # the changed fields did not include the blob column.
    #
    # THESE TESTS FAIL
    def test_audited_serialized_blob_column_when_other_column_is_updated_FAILS
    assert_changes_empty AuditedSerializedReview, { blob: { "a" => UNICODE_CHAR } } do |record|
    record.text = "c"
    end
    end

    # Serialized longTEXT columns that use paper trail WILL reset changes
    # after saving when the field contains unicode characters
    # Serialized blob columns that use paper trail WILL reset changes
    # after saving when the field contains unicode characters and PaperTrail
    # is DISABLED
    #
    # THIS TEST PASSES
    def test_audited_serialized_longtext_column_PASSES
    review = AuditedSerializedReview.create!(longtext: { "a" => "\u2022" })

    # Changes are empty after creation
    assert_empty review.changes

    # Changes are populated after updating serialized column
    review.longtext["b"] = "c"
    refute_empty review.changes
    review.save!

    # This should have been reset after `save!` and should be empty
    assert_empty review.changes
    def test_audited_serialized_blob_column_paper_trail_disabled_PASSES
    PaperTrail.enabled = false
    assert_changes_empty AuditedSerializedReview, { blob: { "a" => UNICODE_CHAR } } do |record|
    record.blob["b"] = "c"
    end
    end

    # Serialized longblob columns that DO NOT use paper trail WILL reset changes
    # after saving when the field contains unicode characters
    # Serialized blob columns that use paper trail will not reset changes
    # after saving when the field contains unicode characters, BUT reloading
    # the record clears them and loads updated field content.
    #
    # THIS TEST PASSES
    def test_unaudited_serialized_longblob_column_PASSES
    review = SerializedReview.create!(longblob: { "a" => "\u2022" })

    # Changes are empty after creation
    assert_empty review.changes

    # Changes are populated after updating serialized column
    review.longblob["b"] = "c"
    refute_empty review.changes
    review.save!

    # This should have been reset after `save!` and should be empty
    assert_empty review.changes
    def test_reloaded_audited_serialized_blob_column_PASSES
    record = assert_changes_empty AuditedSerializedReview, { blob: { "a" => UNICODE_CHAR } }, reload_after_update: true do |record|
    record.blob["b"] = "c"
    end
    assert_equal record.blob["a"], "\u2022"
    assert_equal record.blob["b"], "c"
    end

    # Serialized _text_ columns that use paper trail WILL reset changes
    # after saving when the field contains unicode characters
    #
    # THIS TEST PASSES
    # THESE TESTS PASS
    def test_audited_serialized_tinytext_column_PASSES
    assert_changes_empty AuditedSerializedReview, { tinytext: { "a" => UNICODE_CHAR } } do |record|
    record.tinytext["b"] = "c"
    end
    end

    def test_audited_serialized_text_column_PASSES
    review = AuditedSerializedReview.create!(text: { "a" => "\u2022" })
    assert_changes_empty AuditedSerializedReview, { text: { "a" => UNICODE_CHAR } } do |record|
    record.text["b"] = "c"
    end
    end

    # Changes are empty after creation
    assert_empty review.changes
    def test_audited_serialized_mediumtext_column_PASSES
    assert_changes_empty AuditedSerializedReview, { mediumtext: { "a" => UNICODE_CHAR } } do |record|
    record.mediumtext["b"] = "c"
    end
    end

    # Changes are populated after updating serialized column
    review.text["b"] = "c"
    refute_empty review.changes
    review.save!
    def test_audited_serialized_longtext_column_PASSES
    assert_changes_empty AuditedSerializedReview, { longtext: { "a" => UNICODE_CHAR } } do |record|
    record.longtext["b"] = "c"
    end
    end

    # This should have been reset after `save!` and should be empty
    assert_empty review.changes
    # Serialized blob columns that DO NOT use paper trail WILL reset changes
    # after saving when the field contains unicode characters
    #
    # THIS TEST PASSES
    def test_unaudited_serialized_blob_column_PASSES
    assert_changes_empty SerializedReview, { blob: { "a" => UNICODE_CHAR } } do |record|
    record.blob["b"] = "c"
    end
    end

    # Serialized longblob columns that use paper trail WILL reset changes
    # Serialized blob columns that use paper trail WILL reset changes
    # after saving when the field DOES NOT contain unicode characters
    #
    # THIS TEST PASSES
    def test_audited_serialized_longblob_column_without_unicode_PASSES
    review = AuditedSerializedReview.create!(longblob: { "a" => "2022" })

    # Changes are empty after creation
    assert_empty review.changes

    # Changes are populated after updating serialized column
    review.longblob["b"] = "c"
    refute_empty review.changes
    review.save!

    # This should have been reset after `save!` and should be empty
    assert_empty review.changes
    def test_audited_serialized_blob_column_without_unicode_PASSES
    assert_changes_empty AuditedSerializedReview, { blob: { "a" => "not unicode" } } do |record|
    record.blob["b"] = "c"
    end
    end

    # UNserialized longblob columns that use paper trail WILL reset changes
    # UN-serialized blob columns that use paper trail WILL reset changes
    # after saving when the field contains unicode characters
    #
    # THIS TEST PASSES
    def test_audited_unserialized_longblob_column_PASSES
    review = AuditedReview.create!(longblob: "\u2022")
    assert_empty review.changes
    def test_audited_unserialized_blob_column_PASSES
    assert_changes_empty AuditedReview, { blob: UNICODE_CHAR } do |record|
    record.blob = "c"
    end
    end

    def assert_changes_empty(klass, starting_attributes, options = {})
    reload_after_update = options.delete(:reload_after_update) || false

    # Changes are empty after creation
    record = klass.create!(starting_attributes)
    assert_empty record.changes

    # Changes are populated after updating serialized column
    yield(record)
    refute_empty record.changes
    record.save!

    review.longblob = "something else"
    refute_empty review.changes
    review.save!
    # This should have been reset after `save!` and should be empty
    record.reload if reload_after_update
    assert_empty record.changes

    # This should have been reset after `save!`
    assert_empty review.changes
    record
    end
    end
  5. Chris Bloom revised this gist Oct 14, 2021. 1 changed file with 11 additions and 0 deletions.
    11 changes: 11 additions & 0 deletions README.md
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,11 @@
    I've found a bug that is specific to the following scenario:

    - Using MySQL
    - Using a `longblob` column type
    - Using a [serialized attribute](https://api.rubyonrails.org/classes/ActiveModel/Serialization.html)
    - Using `paper_trail` gem
    - Serialized field contains unicode character

    Under those conditions, after saving changes to the serialized field, `record#changed?` still reports true, and `record#changes` contains an entry for the serialized field where both the before and after elements are identical. Calling `record#reload` clears the changes and loads the record with the changed value. If `paper_trail` is removed from the scenario, the ActiveModel [attribute mutation tracking](https://github.com/rails/rails/blob/v6.1.4.1/activemodel/lib/active_model/attribute_mutation_tracker.rb) works as expected.

    Documented in https://github.com/paper-trail-gem/paper_trail/issues/1348
  6. Chris Bloom revised this gist Oct 14, 2021. 2 changed files with 48 additions and 2 deletions.
    45 changes: 45 additions & 0 deletions rails-serialize-unicode-test.rb
    View file
    Original file line number Diff line number Diff line change
    @@ -22,6 +22,7 @@
    # longblob is not a default Rails column type, but Rails will defer to the
    # mysql2 adapter to manage the mapping
    t.longblob :longblob
    t.longtext :longtext
    t.text :text
    end

    @@ -44,6 +45,7 @@ class Review < ActiveRecord::Base; end
    class SerializedReview < Review
    # Use default YAML serialization
    serialize :longblob
    serialize :longtext
    serialize :text
    end

    @@ -77,6 +79,49 @@ def test_audited_serialized_longblob_column_FAILS
    assert_empty review.changes
    end

    # Serialized longblob columns that use paper trail will not reset changes
    # after saving when the field contains unicode characters, BUT reloading
    # the record clears them and loads updated field content.
    #
    # THIS TEST PASSES
    def test_reloaded_audited_serialized_longblob_column_PASSES
    review = AuditedSerializedReview.create!(longblob: { "a" => "\u2022" })

    # Changes are empty after creation
    assert_empty review.changes

    # Changes are populated after updating serialized column
    review.longblob["b"] = "c"
    refute_empty review.changes
    review.save!

    # This should have been reset after `save!` and should be empty, but isn't
    # until after it is reloaded
    refute_empty review.changes
    assert_empty review.reload.changes
    assert_equal review.longblob["a"], "\u2022"
    assert_equal review.longblob["b"], "c"
    end

    # Serialized longTEXT columns that use paper trail WILL reset changes
    # after saving when the field contains unicode characters
    #
    # THIS TEST PASSES
    def test_audited_serialized_longtext_column_PASSES
    review = AuditedSerializedReview.create!(longtext: { "a" => "\u2022" })

    # Changes are empty after creation
    assert_empty review.changes

    # Changes are populated after updating serialized column
    review.longtext["b"] = "c"
    refute_empty review.changes
    review.save!

    # This should have been reset after `save!` and should be empty
    assert_empty review.changes
    end

    # Serialized longblob columns that DO NOT use paper trail WILL reset changes
    # after saving when the field contains unicode characters
    #
    5 changes: 3 additions & 2 deletions structure.sql
    View file
    Original file line number Diff line number Diff line change
    @@ -15,9 +15,10 @@ CREATE TABLE `ar_internal_metadata` (
    CREATE TABLE `reviews` (
    `id` bigint(20) NOT NULL AUTO_INCREMENT,
    `longblob` longblob,
    `longtext` longtext,
    `text` text,
    PRIMARY KEY (`id`)
    ) ENGINE=InnoDB AUTO_INCREMENT=6 DEFAULT CHARSET=utf8;
    ) ENGINE=InnoDB AUTO_INCREMENT=8 DEFAULT CHARSET=utf8;
    /*!40101 SET character_set_client = @saved_cs_client */;
    /*!40101 SET @saved_cs_client = @@character_set_client */;
    /*!50503 SET character_set_client = utf8mb4 */;
    @@ -32,5 +33,5 @@ CREATE TABLE `versions` (
    `created_at` datetime DEFAULT NULL,
    PRIMARY KEY (`id`),
    KEY `index_versions_on_item_type_and_item_id` (`item_type`,`item_id`)
    ) ENGINE=InnoDB AUTO_INCREMENT=9 DEFAULT CHARSET=utf8;
    ) ENGINE=InnoDB AUTO_INCREMENT=13 DEFAULT CHARSET=utf8;
    /*!40101 SET character_set_client = @saved_cs_client */;
  7. Chris Bloom revised this gist Oct 14, 2021. 3 changed files with 163 additions and 65 deletions.
    55 changes: 0 additions & 55 deletions paper_trail-serialize-unicode-test.rb
    View file
    Original file line number Diff line number Diff line change
    @@ -1,55 +0,0 @@
    # Run with `ruby rails-serialize-unicode-test.rb`

    require "bundler/inline"

    gemfile true do
    gem "rails", "~> 6.1.4.1"
    gem "mysql2", "~> 0.5.3"
    gem "paper_trail", "~> 12.1.0"
    gem "pry-byebug"
    end

    require "active_record"
    require "paper_trail"
    require "paper_trail/frameworks/active_record"
    require "minitest/autorun"

    ActiveRecord::Base.establish_connection(adapter: "mysql2", database: "railstestdb")
    ActiveRecord::Base.logger = Logger.new(STDOUT)

    ActiveRecord::Schema.define do
    create_table :advisory_reviews, force: true do |t|
    t.longblob :advisory_payload, null: false
    end

    create_table :versions, force: true do |t|
    t.string :item_type, null: false, limit: 191
    t.integer :item_id, null: false
    t.string :event, null: false
    t.string :whodunnit
    t.text :object, limit: 1_073_741_823
    t.text :object_changes, limit: 1_073_741_823
    t.datetime :created_at
    end
    add_index :versions, %i[item_type item_id]

    end

    class AdvisoryReview < ActiveRecord::Base
    has_paper_trail
    serialize :advisory_payload
    end

    class AdvisoryReviewTest < Minitest::Test
    def test_unicode_changes
    advisory_review = AdvisoryReview.create!(advisory_payload: { "description" => "\u2022" })
    assert_empty advisory_review.changes

    advisory_review.advisory_payload["not_the_description"] = "foo"
    refute_empty advisory_review.changes
    advisory_review.save!

    # This should have been reset after `save!`
    assert_empty advisory_review.changes
    end
    end
    152 changes: 152 additions & 0 deletions rails-serialize-unicode-test.rb
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,152 @@
    # Run with `ruby rails-serialize-unicode-test.rb`

    require "bundler/inline"

    gemfile true do
    source "https://rubygems.org"
    gem "activerecord", "~> 6.1.4.1"
    gem "mysql2", "~> 0.5.3"
    gem "paper_trail", "~> 12.1.0"
    gem "minitest-reporters"
    end

    require "active_record"
    require "paper_trail"
    require "minitest/autorun"
    require "logger"

    ActiveRecord::Base.establish_connection(adapter: "mysql2", database: "railstestdb")
    ActiveRecord::Base.logger = nil
    ActiveRecord::Schema.define do
    create_table :reviews, force: true do |t|
    # longblob is not a default Rails column type, but Rails will defer to the
    # mysql2 adapter to manage the mapping
    t.longblob :longblob
    t.text :text
    end

    create_table :versions, force: true do |t|
    t.string :item_type, null: false
    t.integer :item_id, null: false
    t.string :event, null: false
    t.string :whodunnit
    t.text :object, limit: 1_073_741_823
    t.text :object_changes, limit: 1_073_741_823
    t.datetime :created_at
    end
    add_index :versions, %i[item_type item_id]
    end

    # ActiveRecord::Base.logger = Logger.new(STDOUT)

    class Review < ActiveRecord::Base; end

    class SerializedReview < Review
    # Use default YAML serialization
    serialize :longblob
    serialize :text
    end

    class AuditedReview < Review
    has_paper_trail
    end

    class AuditedSerializedReview < SerializedReview
    has_paper_trail
    end

    Minitest::Reporters.use! Minitest::Reporters::SpecReporter.new

    class ReviewTest < ActiveSupport::TestCase
    # Serialized longblob columns that use paper trail will not reset changes
    # after saving when the field contains unicode characters
    #
    # THIS TEST FAILS
    def test_audited_serialized_longblob_column_FAILS
    review = AuditedSerializedReview.create!(longblob: { "a" => "\u2022" })

    # Changes are empty after creation
    assert_empty review.changes

    # Changes are populated after updating serialized column
    review.longblob["b"] = "c"
    refute_empty review.changes
    review.save!

    # This should have been reset after `save!` and should be empty
    assert_empty review.changes
    end

    # Serialized longblob columns that DO NOT use paper trail WILL reset changes
    # after saving when the field contains unicode characters
    #
    # THIS TEST PASSES
    def test_unaudited_serialized_longblob_column_PASSES
    review = SerializedReview.create!(longblob: { "a" => "\u2022" })

    # Changes are empty after creation
    assert_empty review.changes

    # Changes are populated after updating serialized column
    review.longblob["b"] = "c"
    refute_empty review.changes
    review.save!

    # This should have been reset after `save!` and should be empty
    assert_empty review.changes
    end

    # Serialized _text_ columns that use paper trail WILL reset changes
    # after saving when the field contains unicode characters
    #
    # THIS TEST PASSES
    def test_audited_serialized_text_column_PASSES
    review = AuditedSerializedReview.create!(text: { "a" => "\u2022" })

    # Changes are empty after creation
    assert_empty review.changes

    # Changes are populated after updating serialized column
    review.text["b"] = "c"
    refute_empty review.changes
    review.save!

    # This should have been reset after `save!` and should be empty
    assert_empty review.changes
    end

    # Serialized longblob columns that use paper trail WILL reset changes
    # after saving when the field DOES NOT contain unicode characters
    #
    # THIS TEST PASSES
    def test_audited_serialized_longblob_column_without_unicode_PASSES
    review = AuditedSerializedReview.create!(longblob: { "a" => "2022" })

    # Changes are empty after creation
    assert_empty review.changes

    # Changes are populated after updating serialized column
    review.longblob["b"] = "c"
    refute_empty review.changes
    review.save!

    # This should have been reset after `save!` and should be empty
    assert_empty review.changes
    end

    # UNserialized longblob columns that use paper trail WILL reset changes
    # after saving when the field contains unicode characters
    #
    # THIS TEST PASSES
    def test_audited_unserialized_longblob_column_PASSES
    review = AuditedReview.create!(longblob: "\u2022")
    assert_empty review.changes

    review.longblob = "something else"
    refute_empty review.changes
    review.save!

    # This should have been reset after `save!`
    assert_empty review.changes
    end
    end
    21 changes: 11 additions & 10 deletions structure.sql
    View file
    Original file line number Diff line number Diff line change
    @@ -1,13 +1,5 @@
    -- $ mysqldump --compact=ON --set-gtid-purged=OFF --no-data=ON railstestdb

    /*!40101 SET @saved_cs_client = @@character_set_client */;
    /*!50503 SET character_set_client = utf8mb4 */;
    CREATE TABLE `advisory_reviews` (
    `id` bigint(20) NOT NULL AUTO_INCREMENT,
    `advisory_payload` longblob NOT NULL,
    PRIMARY KEY (`id`)
    ) ENGINE=InnoDB AUTO_INCREMENT=2 DEFAULT CHARSET=utf8;
    /*!40101 SET character_set_client = @saved_cs_client */;
    /*!40101 SET @saved_cs_client = @@character_set_client */;
    /*!50503 SET character_set_client = utf8mb4 */;
    CREATE TABLE `ar_internal_metadata` (
    @@ -20,9 +12,18 @@ CREATE TABLE `ar_internal_metadata` (
    /*!40101 SET character_set_client = @saved_cs_client */;
    /*!40101 SET @saved_cs_client = @@character_set_client */;
    /*!50503 SET character_set_client = utf8mb4 */;
    CREATE TABLE `reviews` (
    `id` bigint(20) NOT NULL AUTO_INCREMENT,
    `longblob` longblob,
    `text` text,
    PRIMARY KEY (`id`)
    ) ENGINE=InnoDB AUTO_INCREMENT=6 DEFAULT CHARSET=utf8;
    /*!40101 SET character_set_client = @saved_cs_client */;
    /*!40101 SET @saved_cs_client = @@character_set_client */;
    /*!50503 SET character_set_client = utf8mb4 */;
    CREATE TABLE `versions` (
    `id` bigint(20) NOT NULL AUTO_INCREMENT,
    `item_type` varchar(191) NOT NULL,
    `item_type` varchar(255) NOT NULL,
    `item_id` int(11) NOT NULL,
    `event` varchar(255) NOT NULL,
    `whodunnit` varchar(255) DEFAULT NULL,
    @@ -31,5 +32,5 @@ CREATE TABLE `versions` (
    `created_at` datetime DEFAULT NULL,
    PRIMARY KEY (`id`),
    KEY `index_versions_on_item_type_and_item_id` (`item_type`,`item_id`)
    ) ENGINE=InnoDB AUTO_INCREMENT=3 DEFAULT CHARSET=utf8;
    ) ENGINE=InnoDB AUTO_INCREMENT=9 DEFAULT CHARSET=utf8;
    /*!40101 SET character_set_client = @saved_cs_client */;
  8. Chris Bloom revised this gist Oct 14, 2021. 2 changed files with 0 additions and 0 deletions.
    0 rails-serialize-unicode-test.rb → paper_trail-serialize-unicode-test.rb
    View file
    File renamed without changes.
    0 mysqldump.sql → structure.sql
    View file
    File renamed without changes.
  9. Chris Bloom revised this gist Oct 14, 2021. 2 changed files with 36 additions and 93 deletions.
    17 changes: 16 additions & 1 deletion dump.sql → mysqldump.sql
    View file
    Original file line number Diff line number Diff line change
    @@ -1,4 +1,4 @@
    -- $ mysqldump --no-data railstestdb > dump.sql
    -- $ mysqldump --compact=ON --set-gtid-purged=OFF --no-data=ON railstestdb

    /*!40101 SET @saved_cs_client = @@character_set_client */;
    /*!50503 SET character_set_client = utf8mb4 */;
    @@ -17,4 +17,19 @@ CREATE TABLE `ar_internal_metadata` (
    `updated_at` datetime(6) NOT NULL,
    PRIMARY KEY (`key`)
    ) ENGINE=InnoDB DEFAULT CHARSET=utf8;
    /*!40101 SET character_set_client = @saved_cs_client */;
    /*!40101 SET @saved_cs_client = @@character_set_client */;
    /*!50503 SET character_set_client = utf8mb4 */;
    CREATE TABLE `versions` (
    `id` bigint(20) NOT NULL AUTO_INCREMENT,
    `item_type` varchar(191) NOT NULL,
    `item_id` int(11) NOT NULL,
    `event` varchar(255) NOT NULL,
    `whodunnit` varchar(255) DEFAULT NULL,
    `object` longtext,
    `object_changes` longtext,
    `created_at` datetime DEFAULT NULL,
    PRIMARY KEY (`id`),
    KEY `index_versions_on_item_type_and_item_id` (`item_type`,`item_id`)
    ) ENGINE=InnoDB AUTO_INCREMENT=3 DEFAULT CHARSET=utf8;
    /*!40101 SET character_set_client = @saved_cs_client */;
    112 changes: 20 additions & 92 deletions rails-serialize-unicode-test.rb
    View file
    Original file line number Diff line number Diff line change
    @@ -3,14 +3,16 @@
    require "bundler/inline"

    gemfile true do
    # Rails
    gem "rails", "~> 6.1.4.1"

    # Database
    gem "mysql2", "~> 0.5.3"
    gem "paper_trail", "~> 12.1.0"
    gem "pry-byebug"
    end

    require "active_record"
    require "paper_trail"
    require "paper_trail/frameworks/active_record"
    require "minitest/autorun"

    ActiveRecord::Base.establish_connection(adapter: "mysql2", database: "railstestdb")
    ActiveRecord::Base.logger = Logger.new(STDOUT)
    @@ -19,106 +21,32 @@
    create_table :advisory_reviews, force: true do |t|
    t.longblob :advisory_payload, null: false
    end
    end

    require "yaml"

    module NormalYAML
    def self.load(yaml)
    return nil if yaml.nil?

    YAML.safe_load(yaml, [BigDecimal, Date, Time])
    end

    def self.dump(data)
    return nil if data.nil?

    YAML.dump(normalize(data))
    end

    def self.normalize(data)
    case data
    when Array
    [].tap do |normalized|
    data.each do |element|
    normalized << normalize(element)
    end
    end
    when Hash
    {}.tap do |normalized|
    data.sort.each do |key, value|
    unless key.is_a?(String) || key.is_a?(Symbol) || key.is_a?(Integer)
    raise ArgumentError, "NormalYAML requires String, Symbol, or Integer keys."
    end

    new_key =
    case key
    when Symbol then key.to_s
    when /\A\d+\z/ then key.to_i
    else key
    end

    normalized[new_key] = normalize(value)
    end
    end
    when TrueClass, FalseClass, NilClass, Integer, Float
    data
    when String
    String.new(data)
    when BigDecimal
    BigDecimal(data)
    when Date
    Date.new(data.year, data.month, data.day)
    when Time
    utc = data.utc
    sec_with_frac = utc.sec + utc.subsec
    Time.utc(utc.year, utc.month, utc.day, utc.hour, utc.min, sec_with_frac)
    else
    raise ArgumentError,
    "NormalYAML does not support #{data.class.name} serialization."
    end
    rescue StandardError => error
    Rails.logger.debug { "Failed to normalize this data:\n---\n#{data}\n---" }
    raise error
    create_table :versions, force: true do |t|
    t.string :item_type, null: false, limit: 191
    t.integer :item_id, null: false
    t.string :event, null: false
    t.string :whodunnit
    t.text :object, limit: 1_073_741_823
    t.text :object_changes, limit: 1_073_741_823
    t.datetime :created_at
    end
    add_index :versions, %i[item_type item_id]

    def self.normal?(data)
    normalize(data) == data
    rescue StandardError
    false
    end

    def self.valid?(yaml)
    begin
    self.load(yaml)
    rescue StandardError
    false
    end
    true
    end
    end

    class AdvisoryReview < ActiveRecord::Base
    serialize :advisory_payload, NormalYAML
    validates :advisory_payload, exclusion: { in: [nil] } # Forbid nil, allow {}
    def description
    advisory_payload["description"].presence
    end
    has_paper_trail
    serialize :advisory_payload
    end

    require "minitest/autorun"

    class AdvisoryReviewTest < Minitest::Test
    def test_unicode_change
    advisory_review = AdvisoryReview.create!(advisory_payload: { "description" => "n’t" })

    # Access description to ensure encoding handlers run
    assert_equal 3, advisory_review.reload.description.length

    def test_unicode_changes
    advisory_review = AdvisoryReview.create!(advisory_payload: { "description" => "\u2022" })
    assert_empty advisory_review.changes

    advisory_review.advisory_payload["description"] = "n’t"
    assert_predicate advisory_review, :changed?
    advisory_review.advisory_payload["not_the_description"] = "foo"
    refute_empty advisory_review.changes
    advisory_review.save!

    # This should have been reset after `save!`
  10. Chris Bloom revised this gist Oct 14, 2021. 1 changed file with 1 addition and 3 deletions.
    4 changes: 1 addition & 3 deletions dump.sql
    View file
    Original file line number Diff line number Diff line change
    @@ -8,7 +8,6 @@ CREATE TABLE `advisory_reviews` (
    PRIMARY KEY (`id`)
    ) ENGINE=InnoDB AUTO_INCREMENT=2 DEFAULT CHARSET=utf8;
    /*!40101 SET character_set_client = @saved_cs_client */;
    INSERT INTO `advisory_reviews` VALUES (1,_binary '---\ndescription: n’t\n');
    /*!40101 SET @saved_cs_client = @@character_set_client */;
    /*!50503 SET character_set_client = utf8mb4 */;
    CREATE TABLE `ar_internal_metadata` (
    @@ -18,5 +17,4 @@ CREATE TABLE `ar_internal_metadata` (
    `updated_at` datetime(6) NOT NULL,
    PRIMARY KEY (`key`)
    ) ENGINE=InnoDB DEFAULT CHARSET=utf8;
    /*!40101 SET character_set_client = @saved_cs_client */;
    INSERT INTO `ar_internal_metadata` VALUES ('environment','development','2021-10-13 22:29:15.077180','2021-10-13 22:29:15.077180');
    /*!40101 SET character_set_client = @saved_cs_client */;
  11. Chris Bloom revised this gist Oct 14, 2021. 2 changed files with 22 additions and 71 deletions.
    22 changes: 22 additions & 0 deletions dump.sql
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,22 @@
    -- $ mysqldump --no-data railstestdb > dump.sql

    /*!40101 SET @saved_cs_client = @@character_set_client */;
    /*!50503 SET character_set_client = utf8mb4 */;
    CREATE TABLE `advisory_reviews` (
    `id` bigint(20) NOT NULL AUTO_INCREMENT,
    `advisory_payload` longblob NOT NULL,
    PRIMARY KEY (`id`)
    ) ENGINE=InnoDB AUTO_INCREMENT=2 DEFAULT CHARSET=utf8;
    /*!40101 SET character_set_client = @saved_cs_client */;
    INSERT INTO `advisory_reviews` VALUES (1,_binary '---\ndescription: n’t\n');
    /*!40101 SET @saved_cs_client = @@character_set_client */;
    /*!50503 SET character_set_client = utf8mb4 */;
    CREATE TABLE `ar_internal_metadata` (
    `key` varchar(255) NOT NULL,
    `value` varchar(255) DEFAULT NULL,
    `created_at` datetime(6) NOT NULL,
    `updated_at` datetime(6) NOT NULL,
    PRIMARY KEY (`key`)
    ) ENGINE=InnoDB DEFAULT CHARSET=utf8;
    /*!40101 SET character_set_client = @saved_cs_client */;
    INSERT INTO `ar_internal_metadata` VALUES ('environment','development','2021-10-13 22:29:15.077180','2021-10-13 22:29:15.077180');
    71 changes: 0 additions & 71 deletions structure.sql
    View file
    Original file line number Diff line number Diff line change
    @@ -1,71 +0,0 @@
    -- $ mysqldump --no-data railstestdb
    --
    --
    --
    -- MySQL dump 10.13 Distrib 8.0.26, for macos10.15 (x86_64)
    --
    -- Host: localhost Database: railstestdb
    -- ------------------------------------------------------
    -- Server version 5.7.35-log

    /*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
    /*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
    /*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
    /*!50503 SET NAMES utf8mb4 */;
    /*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
    /*!40103 SET TIME_ZONE='+00:00' */;
    /*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */;
    /*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;
    /*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */;
    /*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;
    Warning: A partial dump from a server that has GTIDs will by default include the GTIDs of all transactions, even those that changed suppressed parts of the database. If you don't want to restore GTIDs, pass --set-gtid-purged=OFF. To make a complete dump, pass --all-databases --triggers --routines --events.
    SET @MYSQLDUMP_TEMP_LOG_BIN = @@SESSION.SQL_LOG_BIN;
    SET @@SESSION.SQL_LOG_BIN= 0;
    --
    -- GTID state at the beginning of the backup
    --
    SET @@GLOBAL.GTID_PURGED=/*!80000 '+'*/ '58d9a542-ba3b-11e9-ab1a-d6379339644c:1-1448871';
    --
    -- Table structure for table `advisory_reviews`
    --
    DROP TABLE IF EXISTS `advisory_reviews`;
    /*!40101 SET @saved_cs_client = @@character_set_client */;
    /*!50503 SET character_set_client = utf8mb4 */;
    CREATE TABLE `advisory_reviews` (
    `id` bigint(20) NOT NULL AUTO_INCREMENT,
    `advisory_payload` longblob NOT NULL,
    PRIMARY KEY (`id`)
    ) ENGINE=InnoDB AUTO_INCREMENT=2 DEFAULT CHARSET=utf8;
    /*!40101 SET character_set_client = @saved_cs_client */;
    --
    -- Table structure for table `ar_internal_metadata`
    --
    DROP TABLE IF EXISTS `ar_internal_metadata`;
    /*!40101 SET @saved_cs_client = @@character_set_client */;
    /*!50503 SET character_set_client = utf8mb4 */;
    CREATE TABLE `ar_internal_metadata` (
    `key` varchar(255) NOT NULL,
    `value` varchar(255) DEFAULT NULL,
    `created_at` datetime(6) NOT NULL,
    `updated_at` datetime(6) NOT NULL,
    PRIMARY KEY (`key`)
    ) ENGINE=InnoDB DEFAULT CHARSET=utf8;
    /*!40101 SET character_set_client = @saved_cs_client */;
    SET @@SESSION.SQL_LOG_BIN = @MYSQLDUMP_TEMP_LOG_BIN;
    /*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
    /*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
    /*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */;
    /*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */;
    /*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
    /*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
    /*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
    /*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
    -- Dump completed on 2021-10-13 21:53:51
  12. Chris Bloom revised this gist Oct 14, 2021. 1 changed file with 3 additions and 3 deletions.
    6 changes: 3 additions & 3 deletions rails-serialize-unicode-test.rb
    View file
    Original file line number Diff line number Diff line change
    @@ -110,14 +110,14 @@ def description

    class AdvisoryReviewTest < Minitest::Test
    def test_unicode_change
    advisory_review = AdvisoryReview.create!(advisory_payload: { "description" => "\u2022" })
    advisory_review = AdvisoryReview.create!(advisory_payload: { "description" => "n’t" })

    # Access description to ensure encoding handlers run
    assert_equal 1, advisory_review.reload.description.length
    assert_equal 3, advisory_review.reload.description.length

    assert_empty advisory_review.changes

    advisory_review.advisory_payload["description"] = "\u2019"
    advisory_review.advisory_payload["description"] = "n’t"
    assert_predicate advisory_review, :changed?
    advisory_review.save!

  13. Chris Bloom revised this gist Oct 14, 2021. 1 changed file with 71 additions and 0 deletions.
    71 changes: 71 additions & 0 deletions structure.sql
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,71 @@
    -- $ mysqldump --no-data railstestdb
    --
    --
    --
    -- MySQL dump 10.13 Distrib 8.0.26, for macos10.15 (x86_64)
    --
    -- Host: localhost Database: railstestdb
    -- ------------------------------------------------------
    -- Server version 5.7.35-log

    /*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
    /*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
    /*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
    /*!50503 SET NAMES utf8mb4 */;
    /*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
    /*!40103 SET TIME_ZONE='+00:00' */;
    /*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */;
    /*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;
    /*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */;
    /*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;
    Warning: A partial dump from a server that has GTIDs will by default include the GTIDs of all transactions, even those that changed suppressed parts of the database. If you don't want to restore GTIDs, pass --set-gtid-purged=OFF. To make a complete dump, pass --all-databases --triggers --routines --events.
    SET @MYSQLDUMP_TEMP_LOG_BIN = @@SESSION.SQL_LOG_BIN;
    SET @@SESSION.SQL_LOG_BIN= 0;
    --
    -- GTID state at the beginning of the backup
    --
    SET @@GLOBAL.GTID_PURGED=/*!80000 '+'*/ '58d9a542-ba3b-11e9-ab1a-d6379339644c:1-1448871';
    --
    -- Table structure for table `advisory_reviews`
    --
    DROP TABLE IF EXISTS `advisory_reviews`;
    /*!40101 SET @saved_cs_client = @@character_set_client */;
    /*!50503 SET character_set_client = utf8mb4 */;
    CREATE TABLE `advisory_reviews` (
    `id` bigint(20) NOT NULL AUTO_INCREMENT,
    `advisory_payload` longblob NOT NULL,
    PRIMARY KEY (`id`)
    ) ENGINE=InnoDB AUTO_INCREMENT=2 DEFAULT CHARSET=utf8;
    /*!40101 SET character_set_client = @saved_cs_client */;
    --
    -- Table structure for table `ar_internal_metadata`
    --
    DROP TABLE IF EXISTS `ar_internal_metadata`;
    /*!40101 SET @saved_cs_client = @@character_set_client */;
    /*!50503 SET character_set_client = utf8mb4 */;
    CREATE TABLE `ar_internal_metadata` (
    `key` varchar(255) NOT NULL,
    `value` varchar(255) DEFAULT NULL,
    `created_at` datetime(6) NOT NULL,
    `updated_at` datetime(6) NOT NULL,
    PRIMARY KEY (`key`)
    ) ENGINE=InnoDB DEFAULT CHARSET=utf8;
    /*!40101 SET character_set_client = @saved_cs_client */;
    SET @@SESSION.SQL_LOG_BIN = @MYSQLDUMP_TEMP_LOG_BIN;
    /*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
    /*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
    /*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */;
    /*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */;
    /*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
    /*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
    /*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
    /*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
    -- Dump completed on 2021-10-13 21:53:51
  14. Chris Bloom revised this gist Oct 14, 2021. 1 changed file with 2 additions and 0 deletions.
    2 changes: 2 additions & 0 deletions rails-serialize-unicode-test.rb
    View file
    Original file line number Diff line number Diff line change
    @@ -1,3 +1,5 @@
    # Run with `ruby rails-serialize-unicode-test.rb`

    require "bundler/inline"

    gemfile true do
  15. Chris Bloom created this gist Oct 14, 2021.
    125 changes: 125 additions & 0 deletions rails-serialize-unicode-test.rb
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,125 @@
    require "bundler/inline"

    gemfile true do
    # Rails
    gem "rails", "~> 6.1.4.1"

    # Database
    gem "mysql2", "~> 0.5.3"
    end

    require "active_record"

    ActiveRecord::Base.establish_connection(adapter: "mysql2", database: "railstestdb")
    ActiveRecord::Base.logger = Logger.new(STDOUT)

    ActiveRecord::Schema.define do
    create_table :advisory_reviews, force: true do |t|
    t.longblob :advisory_payload, null: false
    end
    end

    require "yaml"

    module NormalYAML
    def self.load(yaml)
    return nil if yaml.nil?

    YAML.safe_load(yaml, [BigDecimal, Date, Time])
    end

    def self.dump(data)
    return nil if data.nil?

    YAML.dump(normalize(data))
    end

    def self.normalize(data)
    case data
    when Array
    [].tap do |normalized|
    data.each do |element|
    normalized << normalize(element)
    end
    end
    when Hash
    {}.tap do |normalized|
    data.sort.each do |key, value|
    unless key.is_a?(String) || key.is_a?(Symbol) || key.is_a?(Integer)
    raise ArgumentError, "NormalYAML requires String, Symbol, or Integer keys."
    end

    new_key =
    case key
    when Symbol then key.to_s
    when /\A\d+\z/ then key.to_i
    else key
    end

    normalized[new_key] = normalize(value)
    end
    end
    when TrueClass, FalseClass, NilClass, Integer, Float
    data
    when String
    String.new(data)
    when BigDecimal
    BigDecimal(data)
    when Date
    Date.new(data.year, data.month, data.day)
    when Time
    utc = data.utc
    sec_with_frac = utc.sec + utc.subsec
    Time.utc(utc.year, utc.month, utc.day, utc.hour, utc.min, sec_with_frac)
    else
    raise ArgumentError,
    "NormalYAML does not support #{data.class.name} serialization."
    end
    rescue StandardError => error
    Rails.logger.debug { "Failed to normalize this data:\n---\n#{data}\n---" }
    raise error
    end

    def self.normal?(data)
    normalize(data) == data
    rescue StandardError
    false
    end

    def self.valid?(yaml)
    begin
    self.load(yaml)
    rescue StandardError
    false
    end
    true
    end
    end

    class AdvisoryReview < ActiveRecord::Base
    serialize :advisory_payload, NormalYAML
    validates :advisory_payload, exclusion: { in: [nil] } # Forbid nil, allow {}
    def description
    advisory_payload["description"].presence
    end
    end

    require "minitest/autorun"

    class AdvisoryReviewTest < Minitest::Test
    def test_unicode_change
    advisory_review = AdvisoryReview.create!(advisory_payload: { "description" => "\u2022" })

    # Access description to ensure encoding handlers run
    assert_equal 1, advisory_review.reload.description.length

    assert_empty advisory_review.changes

    advisory_review.advisory_payload["description"] = "\u2019"
    assert_predicate advisory_review, :changed?
    advisory_review.save!

    # This should have been reset after `save!`
    assert_empty advisory_review.changes
    end
    end