From 73a5f6ed45172c70def7d1d11a1a3cc92040a126 Mon Sep 17 00:00:00 2001 From: John Paul Ashenfelter Date: Sun, 14 Sep 2025 10:33:43 -0400 Subject: [PATCH 01/10] Force charset to utf8mb4 and utf8mb4_unicode_ci charset/collation --- CONTRIBUTING.md | 7 ++++++ config/database.yml | 25 +++++++++++-------- ...0251127143038_fix_charset_and_collation.rb | 22 ++++++++++++++++ db/schema.rb | 3 +++ 4 files changed, 47 insertions(+), 10 deletions(-) create mode 100644 db/migrate/20251127143038_fix_charset_and_collation.rb diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8da69d05e..cc113f6f1 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -67,6 +67,13 @@ For Docker-based development (recommended): 3. Visit to see the AWBW portal page 4. Log in as a sample user with the default [credentials](#credentials) +### Database encoding +We were getting a different schema encoding on Mac vs Linux (utf8mb4 vs utf8) which was causing some merge conflicts. +To ensure consistent encoding across environments, we have set MySQL database to use `utf8mb4` encoding by default by +overriding DATABASE_URL in `.env` and `.env.example` files: + +``` + ## Dev seeds - Running mise should have run `rake db:seed` diff --git a/config/database.yml b/config/database.yml index cb4f26074..b43a36995 100644 --- a/config/database.yml +++ b/config/database.yml @@ -4,18 +4,23 @@ base: &base collation: utf8mb4_unicode_ci url: <%= ENV.fetch("DATABASE_URL", "trilogy://root@localhost") %> +default: &default + adapter: trilogy + encoding: utf8mb4 # <-- forces UTF-8 everywhere + collation: utf8mb4_unicode_ci # <-- portable across MySQL 5.7/8.0 and MariaDB + pool: <%= ENV.fetch("RAILS_MAX_THREADS", 5) %> + user: root + password: + host: 127.0.0.1 + variables: + # Make sure the session uses utf8mb4 too: + collation_connection: utf8mb4_unicode_ci + sql_mode: STRICT_ALL_TABLES + development: - <<: *base - pool: 5 + <<: *default database: awbw_development -# Warning: The database defined as "test" will be erased and -# re-generated from your development database when you run "rake". -# Do not set this db to the same as development or production. test: - <<: *base - pool: 5 + <<: *default database: awbw_test - -production: - <<: *base diff --git a/db/migrate/20251127143038_fix_charset_and_collation.rb b/db/migrate/20251127143038_fix_charset_and_collation.rb new file mode 100644 index 000000000..6856b54d3 --- /dev/null +++ b/db/migrate/20251127143038_fix_charset_and_collation.rb @@ -0,0 +1,22 @@ +class FixCharsetAndCollation < ActiveRecord::Migration[8.1] + DB = ActiveRecord::Base.connection_db_config.database + TARGET = "utf8mb4" + COLL = "utf8mb4_unicode_ci" + + def up + execute "ALTER DATABASE `#{DB}` CHARACTER SET #{TARGET} COLLATE #{COLL}" + tables.each do |t| + execute "ALTER TABLE `#{t}` CONVERT TO CHARACTER SET #{TARGET} COLLATE #{COLL}" + end + end + + def down + # no-op (irreversible) + end + + private + + def tables + ActiveRecord::Base.connection.tables - ["schema_migrations", "ar_internal_metadata"] + end +end diff --git a/db/schema.rb b/db/schema.rb index 344ab5918..c2a9a0eee 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -315,6 +315,9 @@ create_table "facilitators", id: :integer, charset: "utf8mb4", collation: "utf8mb4_unicode_ci", force: :cascade do |t| t.string "best_time_to_call" t.text "bio", size: :medium + t.text "bio", size: :medium + t.string "city" + t.string "country" t.datetime "created_at", null: false t.integer "created_by_id" t.date "date_of_birth" From 063d9af228fe92841a5792f98ea3a29b0124fa1e Mon Sep 17 00:00:00 2001 From: maebeale Date: Thu, 27 Nov 2025 02:04:19 -0500 Subject: [PATCH 02/10] Merge conflict --- docker-compose.yml | 21 ++++----------------- docker/mysql/charset.cnf | 3 +++ 2 files changed, 7 insertions(+), 17 deletions(-) create mode 100644 docker/mysql/charset.cnf diff --git a/docker-compose.yml b/docker-compose.yml index dae8aad09..507654aa1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,6 +7,7 @@ services: - 3306 volumes: - mysql-data:/var/lib/mysql + - ./docker/mysql/charset.cnf:/etc/mysql/conf.d/charset.cnf:ro - ./db/init:/docker-entrypoint-initdb.d environment: MYSQL_ALLOW_EMPTY_PASSWORD: 1 @@ -31,34 +32,20 @@ services: # If you prefer traditional keys, set DB_* in .env and configure database.yml accordingly. RAILS_ENV: development RAILS_LOG_TO_STDOUT: "true" - VITE_RUBY_HOST: vite - DATABASE_URL: trilogy://root@database:3306/ + DATABASE_URL: trilogy://root@database:3306/?encoding=utf8mb4&collation=utf8mb4_unicode_ci&variables[collation_connection]=utf8mb4_unicode_ci + ports: - 3000:3000 volumes: - ./:/app:cached - /app/tmp + working_dir: /app command: bash -lc "bin/rails s -b 0.0.0.0 -p 3000" tty: true stdin_open: true restart: unless-stopped - vite: - build: - context: . - dockerfile: Dockerfile.dev - env_file: .env - environment: - RAILS_ENV: development - RAILS_LOG_TO_STDOUT: "true" - VITE_RUBY_HOST: 0.0.0.0 - volumes: - - ./:/app:cached - - /app/tmp - command: bin/vite dev - ports: - - 3036:3036 prod: profiles: diff --git a/docker/mysql/charset.cnf b/docker/mysql/charset.cnf new file mode 100644 index 000000000..727b70091 --- /dev/null +++ b/docker/mysql/charset.cnf @@ -0,0 +1,3 @@ +[mysqld] +character-set-server = utf8mb4 +collation-server = utf8mb4_unicode_ci \ No newline at end of file From 5f776871989d609859f9a622827de9d952e189eb Mon Sep 17 00:00:00 2001 From: maebeale Date: Thu, 27 Nov 2025 02:07:04 -0500 Subject: [PATCH 03/10] Update contributing note --- CONTRIBUTING.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index cc113f6f1..affcd2854 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -68,9 +68,9 @@ For Docker-based development (recommended): 4. Log in as a sample user with the default [credentials](#credentials) ### Database encoding -We were getting a different schema encoding on Mac vs Linux (utf8mb4 vs utf8) which was causing some merge conflicts. +Different schema encodings on Mac vs Linux (utf8mb4 vs utf8) was causing some merge challenges in schema.rb. To ensure consistent encoding across environments, we have set MySQL database to use `utf8mb4` encoding by default by -overriding DATABASE_URL in `.env` and `.env.example` files: +overriding DATABASE_URL in `docker-compose.yml`. ``` From 7ac6dc1181e20c5e9a7ed22deec96f06a668fa2d Mon Sep 17 00:00:00 2001 From: Copilot <198982749+Copilot@users.noreply.github.com> Date: Mon, 19 Jan 2026 12:03:14 -0500 Subject: [PATCH 04/10] Force charset to utf8mb4 and utf8mb4_unicode_ci across all environments (#719) * Initial plan * Fix utf8mb4 PR: restore vite, align database.yml, add documentation Co-authored-by: maebeale <7607813+maebeale@users.noreply.github.com> * Update brakeman to avoid ci issue * Update gems to avoid brakeman ci issue --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: maebeale <7607813+maebeale@users.noreply.github.com> Co-authored-by: maebeale --- CONTRIBUTING.md | 12 +++++++++--- Gemfile.lock | 22 ++++++++++++---------- config/database.yml | 32 +++++++++++++++++--------------- docker-compose.yml | 20 +++++++++++++++++--- 4 files changed, 55 insertions(+), 31 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index affcd2854..a2a78c2ec 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -68,11 +68,17 @@ For Docker-based development (recommended): 4. Log in as a sample user with the default [credentials](#credentials) ### Database encoding -Different schema encodings on Mac vs Linux (utf8mb4 vs utf8) was causing some merge challenges in schema.rb. -To ensure consistent encoding across environments, we have set MySQL database to use `utf8mb4` encoding by default by -overriding DATABASE_URL in `docker-compose.yml`. +Different schema encodings on Mac vs Linux (utf8mb4 vs utf8) were causing merge challenges in schema.rb. +To ensure consistent encoding across environments, we have configured utf8mb4 encoding in multiple places: +- `config/database.yml` - Sets encoding and collation at the Rails level +- `docker-compose.yml` - Overrides DATABASE_URL with explicit encoding parameters for Docker development +- `docker/mysql/charset.cnf` - Configures MySQL server defaults +You can override database settings by setting the `DATABASE_URL` environment variable in your `.env` file: ``` +DATABASE_URL=trilogy://user:password@host:port/database?encoding=utf8mb4&collation=utf8mb4_unicode_ci +``` + ## Dev seeds diff --git a/Gemfile.lock b/Gemfile.lock index 2ed41ad4a..e76ac57ca 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -96,8 +96,8 @@ GEM activesupport (>= 5.0) ast (2.4.3) aws-eventstream (1.4.0) - aws-partitions (1.1198.0) - aws-sdk-core (3.240.0) + aws-partitions (1.1206.0) + aws-sdk-core (3.241.4) aws-eventstream (~> 1, >= 1.3.0) aws-partitions (~> 1, >= 1.992.0) aws-sigv4 (~> 1.9) @@ -105,11 +105,11 @@ GEM bigdecimal jmespath (~> 1, >= 1.6.1) logger - aws-sdk-kms (1.118.0) - aws-sdk-core (~> 3, >= 3.239.1) + aws-sdk-kms (1.121.0) + aws-sdk-core (~> 3, >= 3.241.4) aws-sigv4 (~> 1.5) - aws-sdk-s3 (1.208.0) - aws-sdk-core (~> 3, >= 3.234.0) + aws-sdk-s3 (1.212.0) + aws-sdk-core (~> 3, >= 3.241.4) aws-sdk-kms (~> 1) aws-sigv4 (~> 1.5) aws-sigv4 (1.12.1) @@ -195,7 +195,7 @@ GEM globalid (1.3.0) activesupport (>= 6.1) htmlentities (4.3.4) - httparty (0.24.0) + httparty (0.24.2) csv mini_mime (>= 1.0.0) multi_xml (>= 0.5.2) @@ -204,6 +204,9 @@ GEM image_processing (1.14.0) mini_magick (>= 4.9.5, < 6) ruby-vips (>= 2.0.17, < 3) + inky-rb (1.4.2.1) + foundation_emails (~> 2) + nokogiri io-console (0.8.2) irb (1.16.0) pp (>= 0.6.0) @@ -290,7 +293,7 @@ GEM net-smtp premailer (~> 1.7, >= 1.7.9) prettyprint (0.2.0) - prism (1.7.0) + prism (1.8.0) pry (0.15.2) coderay (~> 1.1) method_source (~> 1.0) @@ -345,12 +348,11 @@ GEM thor (~> 1.0, >= 1.2.2) tsort (>= 0.2) zeitwerk (~> 2.6) - rainbow (3.1.1) rake (13.3.1) rb-fsevent (0.11.2) rb-inotify (0.11.1) ffi (~> 1.0) - rdoc (7.0.3) + rdoc (7.1.0) erb psych (>= 4.0.0) tsort diff --git a/config/database.yml b/config/database.yml index b43a36995..39ddd4395 100644 --- a/config/database.yml +++ b/config/database.yml @@ -1,26 +1,28 @@ +# Database configuration +# To override any settings, set the DATABASE_URL environment variable. +# Example: DATABASE_URL=trilogy://user:password@host:port/database?encoding=utf8mb4&collation=utf8mb4_unicode_ci +# +# The encoding and collation settings ensure consistent UTF-8 handling across +# Mac and Linux environments, preventing schema.rb merge conflicts. + base: &base adapter: trilogy encoding: utf8mb4 collation: utf8mb4_unicode_ci url: <%= ENV.fetch("DATABASE_URL", "trilogy://root@localhost") %> -default: &default - adapter: trilogy - encoding: utf8mb4 # <-- forces UTF-8 everywhere - collation: utf8mb4_unicode_ci # <-- portable across MySQL 5.7/8.0 and MariaDB - pool: <%= ENV.fetch("RAILS_MAX_THREADS", 5) %> - user: root - password: - host: 127.0.0.1 - variables: - # Make sure the session uses utf8mb4 too: - collation_connection: utf8mb4_unicode_ci - sql_mode: STRICT_ALL_TABLES - development: - <<: *default + <<: *base + pool: 5 database: awbw_development +# Warning: The database defined as "test" will be erased and +# re-generated from your development database when you run "rake". +# Do not set this db to the same as development or production. test: - <<: *default + <<: *base + pool: 5 database: awbw_test + +production: + <<: *base diff --git a/docker-compose.yml b/docker-compose.yml index 507654aa1..72074cfc1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -32,20 +32,34 @@ services: # If you prefer traditional keys, set DB_* in .env and configure database.yml accordingly. RAILS_ENV: development RAILS_LOG_TO_STDOUT: "true" - DATABASE_URL: trilogy://root@database:3306/?encoding=utf8mb4&collation=utf8mb4_unicode_ci&variables[collation_connection]=utf8mb4_unicode_ci - + VITE_RUBY_HOST: vite + DATABASE_URL: trilogy://root@database:3306/?encoding=utf8mb4&collation=utf8mb4_unicode_ci ports: - 3000:3000 volumes: - ./:/app:cached - /app/tmp - working_dir: /app command: bash -lc "bin/rails s -b 0.0.0.0 -p 3000" tty: true stdin_open: true restart: unless-stopped + vite: + build: + context: . + dockerfile: Dockerfile.dev + env_file: .env + environment: + RAILS_ENV: development + RAILS_LOG_TO_STDOUT: "true" + VITE_RUBY_HOST: 0.0.0.0 + volumes: + - ./:/app:cached + - /app/tmp + command: bin/vite dev + ports: + - 3036:3036 prod: profiles: From e9ec22be4ad29097e910a5d7b991fefd16e2caf5 Mon Sep 17 00:00:00 2001 From: maebeale Date: Mon, 19 Jan 2026 12:28:01 -0500 Subject: [PATCH 05/10] Update migration to today's date and run it --- ...lation.rb => 20260119143438_fix_charset_and_collation.rb} | 0 db/schema.rb | 5 +---- 2 files changed, 1 insertion(+), 4 deletions(-) rename db/migrate/{20251127143038_fix_charset_and_collation.rb => 20260119143438_fix_charset_and_collation.rb} (100%) diff --git a/db/migrate/20251127143038_fix_charset_and_collation.rb b/db/migrate/20260119143438_fix_charset_and_collation.rb similarity index 100% rename from db/migrate/20251127143038_fix_charset_and_collation.rb rename to db/migrate/20260119143438_fix_charset_and_collation.rb diff --git a/db/schema.rb b/db/schema.rb index c2a9a0eee..0d35a6043 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[8.1].define(version: 2026_01_19_143337) do +ActiveRecord::Schema[8.1].define(version: 2026_01_19_143438) do create_table "action_text_mentions", charset: "utf8mb4", collation: "utf8mb4_unicode_ci", force: :cascade do |t| t.bigint "action_text_rich_text_id", null: false t.datetime "created_at", null: false @@ -315,9 +315,6 @@ create_table "facilitators", id: :integer, charset: "utf8mb4", collation: "utf8mb4_unicode_ci", force: :cascade do |t| t.string "best_time_to_call" t.text "bio", size: :medium - t.text "bio", size: :medium - t.string "city" - t.string "country" t.datetime "created_at", null: false t.integer "created_by_id" t.date "date_of_birth" From d1b34d3983b34ef603604443a27223dd456a856b Mon Sep 17 00:00:00 2001 From: maebeale Date: Mon, 19 Jan 2026 12:28:48 -0500 Subject: [PATCH 06/10] Update Gemfile.lock --- Gemfile.lock | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index e76ac57ca..48770ca5f 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -191,7 +191,7 @@ GEM faker (3.5.2) i18n (>= 1.8.11, < 2) feature_flipper (2.0.0) - ffi (1.17.2) + ffi (1.17.2-arm64-darwin) globalid (1.3.0) activesupport (>= 6.1) htmlentities (4.3.4) @@ -204,9 +204,6 @@ GEM image_processing (1.14.0) mini_magick (>= 4.9.5, < 6) ruby-vips (>= 2.0.17, < 3) - inky-rb (1.4.2.1) - foundation_emails (~> 2) - nokogiri io-console (0.8.2) irb (1.16.0) pp (>= 0.6.0) @@ -253,7 +250,6 @@ GEM mini_magick (5.3.1) logger mini_mime (1.1.5) - mini_portile2 (2.8.9) minitest (6.0.1) prism (~> 1.5) msgpack (1.8.0) @@ -270,8 +266,7 @@ GEM net-smtp (0.5.1) net-protocol nio4r (2.7.4) - nokogiri (1.19.0) - mini_portile2 (~> 2.8.2) + nokogiri (1.19.0-arm64-darwin) racc (~> 1.4) orm_adapter (0.5.0) parallel (1.27.0) @@ -348,6 +343,7 @@ GEM thor (~> 1.0, >= 1.2.2) tsort (>= 0.2) zeitwerk (~> 2.6) + rainbow (3.1.1) rake (13.3.1) rb-fsevent (0.11.2) rb-inotify (0.11.1) @@ -494,7 +490,7 @@ GEM zeitwerk (2.7.4) PLATFORMS - ruby + arm64-darwin-23 DEPENDENCIES active_storage_validations (~> 3.0) From 60e6de63f69d30eb1035e0c658cbd167d85812ff Mon Sep 17 00:00:00 2001 From: maebeale Date: Mon, 19 Jan 2026 12:37:26 -0500 Subject: [PATCH 07/10] Retain jp's settings --- config/database.yml | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/config/database.yml b/config/database.yml index 39ddd4395..e989d6d3e 100644 --- a/config/database.yml +++ b/config/database.yml @@ -7,13 +7,20 @@ base: &base adapter: trilogy - encoding: utf8mb4 - collation: utf8mb4_unicode_ci + encoding: utf8mb4 # <-- forces UTF-8 everywhere + collation: utf8mb4_unicode_ci # <-- portable across MySQL 5.7/8.0 and MariaDB + pool: 5 + user: root + password: + host: 127.0.0.1 + variables: + # Make sure the session uses utf8mb4 too: + collation_connection: utf8mb4_unicode_ci + sql_mode: STRICT_ALL_TABLES url: <%= ENV.fetch("DATABASE_URL", "trilogy://root@localhost") %> development: <<: *base - pool: 5 database: awbw_development # Warning: The database defined as "test" will be erased and @@ -21,7 +28,6 @@ development: # Do not set this db to the same as development or production. test: <<: *base - pool: 5 database: awbw_test production: From 5bbaf36d1d38dd74d10cfccc92fee73e08193a95 Mon Sep 17 00:00:00 2001 From: maebeale Date: Mon, 19 Jan 2026 12:51:23 -0500 Subject: [PATCH 08/10] rubocop spacing --- db/migrate/20260119143438_fix_charset_and_collation.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db/migrate/20260119143438_fix_charset_and_collation.rb b/db/migrate/20260119143438_fix_charset_and_collation.rb index 6856b54d3..4d2b8d839 100644 --- a/db/migrate/20260119143438_fix_charset_and_collation.rb +++ b/db/migrate/20260119143438_fix_charset_and_collation.rb @@ -17,6 +17,6 @@ def down private def tables - ActiveRecord::Base.connection.tables - ["schema_migrations", "ar_internal_metadata"] + ActiveRecord::Base.connection.tables - [ "schema_migrations", "ar_internal_metadata" ] end end From 1694e69883bba80a19acdd3c871bb6ad47821d17 Mon Sep 17 00:00:00 2001 From: maebeale Date: Mon, 19 Jan 2026 21:00:37 -0500 Subject: [PATCH 09/10] Update gemfile.lock w more platforms --- Gemfile.lock | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Gemfile.lock b/Gemfile.lock index 48770ca5f..a6b353991 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -192,6 +192,7 @@ GEM i18n (>= 1.8.11, < 2) feature_flipper (2.0.0) ffi (1.17.2-arm64-darwin) + ffi (1.17.2-x86_64-linux-gnu) globalid (1.3.0) activesupport (>= 6.1) htmlentities (4.3.4) @@ -268,6 +269,8 @@ GEM nio4r (2.7.4) nokogiri (1.19.0-arm64-darwin) racc (~> 1.4) + nokogiri (1.19.0-x86_64-linux-gnu) + racc (~> 1.4) orm_adapter (0.5.0) parallel (1.27.0) parser (3.3.10.0) @@ -491,6 +494,7 @@ GEM PLATFORMS arm64-darwin-23 + x86_64-linux DEPENDENCIES active_storage_validations (~> 3.0) From 9c7e5718ebe3ec9fa69058b3440f20befffefd0e Mon Sep 17 00:00:00 2001 From: maebeale Date: Mon, 19 Jan 2026 21:04:12 -0500 Subject: [PATCH 10/10] Add additional platform to Gemfile.lock --- Gemfile.lock | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Gemfile.lock b/Gemfile.lock index a6b353991..68593d655 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -191,6 +191,7 @@ GEM faker (3.5.2) i18n (>= 1.8.11, < 2) feature_flipper (2.0.0) + ffi (1.17.2-aarch64-linux-gnu) ffi (1.17.2-arm64-darwin) ffi (1.17.2-x86_64-linux-gnu) globalid (1.3.0) @@ -267,6 +268,8 @@ GEM net-smtp (0.5.1) net-protocol nio4r (2.7.4) + nokogiri (1.19.0-aarch64-linux-gnu) + racc (~> 1.4) nokogiri (1.19.0-arm64-darwin) racc (~> 1.4) nokogiri (1.19.0-x86_64-linux-gnu) @@ -493,6 +496,7 @@ GEM zeitwerk (2.7.4) PLATFORMS + aarch64-linux arm64-darwin-23 x86_64-linux