This is an automated email from the ASF dual-hosted git repository.
arnold pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/fineract.git
The following commit(s) were added to refs/heads/develop by this push:
new f797e1f638 FINERACT-1398: Standardize charset and collation defaults
f797e1f638 is described below
commit f797e1f6385b2c5146d16ee2fc03acb4d3d64061
Author: airajena <[email protected]>
AuthorDate: Mon Mar 2 23:21:28 2026 +0530
FINERACT-1398: Standardize charset and collation defaults
---
fineract-provider/build.gradle | 4 +-
.../service/DatatableWriteServiceImpl.java | 2 +-
.../tenant-store/changelog-tenant-store.xml | 1 +
...11_standardize_character_set_and_collation.xml} | 12 ++---
.../db/changelog/tenant/changelog-tenant.xml | 1 +
...18_standardize_character_set_and_collation.xml} | 12 ++---
.../service/DatatableWriteServiceImplTest.java | 53 ++++++++++++++++++++++
7 files changed, 64 insertions(+), 21 deletions(-)
diff --git a/fineract-provider/build.gradle b/fineract-provider/build.gradle
index 34a414a9c9..f79a834a84 100644
--- a/fineract-provider/build.gradle
+++ b/fineract-provider/build.gradle
@@ -185,7 +185,7 @@ tasks.register('createDB') {
description = "Creates the MariaDB Database. Needs database name to be
passed (like: -PdbName=someDBname)"
doLast {
def sql = Sql.newInstance('jdbc:mariadb://localhost:3306/', mysqlUser,
mysqlPassword, 'org.mariadb.jdbc.Driver')
- sql.execute('CREATE DATABASE ' + "`$dbName` CHARACTER SET utf8mb4")
+ sql.execute('CREATE DATABASE ' + "`$dbName` CHARACTER SET utf8mb4
COLLATE utf8mb4_unicode_ci")
}
}
@@ -217,7 +217,7 @@ tasks.register('createMySQLDB') {
description = "Creates the MySQL Database. Needs database name to be
passed (like: -PdbName=someDBname)"
doLast {
def sql = Sql.newInstance('jdbc:mysql://localhost:3306/', mysqlUser,
mysqlPassword, 'com.mysql.cj.jdbc.Driver')
- sql.execute('CREATE DATABASE ' + "`$dbName` CHARACTER SET utf8mb4")
+ sql.execute('CREATE DATABASE ' + "`$dbName` CHARACTER SET utf8mb4
COLLATE utf8mb4_unicode_ci")
}
}
diff --git
a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/DatatableWriteServiceImpl.java
b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/DatatableWriteServiceImpl.java
index 5c07474ea5..3f81b76674 100644
---
a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/DatatableWriteServiceImpl.java
+++
b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/DatatableWriteServiceImpl.java
@@ -277,7 +277,7 @@ public class DatatableWriteServiceImpl implements
DatatableWriteService {
sqlBuilder.append(constrainBuilder);
sqlBuilder.append(")");
if (databaseTypeResolver.isMySQL()) {
- sqlBuilder.append(" ENGINE=InnoDB DEFAULT CHARSET=UTF8MB4;");
+ sqlBuilder.append(" ENGINE=InnoDB DEFAULT CHARSET=UTF8MB4
COLLATE=UTF8MB4_UNICODE_CI;");
}
log.debug("SQL:: {}", sqlBuilder);
diff --git
a/fineract-provider/src/main/resources/db/changelog/tenant-store/changelog-tenant-store.xml
b/fineract-provider/src/main/resources/db/changelog/tenant-store/changelog-tenant-store.xml
index de38521dae..7d5a94e783 100644
---
a/fineract-provider/src/main/resources/db/changelog/tenant-store/changelog-tenant-store.xml
+++
b/fineract-provider/src/main/resources/db/changelog/tenant-store/changelog-tenant-store.xml
@@ -31,4 +31,5 @@
<include file="parts/0008_encrypt_existing_ro_tenant_passwords.xml"
relativeToChangelogFile="true"/>
<include file="parts/0009_set_and_encrypt_ro_if_not_exists.xml"
relativeToChangelogFile="true"/>
<include file="parts/0010_set_datetime_precision.xml"
relativeToChangelogFile="true"/>
+ <include file="parts/0011_standardize_character_set_and_collation.xml"
relativeToChangelogFile="true"/>
</databaseChangeLog>
diff --git
a/fineract-provider/src/main/resources/db/changelog/tenant-store/changelog-tenant-store.xml
b/fineract-provider/src/main/resources/db/changelog/tenant-store/parts/0011_standardize_character_set_and_collation.xml
similarity index 56%
copy from
fineract-provider/src/main/resources/db/changelog/tenant-store/changelog-tenant-store.xml
copy to
fineract-provider/src/main/resources/db/changelog/tenant-store/parts/0011_standardize_character_set_and_collation.xml
index de38521dae..7825c41606 100644
---
a/fineract-provider/src/main/resources/db/changelog/tenant-store/changelog-tenant-store.xml
+++
b/fineract-provider/src/main/resources/db/changelog/tenant-store/parts/0011_standardize_character_set_and_collation.xml
@@ -22,13 +22,7 @@
<databaseChangeLog xmlns="http://www.liquibase.org/xml/ns/dbchangelog";
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog
http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-4.1.xsd";>
- <include file="parts/0003_reset_postgresql_sequences.xml"
relativeToChangelogFile="true"/>
- <include file="parts/0004_readonly_database_connection.xml"
relativeToChangelogFile="true"/>
- <include file="parts/0005_jdbc_connection_string.xml"
relativeToChangelogFile="true"/>
- <include file="parts/0006_drop_retry_parameter_columns.xml"
relativeToChangelogFile="true"/>
- <include file="parts/0007_encrypt_existing_tenant_passwords.xml"
relativeToChangelogFile="true"/>
- <include file="parts/0007_x_extend_tenant_ro_passwords.xml"
relativeToChangelogFile="true"/>
- <include file="parts/0008_encrypt_existing_ro_tenant_passwords.xml"
relativeToChangelogFile="true"/>
- <include file="parts/0009_set_and_encrypt_ro_if_not_exists.xml"
relativeToChangelogFile="true"/>
- <include file="parts/0010_set_datetime_precision.xml"
relativeToChangelogFile="true"/>
+ <changeSet author="fineract" id="1" context="tenant_store_db">
+ <sql dbms="mysql,mariadb">ALTER DATABASE CHARACTER SET utf8mb4 COLLATE
utf8mb4_unicode_ci;</sql>
+ </changeSet>
</databaseChangeLog>
diff --git
a/fineract-provider/src/main/resources/db/changelog/tenant/changelog-tenant.xml
b/fineract-provider/src/main/resources/db/changelog/tenant/changelog-tenant.xml
index 38143a7df6..7ad3149322 100644
---
a/fineract-provider/src/main/resources/db/changelog/tenant/changelog-tenant.xml
+++
b/fineract-provider/src/main/resources/db/changelog/tenant/changelog-tenant.xml
@@ -236,4 +236,5 @@
<include
file="parts/0215_transaction_summary_reports_add_buydown_fee_types.xml"
relativeToChangelogFile="true" />
<include file="parts/0216_add_unique_constraint_sms_campaign_name.xml"
relativeToChangelogFile="true" />
<include file="parts/0217_force_withdrawal_configs.xml"
relativeToChangelogFile="true" />
+ <include file="parts/0218_standardize_character_set_and_collation.xml"
relativeToChangelogFile="true" />
</databaseChangeLog>
diff --git
a/fineract-provider/src/main/resources/db/changelog/tenant-store/changelog-tenant-store.xml
b/fineract-provider/src/main/resources/db/changelog/tenant/parts/0218_standardize_character_set_and_collation.xml
similarity index 56%
copy from
fineract-provider/src/main/resources/db/changelog/tenant-store/changelog-tenant-store.xml
copy to
fineract-provider/src/main/resources/db/changelog/tenant/parts/0218_standardize_character_set_and_collation.xml
index de38521dae..9b995bc863 100644
---
a/fineract-provider/src/main/resources/db/changelog/tenant-store/changelog-tenant-store.xml
+++
b/fineract-provider/src/main/resources/db/changelog/tenant/parts/0218_standardize_character_set_and_collation.xml
@@ -22,13 +22,7 @@
<databaseChangeLog xmlns="http://www.liquibase.org/xml/ns/dbchangelog";
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog
http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-4.1.xsd";>
- <include file="parts/0003_reset_postgresql_sequences.xml"
relativeToChangelogFile="true"/>
- <include file="parts/0004_readonly_database_connection.xml"
relativeToChangelogFile="true"/>
- <include file="parts/0005_jdbc_connection_string.xml"
relativeToChangelogFile="true"/>
- <include file="parts/0006_drop_retry_parameter_columns.xml"
relativeToChangelogFile="true"/>
- <include file="parts/0007_encrypt_existing_tenant_passwords.xml"
relativeToChangelogFile="true"/>
- <include file="parts/0007_x_extend_tenant_ro_passwords.xml"
relativeToChangelogFile="true"/>
- <include file="parts/0008_encrypt_existing_ro_tenant_passwords.xml"
relativeToChangelogFile="true"/>
- <include file="parts/0009_set_and_encrypt_ro_if_not_exists.xml"
relativeToChangelogFile="true"/>
- <include file="parts/0010_set_datetime_precision.xml"
relativeToChangelogFile="true"/>
+ <changeSet author="fineract" id="1">
+ <sql dbms="mysql,mariadb">ALTER DATABASE CHARACTER SET utf8mb4 COLLATE
utf8mb4_unicode_ci;</sql>
+ </changeSet>
</databaseChangeLog>
diff --git
a/fineract-provider/src/test/java/org/apache/fineract/infrastructure/dataqueries/service/DatatableWriteServiceImplTest.java
b/fineract-provider/src/test/java/org/apache/fineract/infrastructure/dataqueries/service/DatatableWriteServiceImplTest.java
index 44be562808..a1cd828842 100644
---
a/fineract-provider/src/test/java/org/apache/fineract/infrastructure/dataqueries/service/DatatableWriteServiceImplTest.java
+++
b/fineract-provider/src/test/java/org/apache/fineract/infrastructure/dataqueries/service/DatatableWriteServiceImplTest.java
@@ -24,21 +24,27 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.atLeastOnce;
+import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
+import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
+import com.google.gson.JsonParser;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import
org.apache.fineract.infrastructure.codes.service.CodeReadPlatformService;
import
org.apache.fineract.infrastructure.configuration.domain.ConfigurationDomainService;
+import org.apache.fineract.infrastructure.core.api.JsonCommand;
import
org.apache.fineract.infrastructure.core.serialization.DatatableCommandFromApiJsonDeserializer;
import org.apache.fineract.infrastructure.core.serialization.FromJsonHelper;
import
org.apache.fineract.infrastructure.core.service.database.DatabaseSpecificSQLGenerator;
+import org.apache.fineract.infrastructure.core.service.database.DatabaseType;
import
org.apache.fineract.infrastructure.core.service.database.DatabaseTypeResolver;
import org.apache.fineract.infrastructure.dataqueries.data.DataTableValidator;
+import org.apache.fineract.infrastructure.dataqueries.data.EntityTables;
import
org.apache.fineract.infrastructure.event.business.service.BusinessEventNotifierService;
import
org.apache.fineract.infrastructure.security.service.PlatformSecurityContext;
import org.apache.fineract.portfolio.search.service.SearchUtil;
@@ -245,4 +251,51 @@ class DatatableWriteServiceImplTest {
assertTrue(sql.contains("INSERT INTO x_table_column_code_mappings"),
"SQL should insert into code mappings table");
assertTrue(sql.contains("VALUES (?, ?)"), "SQL should use ?
placeholders");
}
+
+ @Test
+ void testCreateDatatableUsesUtf8mb4UnicodeCiForMySql() {
+ final JsonElement payload = JsonParser.parseString("""
+ {
+ "datatableName": "dt_charset_test",
+ "apptableName": "m_client",
+ "entitySubType": "PERSON",
+ "multiRow": false,
+ "columns": [
+ {
+ "name": "itsAString",
+ "type": "String",
+ "mandatory": true,
+ "length": 10
+ }
+ ]
+ }
+ """);
+
+ final JsonCommand command = mock(JsonCommand.class);
+ when(command.json()).thenReturn(payload.toString());
+ when(command.commandId()).thenReturn(1L);
+
+ when(databaseTypeResolver.isMySQL()).thenReturn(true);
+
when(databaseTypeResolver.databaseType()).thenReturn(DatabaseType.MYSQL);
+
when(configurationDomainService.isConstraintApproachEnabledForDatatables()).thenReturn(false);
+ when(sqlGenerator.currentSchema()).thenReturn("database()");
+ when(sqlGenerator.escape(anyString())).thenAnswer(invocation -> "`" +
invocation.getArgument(0) + "`");
+
when(datatableUtil.resolveEntity("m_client")).thenReturn(EntityTables.CLIENT);
+
when(datatableUtil.getFKField(EntityTables.CLIENT)).thenReturn("client_id");
+ when(fromJsonHelper.parse(anyString())).thenReturn(payload);
+ when(fromJsonHelper.extractJsonArrayNamed(eq("columns"), eq(payload)))
+
.thenReturn(payload.getAsJsonObject().getAsJsonArray("columns"));
+ when(fromJsonHelper.extractStringNamed(eq("datatableName"),
eq(payload))).thenReturn("dt_charset_test");
+ when(fromJsonHelper.extractStringNamed(eq("entitySubType"),
eq(payload))).thenReturn("PERSON");
+ when(fromJsonHelper.extractStringNamed(eq("apptableName"),
eq(payload))).thenReturn("m_client");
+ when(fromJsonHelper.extractBooleanNamed(eq("multiRow"),
eq(payload))).thenReturn(false);
+ when(jdbcTemplate.queryForObject(anyString(), eq(String.class),
eq("dt_charset_test"))).thenReturn("true");
+
+ underTest.createDatatable(command);
+
+ final ArgumentCaptor<String> sqlCaptor =
ArgumentCaptor.forClass(String.class);
+ verify(jdbcTemplate).execute(sqlCaptor.capture());
+ assertTrue(sqlCaptor.getValue().contains("ENGINE=InnoDB DEFAULT
CHARSET=UTF8MB4 COLLATE=UTF8MB4_UNICODE_CI;"),
+ "MySQL table creation must include utf8mb4 charset and
utf8mb4_unicode_ci collation");
+ }
}
