From e29f9cd572d625c3801fbac0d99b1a84b64a50dc Mon Sep 17 00:00:00 2001 From: Fariha Shaikh Date: Tue, 4 Feb 2025 19:34:13 +0000 Subject: [PATCH] MDEV-32362 Remove generated columns from INSERT statements in dump tool Currently, generated columns names and values are present in INSERT statements created by the dump tool. This is incorrect as values for generated columns are calculated based on other column values and should not be explicitly inserted, as seen in MySQL's dump tool. Remove names and values for generated columns from INSERT statements created by mysqldump/mariadb-dump. For tables with one or more generated columns, include a list of non-generated column names in the INSERT statement. All new code of the whole pull request, including one or several files that are either new files or modified ones, are contributed under the BSD-new license. I am contributing on behalf of my employer Amazon Web Services, Inc. --- client/mysqldump.cc | 128 +++++++++++++++++++++---------- mysql-test/main/mysqldump.result | 55 +++++++++++++ mysql-test/main/mysqldump.test | 50 ++++++++++++ 3 files changed, 193 insertions(+), 40 deletions(-) diff --git a/client/mysqldump.cc b/client/mysqldump.cc index fc5d990478e34..efca4d70a2779 100644 --- a/client/mysqldump.cc +++ b/client/mysqldump.cc @@ -104,7 +104,8 @@ #define DUMP_TABLE_SEQUENCE 1 /* until MDEV-35831 is implemented, we'll have to detect VECTOR by name */ -#define MYSQL_TYPE_VECTOR "V" +#define MYSQL_TYPE_VECTOR 1 +#define MYSQL_TYPE_VERS_COL 2 static my_bool ignore_table_data(const uchar *hash_key, size_t len); static void add_load_option(DYNAMIC_STRING *str, const char *option, @@ -3168,6 +3169,7 @@ static uint get_table_structure(const char *table, const char *db, char *table_t char *ignore_flag, my_bool *versioned) { my_bool init=0, delayed, write_data, complete_insert; + my_bool is_generated=0, is_vector=0; my_ulonglong num_fields; char *result_table, *opt_quoted_table; const char *insert_option; @@ -3486,8 +3488,20 @@ static uint get_table_structure(const char *table, const char *db, char *table_t while ((row= mysql_fetch_row(result))) { - if (strstr(row[1],"INVISIBLE")) - complete_insert= 1; + is_generated= 0; + if (row[1]) + { + if (strstr(row[1], "GENERATED") && + !(row[2] && (strcmp(row[2], "ROW START") == 0 || + strcmp(row[2], "ROW END") == 0))) + { + complete_insert= 1; + is_generated= 1; + } + if (strstr(row[1], "INVISIBLE")) + complete_insert= 1; + } + if (vers_hidden && row[2] && strcmp(row[2], "ROW START") == 0) { vers_hidden= 0; @@ -3498,6 +3512,17 @@ static uint get_table_structure(const char *table, const char *db, char *table_t *versioned= 0; } } + + if (is_generated && !path && !opt_dir && !opt_dump_history) + continue; + + dynstr_append_mem_checked(&field_flags, "", 1); + if (row[3] && strcmp(row[3], "vector") == 0) + field_flags.str[field_flags.length-1]|= MYSQL_TYPE_VECTOR; + /* Mark system versioning columns */ + if (row[2] && (strcmp(row[2], "ROW START") == 0 || strcmp(row[2], "ROW END") == 0)) + field_flags.str[field_flags.length-1]|= MYSQL_TYPE_VERS_COL; + if (init) { dynstr_append_checked(&select_field_names, ", "); @@ -3526,11 +3551,6 @@ static uint get_table_structure(const char *table, const char *db, char *table_t if (opt_header) dynstr_append_checked(&select_field_names_for_header, quote_for_equal(row[0], name_buff)); - /* VECTOR doesn't have a type code yet, must be detected by name */ - if (row[3] && strcmp(row[3], "vector") == 0) - dynstr_append_checked(&field_flags, MYSQL_TYPE_VECTOR); - else - dynstr_append_checked(&field_flags, " "); } if (vers_hidden) @@ -3544,7 +3564,11 @@ static uint get_table_structure(const char *table, const char *db, char *table_t "row_end" : "row_end"); dynstr_append_checked(&insert_field_names, ", row_start, row_end"); - dynstr_append_checked(&field_flags, " "); + /* Mark both row_start and row_end as versioning columns */ + dynstr_append_mem_checked(&field_flags, "", 1); + field_flags.str[field_flags.length-1]|= MYSQL_TYPE_VERS_COL; + dynstr_append_mem_checked(&field_flags, "", 1); + field_flags.str[field_flags.length-1]|= MYSQL_TYPE_VERS_COL; } /* @@ -3563,9 +3587,7 @@ static uint get_table_structure(const char *table, const char *db, char *table_t dynstr_append_checked(&insert_pat, "INTO "); dynstr_append_checked(&insert_pat, opt_quoted_table); if (complete_insert) - { dynstr_append_checked(&insert_pat, " ("); - } else { if (extended_insert) @@ -3577,7 +3599,7 @@ static uint get_table_structure(const char *table, const char *db, char *table_t if (complete_insert) dynstr_append_checked(&insert_pat, insert_field_names.str); - num_fields= mysql_num_rows(result) + (vers_hidden ? 2 : 0); + num_fields= field_flags.length; mysql_free_result(result); } else @@ -3629,33 +3651,39 @@ static uint get_table_structure(const char *table, const char *db, char *table_t check_io(sql_file); } - if (write_data) - { - if (opt_replace_into) - dynstr_append_checked(&insert_pat, "REPLACE "); - else - dynstr_append_checked(&insert_pat, "INSERT "); - dynstr_append_checked(&insert_pat, insert_option); - dynstr_append_checked(&insert_pat, "INTO "); - dynstr_append_checked(&insert_pat, result_table); - if (complete_insert) - dynstr_append_checked(&insert_pat, " ("); - else - { - dynstr_append_checked(&insert_pat, " VALUES "); - if (!extended_insert) - dynstr_append_checked(&insert_pat, "("); - } - } - while ((row= mysql_fetch_row(result))) { ulong *lengths= mysql_fetch_lengths(result); + is_generated= 0; + is_vector= 0; + /* VECTOR doesn't have a type code yet, must be detected by name */ if (strncmp(row[SHOW_TYPE], STRING_WITH_LEN("vector(")) == 0) - dynstr_append_checked(&field_flags, MYSQL_TYPE_VECTOR); - else - dynstr_append_checked(&field_flags, " "); + is_vector= 1; + else if (row[SHOW_EXTRA]) + { + if (strstr(row[SHOW_EXTRA], "GENERATED") && + !strstr(row[SHOW_EXTRA], "ROW START") && + !strstr(row[SHOW_EXTRA], "ROW END")) + { + complete_insert= 1; + is_generated= 1; + } + } + if (row[SHOW_EXTRA] && strstr(row[SHOW_EXTRA], "INVISIBLE")) + complete_insert= 1; + + if (is_generated && !path && !opt_dir && !opt_dump_history) + continue; + + dynstr_append_mem_checked(&field_flags, "", 1); + if (is_vector) + field_flags.str[field_flags.length-1]|= MYSQL_TYPE_VECTOR; + /* Mark system versioning columns */ + if (row[SHOW_EXTRA] && (strstr(row[SHOW_EXTRA], "ROW START") || + strstr(row[SHOW_EXTRA], "ROW END"))) + field_flags.str[field_flags.length-1]|= MYSQL_TYPE_VERS_COL; + if (init) { if (!opt_xml && !opt_no_create_info) @@ -3668,11 +3696,12 @@ static uint get_table_structure(const char *table, const char *db, char *table_t dynstr_append_checked(&select_field_names_for_header, ", "); } dynstr_append_checked(&select_field_names, - quote_name(row[SHOW_FIELDNAME], name_buff, 0)); + quote_name(row[SHOW_FIELDNAME], name_buff, 0)); if (opt_header) dynstr_append_checked(&select_field_names_for_header, quote_for_equal(row[SHOW_FIELDNAME], name_buff)); init=1; + if (!opt_no_create_info) { if (opt_xml) @@ -3686,7 +3715,7 @@ static uint get_table_structure(const char *table, const char *db, char *table_t quote_name(row[SHOW_FIELDNAME],name_buff, 0), row[SHOW_TYPE]); else fprintf(sql_file, " %s %s", - quote_name(row[SHOW_FIELDNAME], name_buff, 0), row[SHOW_TYPE]); + quote_name(row[SHOW_FIELDNAME], name_buff, 0), row[SHOW_TYPE]); if (row[SHOW_DEFAULT]) { fputs(" DEFAULT ", sql_file); @@ -3699,9 +3728,29 @@ static uint get_table_structure(const char *table, const char *db, char *table_t check_io(sql_file); } } + + if (write_data) + { + if (opt_replace_into) + dynstr_append_checked(&insert_pat, "REPLACE "); + else + dynstr_append_checked(&insert_pat, "INSERT "); + dynstr_append_checked(&insert_pat, insert_option); + dynstr_append_checked(&insert_pat, "INTO "); + dynstr_append_checked(&insert_pat, result_table); + if (complete_insert) + dynstr_append_checked(&insert_pat, " ("); + else + { + dynstr_append_checked(&insert_pat, " VALUES "); + if (!extended_insert) + dynstr_append_checked(&insert_pat, "("); + } + } + if (complete_insert) dynstr_append_checked(&insert_pat, select_field_names.str); - num_fields= mysql_num_rows(result); + num_fields= field_flags.length; mysql_free_result(result); if (!opt_no_create_info) { @@ -4240,7 +4289,6 @@ static void dump_table(const char *table, const char *db, const uchar *hash_key, uint num_fields; size_t total_length, init_length; my_bool versioned= 0; - MYSQL_RES *res= NULL; MYSQL_FIELD *field; MYSQL_ROW row; @@ -4251,7 +4299,6 @@ static void dump_table(const char *table, const char *db, const uchar *hash_key, --no-data flag below. Otherwise, the create table info won't be printed. */ num_fields= get_table_structure(table, db, table_type, &ignore_flag, &versioned); - /* The "table" could be a view. If so, we don't do anything here. */ @@ -4544,8 +4591,9 @@ static void dump_table(const char *table, const char *db, const uchar *hash_key, */ is_blob= field->type == MYSQL_TYPE_GEOMETRY || field->type == MYSQL_TYPE_BIT || - field_flags.str[i] == MYSQL_TYPE_VECTOR[0] || + field_flags.str[i] & MYSQL_TYPE_VECTOR || (opt_hex_blob && field->charsetnr == 63 && + !(field_flags.str[i] & MYSQL_TYPE_VERS_COL) && (field->type == MYSQL_TYPE_STRING || field->type == MYSQL_TYPE_VAR_STRING || field->type == MYSQL_TYPE_VARCHAR || diff --git a/mysql-test/main/mysqldump.result b/mysql-test/main/mysqldump.result index d0340a7265610..f3fce2518ae32 100644 --- a/mysql-test/main/mysqldump.result +++ b/mysql-test/main/mysqldump.result @@ -26,6 +26,61 @@ INSERT INTO t1 VALUES (1), (2); DROP TABLE t1; # +# MDEV-32362 Remove generated columns from INSERT statements in dump +# Include list of non-generated columns in INSERT statements for tables +# with one or more generated columns +# +CREATE DATABASE dump_generated; +USE dump_generated; +CREATE TABLE t1 (pk INTEGER, a INTEGER, b INTEGER, c VARCHAR(16), +sum INTEGER GENERATED ALWAYS AS (a+b), +sub VARCHAR(4) GENERATED ALWAYS AS (SUBSTRING(c, 1, 4)), +key k1(sum), +key k2(sub) +) engine=innodb; +INSERT INTO t1(pk, a, b, c) VALUES (1, 11, 12, 'oneone'), (2, 21, 22, 'twotwo'); +SELECT * FROM t1; +pk a b c sum sub +1 11 12 oneone 23 oneo +2 21 22 twotwo 43 twot +DELETE FROM t1; +SELECT * FROM t1; +pk a b c sum sub +1 11 12 oneone 23 oneo +2 21 22 twotwo 43 twot +DELETE FROM t1; +LOAD DATA INFILE 'MYSQLTEST_VARDIR/tmp/t1.txt' INTO TABLE t1; +SELECT * FROM t1; +pk a b c sum sub +1 11 12 oneone 23 oneo +2 21 22 twotwo 43 twot +DROP TABLE t1; +# A table with regular columns after generated +CREATE TABLE t2 (pk INTEGER, a INTEGER, b INTEGER, +sum INTEGER GENERATED ALWAYS AS (a+b), +c VARCHAR(16), +key k1(sum) +) engine=innodb; +INSERT INTO t2(pk, a, b, c) VALUES (1, 11, 12, 'oneone'), (2, 21, 22, 'twotwo'); +SELECT * FROM t2; +pk a b sum c +1 11 12 23 oneone +2 21 22 43 twotwo +DELETE FROM t2; +SELECT * FROM t2; +pk a b sum c +1 11 12 23 oneone +2 21 22 43 twotwo +DELETE FROM t2; +LOAD DATA INFILE 'MYSQLTEST_VARDIR/tmp/t2.txt' INTO TABLE t2; +SELECT * FROM t2; +pk a b sum c +1 11 12 23 oneone +2 21 22 43 twotwo +DROP TABLE t2; +DROP DATABASE dump_generated; +USE test; +# # Bug#2005 Long decimal comparison bug. # CREATE TABLE t1 (a decimal(64, 20)); diff --git a/mysql-test/main/mysqldump.test b/mysql-test/main/mysqldump.test index 340f20a4a51a6..25268c4ae7e67 100644 --- a/mysql-test/main/mysqldump.test +++ b/mysql-test/main/mysqldump.test @@ -32,6 +32,56 @@ INSERT INTO t1 VALUES (1), (2); --exec $MYSQL_DUMP --skip-create-options --skip-comments -X test t1 DROP TABLE t1; +--echo # +--echo # MDEV-32362 Remove generated columns from INSERT statements in dump +--echo # Include list of non-generated columns in INSERT statements for tables +--echo # with one or more generated columns +--echo # + +CREATE DATABASE dump_generated; +USE dump_generated; +CREATE TABLE t1 (pk INTEGER, a INTEGER, b INTEGER, c VARCHAR(16), + sum INTEGER GENERATED ALWAYS AS (a+b), + sub VARCHAR(4) GENERATED ALWAYS AS (SUBSTRING(c, 1, 4)), + key k1(sum), + key k2(sub) +) engine=innodb; +INSERT INTO t1(pk, a, b, c) VALUES (1, 11, 12, 'oneone'), (2, 21, 22, 'twotwo'); +SELECT * FROM t1; +--exec $MYSQL_DUMP dump_generated t1 > $MYSQLTEST_VARDIR/tmp/t1.sql +DELETE FROM t1; +--exec $MYSQL dump_generated < $MYSQLTEST_VARDIR/tmp/t1.sql +SELECT * FROM t1; +--exec $MYSQL_DUMP dump_generated t1 --tab=$MYSQLTEST_VARDIR/tmp/ +DELETE FROM t1; +--replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR +--eval LOAD DATA INFILE '$MYSQLTEST_VARDIR/tmp/t1.txt' INTO TABLE t1 +SELECT * FROM t1; +DROP TABLE t1; + +--echo # A table with regular columns after generated + +CREATE TABLE t2 (pk INTEGER, a INTEGER, b INTEGER, + sum INTEGER GENERATED ALWAYS AS (a+b), + c VARCHAR(16), + key k1(sum) +) engine=innodb; +INSERT INTO t2(pk, a, b, c) VALUES (1, 11, 12, 'oneone'), (2, 21, 22, 'twotwo'); +SELECT * FROM t2; +--exec $MYSQL_DUMP dump_generated t2 > $MYSQLTEST_VARDIR/tmp/t2.sql +DELETE FROM t2; +--exec $MYSQL dump_generated < $MYSQLTEST_VARDIR/tmp/t2.sql +SELECT * FROM t2; +--exec $MYSQL_DUMP dump_generated t2 --tab=$MYSQLTEST_VARDIR/tmp/ +DELETE FROM t2; +--replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR +--eval LOAD DATA INFILE '$MYSQLTEST_VARDIR/tmp/t2.txt' INTO TABLE t2 +SELECT * FROM t2; +DROP TABLE t2; + +DROP DATABASE dump_generated; +USE test; + --echo # --echo # Bug#2005 Long decimal comparison bug. --echo #