Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

迁移数据库的时候,创建语句中有转义符,迁移尚未识别到 #324

Open
huoxiaojun2009 opened this issue Feb 18, 2025 · 1 comment
Labels
bug Something isn't working

Comments

@huoxiaojun2009
Copy link

./ape-dts task_config.ini.struct

2025-02-18T16:35:21.787408100+08:00 INFO default_logger - ddl begin: CREATE TABLE IF NOT EXISTS dbops.t_db_data_sync_task (id bigint(20) auto_increment COMMENT '主键ID' NOT NULL ,creator_id bigint(20) unsigned NOT NULL ,creator varchar(32) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL ,create_time datetime NOT NULL ,update_time datetime NOT NULL ,modifier varchar(32) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL ,modifier_id bigint(20) unsigned NOT NULL ,task_name varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci COMMENT '任务名' NOT NULL ,task_cron varchar(50) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci COMMENT '任务Cron表达式' NOT NULL ,src_db_id bigint(20) COMMENT '源数据库ID' NOT NULL ,src_db_name varchar(100) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci COMMENT '源数据库名' NULL ,src_tag_path varchar(200) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci COMMENT '源数据库tag路径' NULL ,target_db_id bigint(20) COMMENT '目标数据库ID' NOT NULL ,target_db_name varchar(150) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci COMMENT '目标数据库名' NULL ,target_tag_path varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci COMMENT '目标数据库tag路径' NULL ,target_table_name varchar(150) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci COMMENT '目标数据库表名' NULL ,data_sql text CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci COMMENT '数据查询sql' NOT NULL ,page_size bigint(20) COMMENT '数据同步分页大小' NOT NULL ,upd_field varchar(100) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci DEFAULT 'id' COMMENT '更新字段,默认'id'' NOT NULL ,upd_field_val varchar(100) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci COMMENT '当前更新值' NULL ,id_rule tinyint(2) DEFAULT '1' COMMENT 'id生成规则:1、MD5(时间戳+更新字段的值)。2、无(不自动生成id,选择无的时候需要指定主键ID字段是数据源哪个字段)' NOT NULL ,pk_field varchar(100) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci DEFAULT 'id' COMMENT '主键id字段名,默认"id"' NULL ,field_map text CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci COMMENT '字段映射json' NULL ,is_deleted tinyint(8) DEFAULT '0' NULL ,delete_time datetime NULL ,status tinyint(4) DEFAULT '1' COMMENT '状态 1启用 2禁用' NOT NULL ,recent_state tinyint(4) DEFAULT '0' COMMENT '最近执行状态 1成功 -1失败' NOT NULL ,task_key varchar(100) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci COMMENT '任务唯一标识' NULL ,running_state tinyint(1) DEFAULT '2' COMMENT '运行时状态 1运行中、2待运行、3已停止' NULL ,duplicate_strategy tinyint(1) DEFAULT '-1' COMMENT '唯一键冲突策略 -1:无,1:忽略,2:覆盖' NULL ,upd_field_src varchar(191) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci COMMENT '更新值来源, 如select name as user_name from user' NULL , PRIMARY KEY (id)) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci COMMENT='数据同步'
2025-02-18T16:35:21.788819840+08:00 ERROR default_logger - ddl failed, error: sqlx error: error returned from database: 1064 (42000): You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near 'id'' NOT NULL ,upd_field_val varchar(100) CHARACTER SET utf8mb4 COLLATE utf8m' at line 1
2025-02-18T16:35:21.832808290+08:00 ERROR default_logger - panic: panicked at /project/dt-task/src/task_runner.rs:326:36:
called Result::unwrap() on an Err value: sqlx error: error returned from database: 1064 (42000): You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near 'id'' NOT NULL ,upd_field_val varchar(100) CHARACTER SET utf8mb4 COLLATE utf8m' at line 1

Caused by:
0: error returned from database: 1064 (42000): You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near 'id'' NOT NULL ,upd_field_val varchar(100) CHARACTER SET utf8mb4 COLLATE utf8m' at line 1
1: 1064 (42000): You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near 'id'' NOT NULL ,upd_field_val varchar(100) CHARACTER SET utf8mb4 COLLATE utf8m' at line 1

Stack backtrace:
0: anyhow::error::<impl core::convert::From for anyhow::Error>::from
1: dt_connector::sinker::base_struct_sinker::BaseStructSinker::sink_structs::{{closure}}
2: <dt_connector::sinker::mysql::mysql_struct_sinker::MysqlStructSinker as dt_connector::Sinker>::sink_struct::{{closure}}
3: <dt_parallelizer::serial_parallelizer::SerialParallelizer as dt_parallelizer::Parallelizer>::sink_struct::{{closure}}
4: <dt_pipeline::base_pipeline::BasePipeline as dt_pipeline::Pipeline>::start::{{closure}}
5: dt_task::task_runner::TaskRunner::start_single_task::{{closure}}::{{closure}}
6: tokio::runtime::task::raw::poll
7: tokio::runtime::scheduler::multi_thread::worker::Context::run_task
8: tokio::runtime::scheduler::multi_thread::worker::Context::run
9: tokio::runtime::context::runtime::enter_runtime
10: tokio::runtime::scheduler::multi_thread::worker::run
11: <tokio::runtime::blocking::task::BlockingTask as core::future::future::Future>::poll
12: tokio::runtime::task::core::Core<T,S>::poll
13: tokio::runtime::task::harness::Harness<T,S>::poll
14: tokio::runtime::blocking::pool::Inner::run
15: std::sys::backtrace::__rust_begin_short_backtrace
16: core::ops::function::FnOnce::call_once{{vtable.shim}}
17: std::sys::pal::unix::thread::Thread::new::thread_start
18: start_thread
at /data/software/huoxj/glibc-2.18/nptl/pthread_create.c:309
19: __clone
at /data/software/huoxj/glibc-2.18/misc/../sysdeps/unix/sysv/linux/x86_64/clone.S:111
2025-02-18T16:35:21.833140898+08:00 ERROR default_logger - panic: panicked at /project/dt-task/src/task_runner.rs:341:31:
called Result::unwrap() on an Err value: JoinError::Panic(Id(18), "called Result::unwrap() on an Err value: sqlx error: error returned from database: 1064 (42000): You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near 'id'' NOT NULL ,upd_field_val varchar(100) CHARACTER SET utf8mb4 COLLATE utf8m' at line 1\n\nCaused by:\n 0: error returned from database: 1064 (42000): You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near 'id'' NOT NULL ,upd_field_val varchar(100) CHARACTER SET utf8mb4 COLLATE utf8m' at line 1\n 1: 1064 (42000): You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near 'id'' NOT NULL ,upd_field_val varchar(100) CHARACTER SET utf8mb4 COLLATE utf8m' at line 1\n\nStack backtrace:\n 0: anyhow::error::<impl core::convert::From for anyhow::Error>::from\n 1: dt_connector::sinker::base_struct_sinker::BaseStructSinker::sink_structs::{{closure}}\n 2: <dt_connector::sinker::mysql::mysql_struct_sinker::MysqlStructSinker as dt_connector::Sinker>::sink_struct::{{closure}}\n 3: <dt_parallelizer::serial_parallelizer::SerialParallelizer as dt_parallelizer::Parallelizer>::sink_struct::{{closure}}\n 4: <dt_pipeline::base_pipeline::BasePipeline as dt_pipeline::Pipeline>::start::{{closure}}\n 5: dt_task::task_runner::TaskRunner::start_single_task::{{closure}}::{{closure}}\n 6: tokio::runtime::task::raw::poll\n 7: tokio::runtime::scheduler::multi_thread::worker::Context::run_task\n 8: tokio::runtime::scheduler::multi_thread::worker::Context::run\n 9: tokio::runtime::context::runtime::enter_runtime\n 10: tokio::runtime::scheduler::multi_thread::worker::run\n 11: <tokio::runtime::blocking::task::BlockingTask as core::future::future::Future>::poll\n 12: tokio::runtime::task::core::Core<T,S>::poll\n 13: tokio::runtime::task::harness::Harness<T,S>::poll\n 14: tokio::runtime::blocking::pool::Inner::run\n 15: std::sys::backtrace::__rust_begin_short_backtrace\n 16: core::ops::function::FnOnce::call_once{{vtable.shim}}\n 17: std::sys::pal::unix::thread::Thread::new::thread_start\n 18: start_thread\n at /data/software/huoxj/glibc-2.18/nptl/pthread_create.c:309\n 19: __clone\n at /data/software/huoxj/glibc-2.18/misc/../sysdeps/unix/sysv/linux/x86_64/clone.S:111", ...)
[root@oraem dts]#

源table创建语句
CREATE TABLE t_db_data_sync_task (
id BIGINT AUTO_INCREMENT NOT NULL COMMENT '主键ID' ,
creator_id BIGINT UNSIGNED NOT NULL,
creator VARCHAR(32) NOT NULL,
create_time DATETIME NOT NULL,
update_time DATETIME NOT NULL,
modifier VARCHAR(32) NOT NULL,
modifier_id BIGINT UNSIGNED NOT NULL,
task_name VARCHAR(255) NOT NULL COMMENT '任务名' ,
task_cron VARCHAR(50) NOT NULL COMMENT '任务Cron表达式' ,
src_db_id BIGINT NOT NULL COMMENT '源数据库ID' ,
src_db_name VARCHAR(100) NULL COMMENT '源数据库名' ,
src_tag_path VARCHAR(200) NULL COMMENT '源数据库tag路径' ,
target_db_id BIGINT NOT NULL COMMENT '目标数据库ID' ,
target_db_name VARCHAR(150) NULL COMMENT '目标数据库名' ,
target_tag_path VARCHAR(255) NULL COMMENT '目标数据库tag路径' ,
target_table_name VARCHAR(150) NULL COMMENT '目标数据库表名' ,
data_sql TEXT NOT NULL COMMENT '数据查询sql' ,
page_size BIGINT NOT NULL COMMENT '数据同步分页大小' ,
upd_field VARCHAR(100) NOT NULL DEFAULT 'id' COMMENT '更新字段,默认'id'' ,
upd_field_val VARCHAR(100) NULL COMMENT '当前更新值' ,
id_rule TINYINT NOT NULL DEFAULT 1 COMMENT 'id生成规则:1、MD5(时间戳+更新字段的值)。2、无(不自动生成id,选择无的时候需要指定主键ID字段是数据源哪个字段)' ,
pk_field VARCHAR(100) NULL DEFAULT 'id' COMMENT '主键id字段名,默认"id"' ,
field_map TEXT NULL COMMENT '字段映射json' ,
is_deleted TINYINT NULL DEFAULT 0 ,
delete_time DATETIME NULL,
status TINYINT NOT NULL DEFAULT 1 COMMENT '状态 1启用 2禁用' ,
recent_state TINYINT NOT NULL DEFAULT 0 COMMENT '最近执行状态 1成功 -1失败' ,
task_key VARCHAR(100) NULL COMMENT '任务唯一标识' ,
running_state TINYINT NULL DEFAULT 2 COMMENT '运行时状态 1运行中、2待运行、3已停止' ,
duplicate_strategy TINYINT NULL DEFAULT -1 COMMENT '唯一键冲突策略 -1:无,1:忽略,2:覆盖' ,
upd_field_src VARCHAR(191) NULL COMMENT '更新值来源, 如select name as user_name from user' ,
PRIMARY KEY (id)
)
ENGINE = InnoDB
COMMENT = '数据同步';

主要是问题是,在create 语句中使用了 upd_field VARCHAR(100) NOT NULL DEFAULT 'id' COMMENT '更新字段,默认'id'' , 做转义,在做数据迁移的时候没有识别到。
ape_dts日志中创建数据库的sql:
CREATE TABLE IF NOT EXISTS dbops.t_db_data_sync_task (
id bigint(20) auto_increment COMMENT '主键ID' NOT NULL,
creator_id bigint(20) unsigned NOT NULL,
creator varchar(32) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL,
create_time datetime NOT NULL,
update_time datetime NOT NULL,
modifier varchar(32) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL,
modifier_id bigint(20) unsigned NOT NULL,
task_name varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci COMMENT '任务名' NOT NULL,
task_cron varchar(50) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci COMMENT '任务Cron表达式' NOT NULL,
src_db_id bigint(20) COMMENT '源数据库ID' NOT NULL,
src_db_name varchar(100) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci COMMENT '源数据库名' NULL,
src_tag_path varchar(200) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci COMMENT '源数据库tag路径' NULL,
target_db_id bigint(20) COMMENT '目标数据库ID' NOT NULL,
target_db_name varchar(150) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci COMMENT '目标数据库名' NULL,
target_tag_path varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci COMMENT '目标数据库tag路径' NULL,
target_table_name varchar(150) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci COMMENT '目标数据库表名' NULL,
data_sql text CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci COMMENT '数据查询sql' NOT NULL,
page_size bigint(20) COMMENT '数据同步分页大小' NOT NULL,
upd_field varchar(100) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci DEFAULT 'id' COMMENT '更新字段,默认' id '' NOT NULL,
upd_field_val varchar(100) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci COMMENT '当前更新值' NULL,
id_rule tinyint(2) DEFAULT '1' COMMENT 'id生成规则:1、MD5(时间戳+更新字段的值)。2、无(不自动生成id,选择无的时候需要指定主键ID字段是数据源哪个字段)' NOT NULL,
pk_field varchar(100) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci DEFAULT 'id' COMMENT '主键id字段名,默认"id"' NULL,
field_map text CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci COMMENT '字段映射json' NULL,
is_deleted tinyint(8) DEFAULT '0' NULL,
delete_time datetime NULL,
status tinyint(4) DEFAULT '1' COMMENT '状态 1启用 2禁用' NOT NULL,
recent_state tinyint(4) DEFAULT '0' COMMENT '最近执行状态 1成功 -1失败' NOT NULL,
task_key varchar(100) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci COMMENT '任务唯一标识' NULL,
running_state tinyint(1) DEFAULT '2' COMMENT '运行时状态 1运行中、2待运行、3已停止' NULL,
duplicate_strategy tinyint(1) DEFAULT '-1' COMMENT '唯一键冲突策略 -1:无,1:忽略,2:覆盖' NULL,
upd_field_src varchar(191) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci COMMENT '更新值来源, 如select name as user_name from user' NULL,
PRIMARY KEY (id)
) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4 COLLATE = utf8mb4_unicode_ci COMMENT = '数据同步'

@huoxiaojun2009 huoxiaojun2009 added the bug Something isn't working label Feb 18, 2025
@qianyiwen2019
Copy link
Collaborator

OK,我们看下这个问题

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Labels
bug Something isn't working
Projects
None yet
Development

No branches or pull requests

2 participants