docker + canal + canal-adapter + es 实现 数据库到es 数据转化增量同步

需求

众所周知，mysql是用来存储数据的，适合查询，但是遇上全文搜索和排序之类的，就变得超级慢，尤其面对大数据量时,针对大结果集分组排序，mysql就不再合适，所以可以考虑将mysql 数据同步至es中，用es来做全文搜索和排序分组等，中间会涉及同步数据问题，就用canal来解决

docker

  es:
    build:
      context: ./services/elasticsearch
      args:
        ELASTICSEARCH_VERSION: ${ELASTICSEARCH_VERSION}
        ELASTICSEARCH_PLUGINS: ${ELASTICSEARCH_PLUGINS}
    container_name: es
    environment:
      - TZ=$TZ
      - discovery.type=single-node
      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
    volumes:
      - ${DATA_DIR}/esdata:/usr/share/elasticsearch/data
      - ${ELASTICSEARCH_CONF_FILE}:/usr/share/elasticsearch/config/elasticsearch.yml
    hostname: elasticsearch
    restart: always
    ports:
      - "${ELASTICSEARCH_HOST_PORT_C}:9200"
      - "${ELASTICSEARCH_HOST_PORT_S}:9300"

  kibana:
    image: kibana:${KIBANA_VERSION}
    container_name: kibana
    environment:
      TZ: "$TZ"
      elasticsearch.hosts: http://es:9200
      I18N_LOCALE: "${KIBANA_I18N_LOCALE}"
    hostname: kibana
    depends_on:
      - es
    restart: always
    ports:
      - "${KIBANA_HOST}:5601"
    canal:
      image: canal/canal-server:v1.1.5
      container_name: canal
      volumes:
        - ${DATA_DIR}/canal/instance.properties:/home/admin/canal-server/conf/example/instance.properties
        - ${DATA_DIR}/canal/canal.properties:/home/admin/canal-server/conf/canal.properties canal/canal-server
      ports:
        - 11111:11111
        - 8329:8329
      restart: always

    canal-adapter:
      image: slpcat/canal-adapter:v1.1.5
      container_name: canal-adapter
      volumes:
        - ${DATA_DIR}/canal-adapter/conf:/opt/canal-adapter/conf
      ports:
        - 8439:8439
      restart: always
    networks:
      default:
        driver: bridge
        ipam:
          driver: default
          # 解除下面的注释可以设置网段，用于nginx等容器固定容器IP
     #config: #  - subnet: 10.0.0.0/24

canal.properties

#################################################
#########         common argument        #############
#################################################
# tcp bind ip
canal.ip =
# register ip to zookeeper
canal.register.ip =
canal.port = 11111
canal.metrics.pull.port = 11112
# canal instance user/passwd
# canal.user = canal
# canal.passwd = E3619321C1A937C46A0D8BD1DAC39F93B27D4458

# canal admin config
#canal.admin.manager = 127.0.0.1:8329
canal.admin.port = 11110
canal.admin.user = admin
canal.admin.passwd = 4ACFE3202A5FF5CF467898FC58AAB1D615029441
# admin auto register
#canal.admin.register.auto = true
#canal.admin.register.cluster =
#canal.admin.register.name =

canal.zkServers =
# flush data to zk
canal.zookeeper.flush.period = 1000
canal.withoutNetty = false
# tcp, kafka, rocketMQ, rabbitMQ, pulsarMQ
canal.serverMode = tcp
# flush meta cursor/parse position to file
canal.file.data.dir = ${canal.conf.dir}
canal.file.flush.period = 1000
## memory store RingBuffer size, should be Math.pow(2,n)
canal.instance.memory.buffer.size = 16384
## memory store RingBuffer used memory unit size , default 1kb
canal.instance.memory.buffer.memunit = 1024 
## meory store gets mode used MEMSIZE or ITEMSIZE
canal.instance.memory.batch.mode = MEMSIZE
canal.instance.memory.rawEntry = true

## detecing config
canal.instance.detecting.enable = false
#canal.instance.detecting.sql = insert into retl.xdual values(1,now()) on duplicate key update x=now()
canal.instance.detecting.sql = select 1
canal.instance.detecting.interval.time = 3
canal.instance.detecting.retry.threshold = 3
canal.instance.detecting.heartbeatHaEnable = false

# support maximum transaction size, more than the size of the transaction will be cut into multiple transactions delivery
canal.instance.transaction.size =  1024
# mysql fallback connected to new master should fallback times
canal.instance.fallbackIntervalInSeconds = 60

# network config
canal.instance.network.receiveBufferSize = 16384
canal.instance.network.sendBufferSize = 16384
canal.instance.network.soTimeout = 30

# binlog filter config
canal.instance.filter.druid.ddl = true
canal.instance.filter.query.dcl = false
canal.instance.filter.query.dml = false
canal.instance.filter.query.ddl = false
canal.instance.filter.table.error = false
canal.instance.filter.rows = false
canal.instance.filter.transaction.entry = false
canal.instance.filter.dml.insert = false
canal.instance.filter.dml.update = false
canal.instance.filter.dml.delete = false

# binlog format/image check
canal.instance.binlog.format = ROW,STATEMENT,MIXED 
canal.instance.binlog.image = FULL,MINIMAL,NOBLOB

# binlog ddl isolation
canal.instance.get.ddl.isolation = false

# parallel parser config
canal.instance.parser.parallel = true
## concurrent thread number, default 60% available processors, suggest not to exceed Runtime.getRuntime().availableProcessors()
#canal.instance.parser.parallelThreadSize = 16
## disruptor ringbuffer size, must be power of 2
canal.instance.parser.parallelBufferSize = 256

# table meta tsdb info
canal.instance.tsdb.enable = true
canal.instance.tsdb.dir = ${canal.file.data.dir:../conf}/${canal.instance.destination:}
canal.instance.tsdb.url = jdbc:h2:${canal.instance.tsdb.dir}/h2;CACHE_SIZE=1000;MODE=MYSQL;
canal.instance.tsdb.dbUsername = canal
canal.instance.tsdb.dbPassword = canal
# dump snapshot interval, default 24 hour
canal.instance.tsdb.snapshot.interval = 24
# purge snapshot expire , default 360 hour(15 days)
canal.instance.tsdb.snapshot.expire = 360

#################################################
#########         destinations        #############
#################################################
canal.destinations = example
# conf root dir
canal.conf.dir = ../conf
# auto scan instance dir add/remove and start/stop instance
canal.auto.scan = true
canal.auto.scan.interval = 5
# set this value to 'true' means that when binlog pos not found, skip to latest.
# WARN: pls keep 'false' in production env, or if you know what you want.
canal.auto.reset.latest.pos.mode = false

canal.instance.tsdb.spring.xml = classpath:spring/tsdb/h2-tsdb.xml
#canal.instance.tsdb.spring.xml = classpath:spring/tsdb/mysql-tsdb.xml

canal.instance.global.mode = spring
canal.instance.global.lazy = false
canal.instance.global.manager.address = ${canal.admin.manager}
#canal.instance.global.spring.xml = classpath:spring/memory-instance.xml
canal.instance.global.spring.xml = classpath:spring/file-instance.xml
#canal.instance.global.spring.xml = classpath:spring/default-instance.xml

##################################################
#########           MQ Properties      #############
##################################################
# aliyun ak/sk , support rds/mq
canal.aliyun.accessKey =
canal.aliyun.secretKey =
canal.aliyun.uid=

canal.mq.flatMessage = true
canal.mq.canalBatchSize = 50
canal.mq.canalGetTimeout = 100
# Set this value to "cloud", if you want open message trace feature in aliyun.
canal.mq.accessChannel = local

canal.mq.database.hash = true
canal.mq.send.thread.size = 30
canal.mq.build.thread.size = 8

##################################################
#########              Kafka              #############
##################################################
kafka.bootstrap.servers = 127.0.0.1:9092
kafka.acks = all
kafka.compression.type = none
kafka.batch.size = 16384
kafka.linger.ms = 1
kafka.max.request.size = 1048576
kafka.buffer.memory = 33554432
kafka.max.in.flight.requests.per.connection = 1
kafka.retries = 0

kafka.kerberos.enable = false
kafka.kerberos.krb5.file = ../conf/kerberos/krb5.conf
kafka.kerberos.jaas.file = ../conf/kerberos/jaas.conf

# sasl demo
# kafka.sasl.jaas.config = org.apache.kafka.common.security.scram.ScramLoginModule required \\n username=\"alice\" \\npassword="alice-secret\";
# kafka.sasl.mechanism = SCRAM-SHA-512
# kafka.security.protocol = SASL_PLAINTEXT

##################################################
#########             RocketMQ         #############
##################################################
rocketmq.producer.group = test
rocketmq.enable.message.trace = false
rocketmq.customized.trace.topic =
rocketmq.namespace =
rocketmq.namesrv.addr = 127.0.0.1:9876
rocketmq.retry.times.when.send.failed = 0
rocketmq.vip.channel.enabled = false
rocketmq.tag = 

##################################################
#########             RabbitMQ         #############
##################################################
rabbitmq.host =
rabbitmq.virtual.host =
rabbitmq.exchange =
rabbitmq.username =
rabbitmq.password =
rabbitmq.deliveryMode =


##################################################
#########               Pulsar         #############
##################################################
pulsarmq.serverUrl =
pulsarmq.roleToken =
pulsarmq.topicTenantPrefix =

instance.properties

#################################################
## mysql serverId , v1.0.26+ will autoGen
# canal.instance.mysql.slaveId=0

# enable gtid use true/false
canal.instance.gtidon=false

# position info
canal.instance.master.address=mysql:3306
canal.instance.master.journal.name=
canal.instance.master.position=
canal.instance.master.timestamp=
canal.instance.master.gtid=

# rds oss binlog
canal.instance.rds.accesskey=
canal.instance.rds.secretkey=
canal.instance.rds.instanceId=

# table meta tsdb info
canal.instance.tsdb.enable=true
#canal.instance.tsdb.url=jdbc:mysql://127.0.0.1:3306/canal_tsdb
#canal.instance.tsdb.dbUsername=canal
#canal.instance.tsdb.dbPassword=canal

#canal.instance.standby.address =
#canal.instance.standby.journal.name =
#canal.instance.standby.position =
#canal.instance.standby.timestamp =
#canal.instance.standby.gtid=

# username/password
canal.instance.dbUsername=root
canal.instance.dbPassword=123456
canal.instance.connectionCharset = UTF-8
# enable druid Decrypt database password
canal.instance.enableDruid=false
#canal.instance.pwdPublicKey=MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBALK4BUxdDltRRE5/zXpVEVPUgunvscYFtEip3pmLlhrWpacX7y7GCMo2/JM6LeHmiiNdH1FWgGCpUfircSwlWKUCAwEAAQ==

# table regex
canal.instance.filter.regex=sass_www17\\.dt_sell_5,sass_www17\\.dt_company,sass_www17\\.dt_school,sass_www17\\.dt_member
# table black regex
canal.instance.filter.black.regex=mysql\\.slave_.*
# table field filter(format: schema1.tableName1:field1/field2,schema2.tableName2:field1/field2)
#canal.instance.filter.field=test1.t_product:id/subject/keywords,test2.t_company:id/name/contact/ch
# table field black filter(format: schema1.tableName1:field1/field2,schema2.tableName2:field1/field2)
#canal.instance.filter.black.field=test1.t_product:subject/product_image,test2.t_company:id/name/contact/ch

# mq config
canal.mq.topic=example
# dynamic topic route by schema or table regex
#canal.mq.dynamicTopic=mytest1.user,topic2:mytest2\\..*,.*\\..*
canal.mq.partition=0
# hash partition config
#canal.mq.enableDynamicQueuePartition=false
#canal.mq.partitionsNum=3
#canal.mq.dynamicTopicPartitionNum=test.*:4,mycanal:6
#canal.mq.partitionHash=test.table:id^name,.*\\..*
#
# multi stream for polardbx
canal.instance.multi.stream.on=false
#################################################

• canal.destinations 要与 canal.mq.topic 一致
• canal.instance.filter.regex 一定要指定 不然不会进行同步
• canal.instance.dbUsername canal.instance.dbPassword canal.instance.master.address=mysql:3306 需要自己填写

canal-adapter

配置文件路径

• canal-adapter/conf/application.yml
• canal-adapter/conf/es7/ceshi.yml

application.yml

server:
  port: 8439
spring:
  jackson:
    date-format: yyyy-MM-dd HH:mm:ss
    time-zone: GMT+8
    default-property-inclusion: non_null

canal.conf:
  mode: tcp
  flatMessage: true
  zookeeperHosts:
  syncBatchSize: 1000
  retries: 0
  timeout:
  accessKey:
  secretKey:
  consumerProperties:
    # canal tcp consumer
    canal.tcp.server.host: canal:11111
    canal.tcp.batch.size: 500
    canal.tcp.username:
    canal.tcp.password:
  srcDataSources:
    defaultDS:
      url: jdbc:mysql://mysql:3306/sass_www17?useUnicode=true&useSSL=false
      username: root
      password: 123456
  canalAdapters:
    - instance: example # canal instance Name or mq topic name
      groups:
        - groupId: g1
          outerAdapters:
            - name: logger
            - name: es7 # 该版本发现只能是es7/es6
              hosts: es:9300 # 127.0.0.1:9200 for rest mode
              properties:
                mode: transport # transport # or rest
                cluster.name: docker-cluster

ceshi.yml

dataSourceKey: defaultDS
destination: example
groupId: g1
esMapping:
  _index: default_view_courses
  _id: _id
  _type: _doc
  upsert: true
  #  pk: id
  sql: "SELECT a.itemid as _id, a.itemid, a.areaid, a.catid, a.typecatid, a.typeid , a.req_id, b.userid AS school_userid, b.company AS school_zh_name, b.companyen AS school_name , b.thumb AS school_logo, b.homepage AS school_url, b.linkurl AS school_zh_url, b.username, b.ranking AS school_ranking , a.schoolid AS college_id, c.title AS college_name, a.ranking, a.titleen , a.url, a.mintitle, a.duration, a.durationunit, a.price , a.priceunit, a.uniacid FROM dt_sell_5 a LEFT JOIN dt_company b ON b.username = a.username LEFT JOIN dt_school c ON c.itemid = a.schoolid"
  #  objFields:
  #    _labels: array:;
  # etlCondition: " WHERE `dt_sell_5`.`status` IN (1, 2, 3) AND (`dt_company`.`groupid` >= 6 OR (`dt_member`.`groupid` > 4 AND `dt_member`.`regid` >= 6)) AND `dt_company`.`groupid` NOT IN (11, 15) AND `dt_sell_5`.`typeid` IN (0, 1, 2, 3, 4, 5)"
  commitBatch: 3000

一键启动服务

docker compose up -d

查看canal log

[root@localhost dnmp]# docker compose  logs canal
WARN[0000] /root/dnmp/docker-compose.yml: `version` is obsolete 
canal  | DOCKER_DEPLOY_TYPE=VM
canal  | ==> INIT /alidata/init/02init-sshd.sh
canal  | ==> EXIT CODE: 0
canal  | ==> INIT /alidata/init/fix-hosts.py
canal  | ==> EXIT CODE: 0
canal  | ==> INIT DEFAULT
canal  | Generating SSH1 RSA host key: [  OK  ]
canal  | Starting sshd: [  OK  ]
canal  | Starting crond: [  OK  ]
canal  | ==> INIT DONE
canal  | ==> RUN /home/admin/app.sh
canal  | ==> START ...
canal  | start canal ...
canal  | start canal successful
canal  | ==> START SUCCESSFUL ...
canal  | DOCKER_DEPLOY_TYPE=VM
canal  | ==> INIT /alidata/init/02init-sshd.sh
canal  | ==> EXIT CODE: 0
canal  | ==> INIT /alidata/init/fix-hosts.py
canal  | ==> EXIT CODE: 0
canal  | ==> INIT DEFAULT
canal  | Starting sshd: [  OK  ]
canal  | Starting crond: [  OK  ]
canal  | ==> INIT DONE
canal  | ==> RUN /home/admin/app.sh
canal  | ==> START ...
canal  | start canal ...
canal  | start canal successful
canal  | ==> START SUCCESSFUL ...
canal  | DOCKER_DEPLOY_TYPE=VM
canal  | ==> INIT /alidata/init/02init-sshd.sh
canal  | ==> EXIT CODE: 0
canal  | ==> INIT /alidata/init/fix-hosts.py
canal  | ==> EXIT CODE: 0
canal  | ==> INIT DEFAULT
canal  | Starting sshd: [  OK  ]
canal  | Starting crond: [  OK  ]
canal  | ==> INIT DONE
canal  | ==> RUN /home/admin/app.sh
canal  | ==> START ...
canal  | start canal ...
canal  | start canal successful
canal  | ==> START SUCCESSFUL ...

canal启动成功，查看配置的实例日志

查看example的日志

docker compose exec canal sh
cd /home/admin/canal-server/logs/example
tail -f example.log

sh-4.1# tail -f example.log 
2024-05-28 10:45:46.070 [main] INFO  c.a.otter.canal.instance.spring.CanalInstanceWithSpring - start CannalInstance for 1-example 
2024-05-28 10:45:46.409 [main] WARN  c.a.o.canal.parse.inbound.mysql.dbsync.LogEventConvert - --> init table filter : ^sass_www17\.dt_company$|^sass_www17\.dt_sell_5$|^sass_www17\.dt_school$|^sass_www17\.dt_member$
2024-05-28 10:45:46.409 [main] WARN  c.a.o.canal.parse.inbound.mysql.dbsync.LogEventConvert - --> init table black filter : ^mysql\.slave_.*$
2024-05-28 10:45:47.273 [destination = example , address = mysql/172.18.0.6:3306 , EventParser] WARN  c.a.o.c.p.inbound.mysql.rds.RdsBinlogEventParserProxy - ---> begin to find start position, it will be long time for reset or first position
2024-05-28 10:45:47.331 [main] INFO  c.a.otter.canal.instance.core.AbstractCanalInstance - start successful....
2024-05-28 10:45:47.528 [destination = example , address = mysql/172.18.0.6:3306 , EventParser] WARN  c.a.o.c.p.inbound.mysql.rds.RdsBinlogEventParserProxy - prepare to find start position just last position
 {"identity":{"slaveId":-1,"sourceAddress":{"address":"mysql","port":3306}},"postion":{"gtid":"","included":false,"journalName":"mysql-bin.000003","position":258012,"serverId":1,"timestamp":1716864048000}}
2024-05-28 10:45:49.840 [destination = example , address = mysql/172.18.0.6:3306 , EventParser] WARN  c.a.o.c.p.inbound.mysql.rds.RdsBinlogEventParserProxy - ---> find start position successfully, EntryPosition[included=false,journalName=mysql-bin.000003,position=258012,serverId=1,gtid=,timestamp=1716864048000] cost : 2216ms , the next step is binlog dump

查看canal-adapter 日志

docker compose logs -f canal-adapter

canal-adapter  | 2024-05-30 10:37:26.450  INFO 7 --- [           main] o.elasticsearch.plugins.PluginsService   : loaded plugin [org.elasticsearch.percolator.PercolatorPlugin]
canal-adapter  | 2024-05-30 10:37:26.450  INFO 7 --- [           main] o.elasticsearch.plugins.PluginsService   : loaded plugin [org.elasticsearch.script.mustache.MustachePlugin]
canal-adapter  | 2024-05-30 10:37:26.450  INFO 7 --- [           main] o.elasticsearch.plugins.PluginsService   : loaded plugin [org.elasticsearch.transport.Netty4Plugin]
canal-adapter  | 2024-05-30 10:37:28.840  INFO 7 --- [           main] c.a.o.c.c.a.e.c.c.ESSyncConfigLoader     : ## Start loading es mapping config ... 
canal-adapter  | 2024-05-30 10:37:28.907  INFO 7 --- [           main] c.a.o.c.c.a.e.c.c.ESSyncConfigLoader     : ## ES mapping config loaded
canal-adapter  | 2024-05-30 10:37:29.167  INFO 7 --- [           main] c.a.o.c.a.l.loader.CanalAdapterLoader    : Load canal adapter: es7 succeed
canal-adapter  | 2024-05-30 10:37:29.174  INFO 7 --- [           main] c.a.o.c.c.core.spi.ExtensionLoader       : extension classpath dir: /opt/canal-adapter/plugin
canal-adapter  | 2024-05-30 10:37:29.229  INFO 7 --- [           main] c.a.o.c.a.l.loader.CanalAdapterLoader    : Start adapter for canal-client mq topic: example-g1 succeed
canal-adapter  | 2024-05-30 10:37:29.229  INFO 7 --- [           main] c.a.o.c.a.l.loader.CanalAdapterService   : ## the canal client adapters are running now ......
canal-adapter  | 2024-05-30 10:37:29.238  INFO 7 --- [       Thread-3] c.a.o.c.a.l.loader.AdapterProcessor      : =============> Start to connect destination: example <=============
canal-adapter  | 2024-05-30 10:37:29.287  INFO 7 --- [           main] o.s.b.w.embedded.tomcat.TomcatWebServer  : Tomcat started on port(s): 8439 (http) with context path ''
canal-adapter  | 2024-05-30 10:37:29.294  INFO 7 --- [           main] c.a.o.c.a.l.CanalAdapterApplication      : Started CanalAdapterApplication in 12.077 seconds (JVM running for 13.25)
canal-adapter  | 2024-05-30 10:37:29.496  INFO 7 --- [       Thread-3] c.a.o.c.a.l.loader.AdapterProcessor      : =============> Subscribe destination: example succeed <=============

canal-adapter 同步操作

update

canal-adapter  | 2024-05-30 10:53:29.522  INFO 7 --- [pool-1-thread-1] c.a.o.c.c.a.logger.LoggerAdapterExample  : DML: {"data":[{"itemid":487795,"catid":636,"mycatid":0,"typeid":0,"areaid":16,"elite":0,"mintitle":"俄罗斯21212测试","title":null,"titleen":"","letter_order":0,"style":"","fee":0.0,"introduce":"","n1":"","n2":"","n3":"","v1":"","v2":"","v3":"","brand":"","unit":"","price":0.0,"minamount":0.0,"amount":0.0,"days":0,"tag":"","keyword":"","pptword":"","hits":0,"thumb":"","thumb1":"","thumb2":"","username":"usyd","groupid":0,"company":"","vip":0,"validated":0,"truename":"","telephone":"","mobile":"","address":"","email":"","msn":"","qq":"","ali":"","skype":"","totime":0,"editor":"admin","edittime":1709621826,"editdate":"2024-03-05","addtime":1709621793,"adddate":"2024-03-05","ip":"192.168.2.101","template":"0","status":3,"linkurl":"","filepath":"","note":"","level":0,"duration":"","academicreq":"","ranking":0,"programnote":"","subjects":"","immigrate":0,"careeroptions":"","jobcareer":"","salary":"","salarycn":"","careerppt":"","employrate":0.0,"priceunit":"","url":"","ieltstotal":0.0,"toefltotal":0.0,"entrymonth":"","star":null,"companyranking":0,"seo_title":null,"tagids":null,"prologue":"","assocode":null,"schoolid":4,"evaluate":"","baidushoufou":0,"baiduurlsubmit":"","baiduxzsubmit":"","baiduxzoriginalsubmit":"","baidumipsubmit":"","baidumsubmit":"","baiduxzlssubmit":"","cricode":"","subdivision":"","academicreqen":"","degree":"","req_id":0,"coursetype":0,"anysubject":0,"typecatid":",6925,","ieltslistening":0.0,"ieltsreading":0.0,"ieltsspeaking":0.0,"ieltswriting":0.0,"toefllistening":0,"toeflreading":0,"toeflspeaking":0,"toeflwriting":0,"ptelistening":0,"ptereading":0,"pteoverall":0,"ptespeaking":0,"petwriting":0,"gpa":0.0,"background":"","campuses":"","firstdate":"","tags":"","gpatype":"","specificdegree":0.0,"workexp":"","premastersupport":"","languageexpired":0,"psesupport":0,"porfessionalqualification":"","practice":"","applytips":"","languageapply":"","applydeadline":"","interview":"","writtenexam":"","writingsample":"","studyproposal":"","ps":"","rl":"","cv":"","courselist":"","gpacert":"","portfolio":"","deposit":"","defer":"","atas":"","gmat":"","degreetitle":"","coursename":"","applicationfees":0,"durationunit":"","duolinguo":"","credits":"","gre":"","syllabus":"","prologuecn":"","creator":"","destinationschool":"","alevel":"","ossd":"","ib":0.0,"bchighshool":0.0,"sat":0,"act":0,"foundation":0.0,"gaokao":0.0,"cet6":"","maxduration":0.0,"ageragegpa":0.0,"teachingtype":"","location":"","state":"","agent":0,"stem":0,"numberofcourse":0,"numberofterm":0,"limitednumber":0,"limitedunit":0,"averagetoefl":0.0,"averageielts":0.0,"averagegre":0.0,"averagegmat":0.0,"averagelsat":0.0,"essay":"","shortanswer":"","optionalessay":"","videoessay":"","qualificationassess":"","lsat":"","mcat":"","schoolid1":"","schoolidzh1":"","schoolid2":"","schoolidzh2":"","toeflehomeedition":"","toeflmybest":"","ieltsindicator":"","ieltsonline":"","pteonline":"","duolinguoliteracy":"","duolinguoproduction":"","duolinguoconversation":"","duolinguocomprehension":"","languagereq":"","test1":"","uniacid":66,"sync_id":91773}],"database":"sass_www17","destination":"example","es":1717037609000,"groupId":"g1","isDdl":false,"old":[{"areaid":15}],"pkNames":["itemid"],"sql":"","table":"dt_sell_5","ts":1717037609520,"type":"UPDATE"}

insert

canal-adapter  | 2024-05-30 10:55:13.312  INFO 7 --- [pool-1-thread-1] c.a.o.c.c.a.logger.LoggerAdapterExample  : DML: {"data":[{"itemid":487807,"catid":636,"mycatid":0,"typeid":0,"areaid":16,"elite":0,"mintitle":"俄罗斯21212测试","title":null,"titleen":"","letter_order":0,"style":"","fee":0.0,"introduce":"","n1":"","n2":"","n3":"","v1":"","v2":"","v3":"","brand":"","unit":"","price":0.0,"minamount":0.0,"amount":0.0,"days":0,"tag":"","keyword":"","pptword":"","hits":0,"thumb":"","thumb1":"","thumb2":"","username":"usyd","groupid":0,"company":"","vip":0,"validated":0,"truename":"","telephone":"","mobile":"","address":"","email":"","msn":"","qq":"","ali":"","skype":"","totime":0,"editor":"admin","edittime":1709621826,"editdate":"2024-03-05","addtime":1709621793,"adddate":"2024-03-05","ip":"192.168.2.101","template":"0","status":3,"linkurl":"","filepath":"","note":"","level":0,"duration":"","academicreq":"","ranking":0,"programnote":"","subjects":"","immigrate":0,"careeroptions":"","jobcareer":"","salary":"","salarycn":"","careerppt":"","employrate":0.0,"priceunit":"","url":"","ieltstotal":0.0,"toefltotal":0.0,"entrymonth":"","star":null,"companyranking":0,"seo_title":null,"tagids":null,"prologue":"","assocode":null,"schoolid":4,"evaluate":"","baidushoufou":0,"baiduurlsubmit":"","baiduxzsubmit":"","baiduxzoriginalsubmit":"","baidumipsubmit":"","baidumsubmit":"","baiduxzlssubmit":"","cricode":"","subdivision":"","academicreqen":"","degree":"","req_id":0,"coursetype":0,"anysubject":0,"typecatid":",6925,","ieltslistening":0.0,"ieltsreading":0.0,"ieltsspeaking":0.0,"ieltswriting":0.0,"toefllistening":0,"toeflreading":0,"toeflspeaking":0,"toeflwriting":0,"ptelistening":0,"ptereading":0,"pteoverall":0,"ptespeaking":0,"petwriting":0,"gpa":0.0,"background":"","campuses":"","firstdate":"","tags":"","gpatype":"","specificdegree":0.0,"workexp":"","premastersupport":"","languageexpired":0,"psesupport":0,"porfessionalqualification":"","practice":"","applytips":"","languageapply":"","applydeadline":"","interview":"","writtenexam":"","writingsample":"","studyproposal":"","ps":"","rl":"","cv":"","courselist":"","gpacert":"","portfolio":"","deposit":"","defer":"","atas":"","gmat":"","degreetitle":"","coursename":"","applicationfees":0,"durationunit":"","duolinguo":"","credits":"","gre":"","syllabus":"","prologuecn":"","creator":"","destinationschool":"","alevel":"","ossd":"","ib":0.0,"bchighshool":0.0,"sat":0,"act":0,"foundation":0.0,"gaokao":0.0,"cet6":"","maxduration":0.0,"ageragegpa":0.0,"teachingtype":"","location":"","state":"","agent":0,"stem":0,"numberofcourse":0,"numberofterm":0,"limitednumber":0,"limitedunit":0,"averagetoefl":0.0,"averageielts":0.0,"averagegre":0.0,"averagegmat":0.0,"averagelsat":0.0,"essay":"","shortanswer":"","optionalessay":"","videoessay":"","qualificationassess":"","lsat":"","mcat":"","schoolid1":"","schoolidzh1":"","schoolid2":"","schoolidzh2":"","toeflehomeedition":"","toeflmybest":"","ieltsindicator":"","ieltsonline":"","pteonline":"","duolinguoliteracy":"","duolinguoproduction":"","duolinguoconversation":"","duolinguocomprehension":"","languagereq":"","test1":"","uniacid":66,"sync_id":91773}],"database":"sass_www17","destination":"example","es":1717037713000,"groupId":"g1","isDdl":false,"old":null,"pkNames":["itemid"],"sql":"","table":"dt_sell_5","ts":1717037713311,"type":"INSERT"}

delete

canal-adapter  | 2024-05-30 10:55:50.495  INFO 7 --- [pool-1-thread-1] c.a.o.c.c.a.logger.LoggerAdapterExample  : DML: {"data":[{"itemid":487807,"catid":636,"mycatid":0,"typeid":0,"areaid":16,"elite":0,"mintitle":"俄罗斯21212测试","title":null,"titleen":"","letter_order":0,"style":"","fee":0.0,"introduce":"","n1":"","n2":"","n3":"","v1":"","v2":"","v3":"","brand":"","unit":"","price":0.0,"minamount":0.0,"amount":0.0,"days":0,"tag":"","keyword":"","pptword":"","hits":0,"thumb":"","thumb1":"","thumb2":"","username":"usyd","groupid":0,"company":"","vip":0,"validated":0,"truename":"","telephone":"","mobile":"","address":"","email":"","msn":"","qq":"","ali":"","skype":"","totime":0,"editor":"admin","edittime":1709621826,"editdate":"2024-03-05","addtime":1709621793,"adddate":"2024-03-05","ip":"192.168.2.101","template":"0","status":3,"linkurl":"","filepath":"","note":"","level":0,"duration":"","academicreq":"","ranking":0,"programnote":"","subjects":"","immigrate":0,"careeroptions":"","jobcareer":"","salary":"","salarycn":"","careerppt":"","employrate":0.0,"priceunit":"","url":"","ieltstotal":0.0,"toefltotal":0.0,"entrymonth":"","star":null,"companyranking":0,"seo_title":null,"tagids":null,"prologue":"","assocode":null,"schoolid":4,"evaluate":"","baidushoufou":0,"baiduurlsubmit":"","baiduxzsubmit":"","baiduxzoriginalsubmit":"","baidumipsubmit":"","baidumsubmit":"","baiduxzlssubmit":"","cricode":"","subdivision":"","academicreqen":"","degree":"","req_id":0,"coursetype":0,"anysubject":0,"typecatid":",6925,","ieltslistening":0.0,"ieltsreading":0.0,"ieltsspeaking":0.0,"ieltswriting":0.0,"toefllistening":0,"toeflreading":0,"toeflspeaking":0,"toeflwriting":0,"ptelistening":0,"ptereading":0,"pteoverall":0,"ptespeaking":0,"petwriting":0,"gpa":0.0,"background":"","campuses":"","firstdate":"","tags":"","gpatype":"","specificdegree":0.0,"workexp":"","premastersupport":"","languageexpired":0,"psesupport":0,"porfessionalqualification":"","practice":"","applytips":"","languageapply":"","applydeadline":"","interview":"","writtenexam":"","writingsample":"","studyproposal":"","ps":"","rl":"","cv":"","courselist":"","gpacert":"","portfolio":"","deposit":"","defer":"","atas":"","gmat":"","degreetitle":"","coursename":"","applicationfees":0,"durationunit":"","duolinguo":"","credits":"","gre":"","syllabus":"","prologuecn":"","creator":"","destinationschool":"","alevel":"","ossd":"","ib":0.0,"bchighshool":0.0,"sat":0,"act":0,"foundation":0.0,"gaokao":0.0,"cet6":"","maxduration":0.0,"ageragegpa":0.0,"teachingtype":"","location":"","state":"","agent":0,"stem":0,"numberofcourse":0,"numberofterm":0,"limitednumber":0,"limitedunit":0,"averagetoefl":0.0,"averageielts":0.0,"averagegre":0.0,"averagegmat":0.0,"averagelsat":0.0,"essay":"","shortanswer":"","optionalessay":"","videoessay":"","qualificationassess":"","lsat":"","mcat":"","schoolid1":"","schoolidzh1":"","schoolid2":"","schoolidzh2":"","toeflehomeedition":"","toeflmybest":"","ieltsindicator":"","ieltsonline":"","pteonline":"","duolinguoliteracy":"","duolinguoproduction":"","duolinguoconversation":"","duolinguocomprehension":"","languagereq":"","test1":"","uniacid":66,"sync_id":91773}],"database":"sass_www17","destination":"example","es":1717037750000,"groupId":"g1","isDdl":false,"old":null,"pkNames":["itemid"],"sql":"","table":"dt_sell_5","ts":1717037750494,"type":"DELETE"}

本作品采用《CC 协议》，转载必须注明作者和本文链接