详解 Redis 迁移工具 redis-shark4.0.3

Wget https://github.com/tair-opensource/RedisShake/archive/refs/tags/v4.0.5.tar.gz
yum install go 
tar -xvf redis-shake-linux-amd64.tar.gz 
cd redis-shark 
[root@ redis-shark]# ls 
1shake_online.toml 2shake_rdb.toml data redis-shake shake.toml.bak

[root@idc-zabbix12 redis-shark]# vim 1shake_online.toml

function = ""

[sync_reader]
version = "7.0" ###source集群版本
cluster = true ###集群
address = "10.0.0.8:9101" ###source只需填写一个主节点地址即可
username = ""
password = "Bj***w"
tls = false
sync_rdb = true 
sync_aof = true
prefer_replica = true

# [scan_reader]
# cluster = false # set to true if source is a redis cluster
# address = "127.0.0.1:6379" # when cluster is true, set address to one of the cluster node
# username = "" # keep empty if not using ACL
# password = "" # keep empty if no authentication is required
# ksn = false # set to true to enabled Redis keyspace notifications (KSN) subscription
# tls = false
# dbs = [] # set you want to scan dbs such as [1,5,7], if you don't want to scan all
# prefer_replica = true # set to true if you want to sync from replica node

##文件依次执行
[rdb_reader]
#filepath = "/opt/bakredis-shake.bak/local_dump.0"
#filepath = "/opt/bakredis-shake.bak/local_dump.1"
#filepath = "/opt/bakredis-shake.bak/local_dump.2"

# [aof_reader]
# filepath = "/tmp/.aof"
# timestamp = 0 # subsecond

[redis_writer]
version = "7.0" ###目标集群版本
cluster = true ###集群
address = "10.28.29.51:9101" ###target集群只需填写一个主节点地址即可
username = ""
password = "B***Zw"
tls = false
off_reply = false

[advanced]
dir = "data" ###生成的数据目录
ncpu = 4 # runtime.GOMAXPROCS, 0 means use runtime.NumCPU() cpu cores
pprof_port = 0 # pprof port, 0 means disable
status_port = 0 # status port, 0 means disable

# log
log_file = "shake.log" ###日志文件
log_level = "info" # debug, info or warn
log_interval = 5 # in seconds

rdb_restore_command_behavior = "panic" # panic, rewrite or skip
pipeline_count_limit = 1024
target_redis_client_max_querybuf_len = 1024_000_000
target_redis_proto_max_bulk_len = 512_000_000
aws_psync = "" # example: aws_psync = "10.0.0.1:6379@nmfu2sl5osync,10.0.0.1:6379@xhma21xfkssync"
empty_db_before_sync = false

[module]
target_mbbloom_version = 20603

[root@idc-zabbix12 redis-shark]# ./redis-shake 1shake_online.toml

[root@idc-zabbix12 redis-shark]# cat 2shake_rdb.toml 
function = ""
#[sync_reader]
#version = "7.0"
#cluster = true
#address = "10.28.29.8:9101"
#username = ""
#password = "Bjmr0cakP7Zw"
#tls = false
#sync_rdb = true 
#sync_aof = true
#prefer_replica = true

# [scan_reader]
# cluster = false # set to true if source is a redis cluster
# address = "127.0.0.1:6379" # when cluster is true, set address to one of the cluster node
# username = "" # keep empty if not using ACL
# password = "" # keep empty if no authentication is required
# ksn = false # set to true to enabled Redis keyspace notifications (KSN) subscription
# tls = false
# dbs = [] # set you want to scan dbs such as [1,5,7], if you don't want to scan all
# prefer_replica = true # set to true if you want to sync from replica node

##文件依次执行 0、1、2 
[rdb_reader]
filepath = "/opt/bakredis-shake.bak/local_dump.0" ###rdb文件，source集群是3分片集群
#filepath = "/opt/bakredis-shake.bak/local_dump.1"
#filepath = "/opt/bakredis-shake.bak/local_dump.2"

# [aof_reader]
# filepath = "/tmp/.aof"
# timestamp = 0 # subsecond

[redis_writer]
version = "7.0" 
cluster = true
address = "10.0.0.51:9101" ####目标导入集群
username = ""
password = "Bj***Zw"
tls = false
off_reply = false

[advanced]
dir = "data"
ncpu = 4 # runtime.GOMAXPROCS, 0 means use runtime.NumCPU() cpu cores
pprof_port = 0 # pprof port, 0 means disable
status_port = 0 # status port, 0 means disable

# log
log_file = "shake.log"
log_level = "info" # debug, info or warn
log_interval = 5 # in seconds

rdb_restore_command_behavior = "panic" # panic, rewrite or skip
pipeline_count_limit = 1024
target_redis_client_max_querybuf_len = 1024_000_000
target_redis_proto_max_bulk_len = 512_000_000
aws_psync = "" # example: aws_psync = "10.0.0.1:6379@nmfu2sl5osync,10.0.0.1:6379@xhma21xfkssync"
empty_db_before_sync = false

[module]
target_mbbloom_version = 20603

[root@idc-zabbix12 redis-shark]# ./redis-shake 2shake_rdb.toml

[root@idc-zabbix12 redis-shark]# ./redis-shake 2shake_rdb.toml

[root@idc-zabbix12 redis-shark]# ./redis-shake 2shake_rdb.toml

[scan_reader]
cluster = false # set to true if source is a redis cluster
address = "127.0.0.1:6379" # when cluster is true, set address to one of the cluster node
username = "" # keep empty if not using ACL
password = "" # keep empty if no authentication is required
tls = false
ksn = false # set to true to enabled Redis keyspace notifications (KSN) subscription
dbs = [] # set you want to scan dbs, if you don't want to scan all

function = """
shake.log(DB)
if DB == 0
then
 return
end
shake.call(DB, ARGV)
"""

[sync_reader]
address = "127.0.0.1:6379"

[redis_writer]
address = "127.0.0.1:6380"

local prefix = "user:"
local prefix_len = #prefix

if string.sub(KEYS[1], 1, prefix_len) ~= prefix then
 return
end

shake.call(DB, ARGV)

shake.log(DB)
if DB == 0
then
 return
end
shake.call(DB, ARGV)

if GROUP == "HASH" then
 return
end
shake.call(DB, ARGV)

if GROUP == "SCRIPTING" then
 return
end
shake.call(DB, ARGV)

local prefix_old = "prefix_old_"
local prefix_new = "prefix_new_"

shake.log("old=" .. table.concat(ARGV, " "))

for i, index in ipairs(KEY_INDEXES) do
 local key = ARGV[index]
 if string.sub(key, 1, #prefix_old) == prefix_old then
 ARGV[index] = prefix_new .. string.sub(key, #prefix_old + 1)
 end
end

shake.log("new=" .. table.concat(ARGV, " "))
shake.call(DB, ARGV)

local db1 = 1
local db2 = 2

if DB == db1 then
 DB = db2
elseif DB == db2 then
 DB = db1
end
shake.call(DB, ARGV)