Redis集群
生产环境配置
2C8G三台，三主三从，一台二实例
注意事项：一台2实例部署，主从节点要交叉，避免主从节点同时宕机影响业务
部署步骤
[*]下载软件包wget http://download.redis.io/releases/redis-5.0.5.tar.gz
[*]安装依赖yum -y install gcc gcc-c++
[*]解压tar.gz包tar xf redis-5.0.5.tar.gz -C /usr/local/
[*]编译cd /usr/local/redis-5.0.5/
make && make install
cp ./src/redis-server /usr/bin/ && cp ./src/redis-cli /usr/bin/
[*]Redis优化echo 511 > /proc/sys/net/core/somaxconn
echo "vm.overcommit_memory = 1" >> /etc/sysctl.conf && echo "net.core.somaxconn = 1024" >> /etc/sysctl.conf
echo never > /sys/kernel/mm/transparent_hugepage/enabled
#开机启动
echo "echo never > /sys/kernel/mm/transparent_hugepage/enabled" >> /etc/rc.d/rc.local
[*]创建目录及配置文件mkdir /usr/local/redis-cluster/{7000,7001}/data -pv
cp redis.conf /usr/local/redis-cluster/7000/
# grep "^" 7000/redis.conf
bind 0.0.0.0
protected-mode yes
port 7000
tcp-backlog 511
timeout 0
tcp-keepalive 30
daemonize yes
supervised no
pidfile /var/run/redis_7000.pid
loglevel notice
logfile "/usr/local/redis-cluster/7000/redis.log"
databases 16
always-show-logo yes
save ""
stop-writes-on-bgsave-error yes
rdbcompression yes
rdbchecksum yes
dbfilename dump.rdb
dir /usr/local/redis-cluster/7000/data
replica-serve-stale-data yes
replica-read-only yes
repl-diskless-sync no
repl-diskless-sync-delay 5
repl-disable-tcp-nodelay no
replica-priority 100
maxclients 10000
maxmemory 3g
maxmemory-policy allkeys-lru
lazyfree-lazy-eviction no
lazyfree-lazy-expire no
lazyfree-lazy-server-del no
replica-lazy-flush no
appendonly no
appendfilename "appendonly.aof"
appendfsync everysec
no-appendfsync-on-rewrite no
auto-aof-rewrite-percentage 100
auto-aof-rewrite-min-size 64mb
aof-load-truncated yes
aof-use-rdb-preamble yes
lua-time-limit 5000
cluster-enabled yes
cluster-config-file nodes-7000.conf
cluster-node-timeout 5000
cluster-require-full-coverage no
slowlog-log-slower-than 10000
slowlog-max-len 128
latency-monitor-threshold 0
notify-keyspace-events ""
hash-max-ziplist-entries 512
hash-max-ziplist-value 64
list-max-ziplist-size -2
list-compress-depth 0
set-max-intset-entries 512
zset-max-ziplist-entries 128
zset-max-ziplist-value 64
hll-sparse-max-bytes 3000
stream-node-max-bytes 4096
stream-node-max-entries 100
activerehashing yes
client-output-buffer-limit normal 0 0 0
client-output-buffer-limit replica 256mb 64mb 60
client-output-buffer-limit pubsub 32mb 8mb 60
hz 10
dynamic-hz yes
aof-rewrite-incremental-fsync yes
rdb-save-incremental-fsync yes
#修改配置信息
cp 7000/redis.conf 7001/
sed -i 's/7000/7001/g' 7001/redis.conf
#同理另外两台节点同理
mkdir /usr/local/redis-cluster/{7002,7003}/data -pv
sed -i 's/7000/7001/g' 7002/redis.conf
sed -i 's/7000/7001/g' 7003/redis.conf
mkdir /usr/local/redis-cluster/{7004,7005}/data -pv
sed -i 's/7000/7004/g' 7004/redis.conf
sed -i 's/7000/7005/g' 7005/redis.conf
[*]创建启动脚本及开启启动#第一个节点
# cat start_redis_cluster.sh
#!/bin/bash
cd /usr/local/redis-cluster
cd 7000 && /usr/bin/redis-server /usr/local/redis-cluster/7000/redis.conf
cd ../7001 &&/usr/bin/redis-server /usr/local/redis-cluster/7001/redis.conf
#第二个节点
# cat start_redis_cluster.sh
#!/bin/bash
cd /usr/local/redis-cluster
cd 7002 && /usr/bin/redis-server /usr/local/redis-cluster/7002/redis.conf
cd ../7003 &&/usr/bin/redis-server /usr/local/redis-cluster/7003/redis.conf
#第三个节点
# cat start_redis_cluster.sh
#!/bin/bash
cd /usr/local/redis-cluster
cd 7004 && /usr/bin/redis-server /usr/local/redis-cluster/7004/redis.conf
cd ../7005 &&/usr/bin/redis-server /usr/local/redis-cluster/7005/redis.conf
#启动节点
# sh start_redis_cluster.sh
# sh start_redis_cluster.sh
# sh start_redis_cluster.sh
#配置开机启动
# cat /etc/rc.d/rc.local
#!/bin/bash
# THIS FILE IS ADDED FOR COMPATIBILITY PURPOSES
#
# It is highly advisable to create own systemd services or udev rules
# to run scripts during boot instead of using this file.
#
# In contrast to previous versions due to parallel execution during boot
# this script will NOT be run after all other services.
#
# Please note that you must run 'chmod +x /etc/rc.d/rc.local' to ensure
# that this script will be executed during boot.
touch /var/lock/subsys/local
echo never > /sys/kernel/mm/transparent_hugepage/enabled
/usr/bin/sh /usr/local/redis-cluster/start_redis_cluster.sh
/usr/bin/sh /usr/local/redis-cluster/start_exporter.sh
[*]创建集群redis-cli --cluster create 172.16.83.248:7000 172.16.83.248:7001 172.16.83.249:7002 172.16.83.249:7003 172.16.83.250:7004 172.16.83.250:7005 --cluster-replicas 1
[*]查看集群状态# redis-cli -c -p 7000 cluster nodes
18a0169a22e7b440213728b1fb45ae5d59bf8eef 172.16.83.249:7002@17002 master - 0 1601184964498 41 connected 6251-10922
83280280d19223180eabffb1a88ff078efdc748d 172.16.83.249:7003@17003 slave 27c6bff6b4b20d5620a81879324cf61a8e2439a1 0 1601184965000 39 connected
3933efb9b77e474ab036d7741bc4517746483496 172.16.83.248:7001@17001 slave 16ab783181c05ed6c0e3db933be6a1f56efd3317 0 1601184964097 42 connected
16ab783181c05ed6c0e3db933be6a1f56efd3317 172.16.83.250:7004@17004 master - 0 1601184965501 42 connected 88-332 5461-5794 10923-11255 11711-16383
30469e7f2ce74d6e0ff224463468909aa268c6cf 172.16.83.250:7005@17005 slave 18a0169a22e7b440213728b1fb45ae5d59bf8eef 0 1601184965501 41 connected
27c6bff6b4b20d5620a81879324cf61a8e2439a1 172.16.83.248:7000@17000 myself,master - 0 1601184965000 39 connected 0-87 333-5460 5795-6250 11256-11710
# redis-cli--cluster check 127.0.0.1:7001
172.16.83.249:7002 (18a0169a...) -> 10 keys | 4672 slots | 1 slaves.
172.16.83.250:7004 (16ab7831...) -> 9 keys | 5585 slots | 1 slaves.
172.16.83.248:7000 (27c6bff6...) -> 5 keys | 6127 slots | 1 slaves.
24 keys in 3 masters.
0.00 keys per slot on average.
>>> Performing Cluster Check (using node 127.0.0.1:7001)
S: 3933efb9b77e474ab036d7741bc4517746483496 127.0.0.1:7001
slots: (0 slots) slave
replicates 16ab783181c05ed6c0e3db933be6a1f56efd3317
M: 18a0169a22e7b440213728b1fb45ae5d59bf8eef 172.16.83.249:7002
slots: (4672 slots) master
1 additional replica(s)
S: 30469e7f2ce74d6e0ff224463468909aa268c6cf 172.16.83.250:7005
slots: (0 slots) slave
replicates 18a0169a22e7b440213728b1fb45ae5d59bf8eef
S: 83280280d19223180eabffb1a88ff078efdc748d 172.16.83.249:7003
slots: (0 slots) slave
replicates 27c6bff6b4b20d5620a81879324cf61a8e2439a1
M: 16ab783181c05ed6c0e3db933be6a1f56efd3317 172.16.83.250:7004
slots:,,, (5585 slots) master
1 additional replica(s)
M: 27c6bff6b4b20d5620a81879324cf61a8e2439a1 172.16.83.248:7000
slots:,,, (6127 slots) master
1 additional replica(s)
All nodes agree about slots configuration.
>>> Check for open slots...
>>> Check slots coverage...
All 16384 slots covered.
扩容节点内存步骤
说明：线上Redis节点内存接近限制3g，进行无感知进行升级内存。
[*]查看主从节点分配；
[*]确认主从不在同一节点；
[*]逐台操作从节点，进行扩容服务器配置；
[*]动态升级内存限制；config set maxmemory 6g
[*]再进行从升级主操作在slave的Redis执行 CLUSTER FAILOVER
slave会接管主, master会转化为salve
说明：故障转移的主服务器的客户端都已停止。同时，主服务器将其复制偏移发送到从服务器，从服务器等待到达其一侧的偏移。达到复制偏移量后，故障转移将开始，并且将向旧的主服务器通知配置切换。当客户端在旧的主服务器上解除阻止时，它们将被重定向到新的主服务器；
[*]此时从变为主再进行升级主变为从节点服务器，升级配置；
[*]再动态升级内存限；config set maxmemory 6g
[*]此时升级配置扩容内存已完成，注意检查主从分配信息，确保主从不在同一点，保证服务的高可用。
prometheus监控Redis
[*]客户端脚本# cat start_exporter.sh
#!/bin/bash
cd /usr/local/redis-cluster
ports=`netstat -anltp|grep LISTEN|grep -v 1700|egrep "(700|6379)"|awk '{print $4}'|cut -d ":" -f 2`
echo "find ports: $ports"
for p in $ports
do
ip=`ip addr |grep 172|grep inet|awk '{print $2}'|awk -F "/" '{print $1}'`
echo "start exporter 127.0.0.1:${p} with ${ip}:5${p}"
nohup ./redis_exporter -web.listen-address ":5${p}" -redis.addr 127.0.0.1:${p} &
done
[*]prometheus服务端配置# cat redis-test-yc.yaml
---
apiVersion: v1
kind: Service
metadata:
labels:
node: redis-exporter
name: redis-test-yc
namespace: monitoring
spec:
ports:
- name: metrics
port: 8080
sessionAffinity: None
type: ClusterIP
status:
loadBalancer: {}
---
apiVersion: v1
kind: Endpoints
metadata:
name: redis-test-yc
namespace: monitoring
subsets:
- addresses:
- ip: 172.16.83.248
ports:
- name: metrics
port: 57000
protocol: TCP
- addresses:
- ip: 172.16.83.248
ports:
- name: metrics
port: 57001
protocol: TCP
- addresses:
- ip: 172.16.83.249
ports:
- name: metrics
port: 57002
protocol: TCP
- addresses:
- ip: 172.16.83.249
ports:
- name: metrics
port: 57003
protocol: TCP
- addresses:
- ip: 172.16.83.250
ports:
- name: metrics
port: 57004
protocol: TCP
- addresses:
- ip: 172.16.83.250
ports:
- name: metrics
port: 57005
protocol: TCP
[*]查看prometheus后台监控项
页:
[1]