linuxea: 基于kubernetes的etcd 3.3.10外部集群


etcd是一个分布式键值存储,它提供了一种在一组机器上存储数据的可靠方法。它是开源的,可在GitHub上获得。etcd在网络分区期间优雅地处理leader选举,并且可以容忍机器故障,包括leader。

应用程序可以将数据读写到etcd中。一个简单的用例是将etcd中的数据库连接详细信息或功能标记存储为键值对。可以监视这些值,允许您的应用在更改时重新配置。

高级用法利用一致性保证来实现数据库leader选举或跨工作集群进行分布式锁定
etcd是用Go编写的,它具有出色的跨平台支持,较小二进制文件和活跃的社区。etcd机器之间的通信通过Raft一致性算法处理。

此处的ETCD主要用来部署kubernetes高可用集群,此后的使用都是基于kubernetes。

参考kubernetes官网:https://kubernetes.io/docs/tasks/administer-cluster/configure-upgrade-etcd/

etcd配置:https://github.com/etcd-io/etcd/blob/master/Documentation/op-guide/hardware.md#hardware-recommendations
集群文档:https://github.com/etcd-io/etcd/blob/master/Documentation/op-guide/clustering.md
示例参考:https://github.com/etcd-io/etcd/tree/master/hack/tls-setup

参考:https://github.com/etcd-io/etcd/tree/master/hack/tls-setup/config
参考:https://k8smeetup.github.io/docs/setup/independent/high-availability/
我们首先配置etcd证书,etcd我们将会用在kubernetes上。这一步是必须的
相比较github上etcd的示例,我们简单修改下

在这之前 ,我们有必要修改一下主机名,并且配通所有的ssh

hostnamectl set-hostname etcd1
hostnamectl set-hostname etcd2
hostnamectl set-hostname etcd3
cat >>  /etc/hosts << EOF
172.25.50.16 etcd1
172.25.50.17 etcd2
172.25.50.18 etcd3
EOF
[root@linuxea.com-16 /etc/etcda]# ssh-keygen -t rsa
[root@linuxea.com-16 /etc/etcda]# for i in 172.25.50.{17,18};do ssh-copy-id $i; done

I. 安装cfssl和cfssljson

[root@linuxea.com-16 ~]#  curl -so /usr/local/bin/cfssl https://pkg.cfssl.org/R1.2/cfssl_linux-amd64
[root@linuxea.com-16 ~]#  curl -o /usr/local/bin/cfssljson https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64
[root@linuxea.com-16 ~]#  chmod +x /usr/local/bin/cfssl*

II. 生成证书

[root@linuxea.com-16 ~]# mkdir -p /etc/kubernetes/pki/etcd
[root@linuxea.com-16 ~]# cd /etc/kubernetes/pki/etcd
  • ca-config.json
cat > cat /etc/kubernetes/pki/etcd/ca-config.json  << EOF
{
     "signing": {
         "default": {
             "expiry": "876000h"
         },
         "profiles": {
             "server": {
                 "expiry": "876000h",
                 "usages": [
                     "signing",
                     "key encipherment",
                     "server auth",
                     "client auth"
                 ]
             },
             "client": {
                 "expiry": "876000h",
                 "usages": [
                     "signing",
                     "key encipherment",
                     "client auth"
                 ]
             },
             "peer": {
                 "expiry": "876000h",
                 "usages": [
                     "signing",
                     "key encipherment",
                     "server auth",
                     "client auth"
                 ]
             }
         }
     }
 }
EOF
  • ca-csr.json
cat > ca-csr.json << EOL
{
  "CN": "etcd",
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "Shanghai",
      "L": "Shanghai",
      "O": "etcd",
      "OU": "Etcd Security"
    }
  ]
}
EOL

生成CA证书

[root@linuxea.com-16 /etc/kubernetes/pki/etcd]# cfssl gencert -initca ca-csr.json | cfssljson -bare ca -
2018/12/25 16:29:14 [INFO] generating a new CA key and certificate from CSR
2018/12/25 16:29:14 [INFO] generate received request
2018/12/25 16:29:14 [INFO] received CSR
2018/12/25 16:29:14 [INFO] generating key: rsa-2048
2018/12/25 16:29:14 [INFO] encoded CSR
2018/12/25 16:29:14 [INFO] signed certificate with serial number 472142876620060394898834048533122419461412171471
[root@linuxea.com-16 /etc/kubernetes/pki/etcd]# ll
total 20
-rw-r--r-- 1 root root  905 Dec 25 16:28 ca-config.json
-rw-r--r-- 1 root root 1005 Dec 25 16:29 ca.csr
-rw-r--r-- 1 root root  212 Dec 25 16:29 ca-csr.json
-rw------- 1 root root 1679 Dec 25 16:29 ca-key.pem
-rw-r--r-- 1 root root 1371 Dec 25 16:29 ca.pem

生成 etcd 客户端证书

[root@linuxea.com-16 /etc/kubernetes/pki/etcd]# cat client.json 
 {
     "CN": "client",
     "key": {
         "algo": "ecdsa",
         "size": 256
     }
 }
[root@linuxea.com-16 /etc/kubernetes/pki/etcd]# cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=client client.json | cfssljson -bare client
2018/12/25 16:29:56 [INFO] generate received request
2018/12/25 16:29:56 [INFO] received CSR
2018/12/25 16:29:56 [INFO] generating key: ecdsa-256
2018/12/25 16:29:56 [INFO] encoded CSR
2018/12/25 16:29:56 [INFO] signed certificate with serial number 644510971695673396838569226835778482472560755733
2018/12/25 16:29:56 [WARNING] This certificate lacks a "hosts" field. This makes it unsuitable for
websites. For more information see the Baseline Requirements for the Issuance and Management
of Publicly-Trusted Certificates, v.1.1.6, from the CA/Browser Forum (https://cabforum.org);
specifically, section 10.2.3 ("Information Requirements").

如下

[root@linuxea.com-16 /etc/kubernetes/pki/etcd]# ll
total 36
-rw-r--r-- 1 root root  905 Dec 25 16:28 ca-config.json
-rw-r--r-- 1 root root 1005 Dec 25 16:29 ca.csr
-rw-r--r-- 1 root root  212 Dec 25 16:29 ca-csr.json
-rw------- 1 root root 1679 Dec 25 16:29 ca-key.pem
-rw-r--r-- 1 root root 1371 Dec 25 16:29 ca.pem
-rw-r--r-- 1 root root  351 Dec 25 16:29 client.csr
-rw-r--r-- 1 root root   95 Dec 25 16:29 client.json
-rw------- 1 root root  227 Dec 25 16:29 client-key.pem
-rw-r--r-- 1 root root  997 Dec 25 16:29 client.pem

config.json

对于config.json有两种方式,第一种,使用官网的,如下

[root@linuxea.com-16 /etc/kubernetes/pki/etcd]# cfssl print-defaults csr > config.json
[root@linuxea.com-16 /etc/kubernetes/pki/etcd]# sed -i '0,/CN/{s/example\.net/'"$PEER_NAME"'/}' config.json
[root@linuxea.com-16 /etc/kubernetes/pki/etcd]# sed -i 's/www\.example\.net/'"$PRIVATE_IP"'/' config.json
[root@linuxea.com-16 /etc/kubernetes/pki/etcd]# sed -i 's/example\.net/'"$PUBLIC_IP"'/' config.json
[root@linuxea.com-16 /etc/kubernetes/pki/etcd]# cat config.json
{
    "CN": "etcd1",
    "hosts": [
        "",
        "172.25.50.16"
    ],
    "key": {
        "algo": "ecdsa",
        "size": 256
    },
    "names": [
        {
            "C": "US",
            "L": "CA",
            "ST": "San Francisco"
        }
    ]
}

第二种方式,直接在这里编辑, 填写参与集群的ip

cat > /etc/kubernetes/pki/etcd/config.json << EOF
{
    "CN": "etcd1",
    "hosts": [
      "127.0.0.1",
      "172.25.50.16",
      "172.25.50.17",
      "172.25.50.18"
    ],
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
            "C": "CN",
            "ST": "Shanghai",
            "L": "Shanghai",
            "O": "etcd",
            "OU": "Etcd Security"
        }
    ]
}
EOF

运行 cfssl 命令,将会生成peer.pem、peer-key.pem、server.pem、server-key.pem。

[root@linuxea.com-16 /etc/kubernetes/pki/etcd]# cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=server config.json | cfssljson -bare server
2018/12/25 16:37:53 [INFO] generate received request
2018/12/25 16:37:53 [INFO] received CSR
2018/12/25 16:37:53 [INFO] generating key: rsa-2048
2018/12/25 16:37:54 [INFO] encoded CSR
2018/12/25 16:37:54 [INFO] signed certificate with serial number 397776469717117599117003178668354588092528739871
2018/12/25 16:37:54 [WARNING] This certificate lacks a "hosts" field. This makes it unsuitable for
websites. For more information see the Baseline Requirements for the Issuance and Management
of Publicly-Trusted Certificates, v.1.1.6, from the CA/Browser Forum (https://cabforum.org);
specifically, section 10.2.3 ("Information Requirements").
[root@linuxea.com-16 /etc/kubernetes/pki/etcd]#  cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=peer config.json | cfssljson -bare peer
2018/12/25 16:37:59 [INFO] generate received request
2018/12/25 16:37:59 [INFO] received CSR
2018/12/25 16:37:59 [INFO] generating key: rsa-2048
2018/12/25 16:37:59 [INFO] encoded CSR
2018/12/25 16:37:59 [INFO] signed certificate with serial number 453856739993256449551996181659627954567417235192
2018/12/25 16:37:59 [WARNING] This certificate lacks a "hosts" field. This makes it unsuitable for
websites. For more information see the Baseline Requirements for the Issuance and Management
of Publicly-Trusted Certificates, v.1.1.6, from the CA/Browser Forum (https://cabforum.org);
specifically, section 10.2.3 ("Information Requirements").

如下

[root@linuxea.com-16 /etc/kubernetes/pki/etcd]# ll
total 64
-rw-r--r-- 1 root root  905 Dec 25 16:28 ca-config.json
-rw-r--r-- 1 root root 1005 Dec 25 16:29 ca.csr
-rw-r--r-- 1 root root  212 Dec 25 16:29 ca-csr.json
-rw------- 1 root root 1679 Dec 25 16:29 ca-key.pem
-rw-r--r-- 1 root root 1371 Dec 25 16:29 ca.pem
-rw-r--r-- 1 root root  351 Dec 25 16:29 client.csr
-rw-r--r-- 1 root root   95 Dec 25 16:29 client.json
-rw------- 1 root root  227 Dec 25 16:29 client-key.pem
-rw-r--r-- 1 root root  997 Dec 25 16:29 client.pem
-rw-r--r-- 1 root root  375 Dec 25 16:37 config.json
-rw-r--r-- 1 root root 1078 Dec 25 16:37 peer.csr
-rw------- 1 root root 1679 Dec 25 16:37 peer-key.pem
-rw-r--r-- 1 root root 1456 Dec 25 16:37 peer.pem
-rw-r--r-- 1 root root 1078 Dec 25 16:37 server.csr
-rw------- 1 root root 1679 Dec 25 16:37 server-key.pem
-rw-r--r-- 1 root root 1456 Dec 25 16:37 server.pem

证书传递

将这些生成的证书复制到etcd2和etcd3上

[root@linuxea.com-16 /etc/kubernetes/pki/etcd]# for i in 172.25.50.{17,18} ;do scp -r /etc/kubernetes $i:/etc ;done
ca-config.json                   100%  905   635.1KB/s   00:00    
ca-csr.json                      100%  212   174.5KB/s   00:00    
ca.pem                           100% 1371     1.0MB/s   00:00    
ca-key.pem                       100% 1679     1.3MB/s   00:00    
ca.csr                           100% 1005   773.7KB/s   00:00    
client.json                      100%   95    76.1KB/s   00:00    
client.pem                       100%  997   751.7KB/s   00:00    
client-key.pem                   100%  227   171.9KB/s   00:00    
client.csr                       100%  351   256.1KB/s   00:00    
config.json                      100%  375    96.5KB/s   00:00    
server.pem                       100% 1456   365.4KB/s   00:00    
server-key.pem                   100% 1679   425.8KB/s   00:00    
server.csr                       100% 1078   276.8KB/s   00:00    
peer.pem                         100% 1456   366.5KB/s   00:00    
peer-key.pem                     100% 1679   439.1KB/s   00:00    
peer.csr                         100% 1078   289.2KB/s   00:00    
ca-config.json                   100%  905   569.4KB/s   00:00    
ca-csr.json                      100%  212   134.9KB/s   00:00    
ca.pem                           100% 1371   944.5KB/s   00:00    
ca-key.pem                       100% 1679     1.1MB/s   00:00    
ca.csr                           100% 1005   605.7KB/s   00:00    
client.json                      100%   95    63.3KB/s   00:00    
client.pem                       100%  997   748.6KB/s   00:00    
client-key.pem                   100%  227   151.1KB/s   00:00    
client.csr                       100%  351   244.9KB/s   00:00    
config.json                      100%  375    90.4KB/s   00:00    
server.pem                       100% 1456   322.2KB/s   00:00    
server-key.pem                   100% 1679   372.1KB/s   00:00    
server.csr                       100% 1078   253.3KB/s   00:00    
peer.pem                         100% 1456   325.0KB/s   00:00    
peer-key.pem                     100% 1679   394.1KB/s   00:00    
peer.csr                         100% 1078   259.3KB/s   00:00  

III. 安装etcd

这里的环境变量在三台参与集群的机器分别运行

版本3.3.10

export ETCD_VERSION=v3.3.10
curl -sSL https://github.com/coreos/etcd/releases/download/${ETCD_VERSION}/etcd-${ETCD_VERSION}-linux-amd64.tar.gz | tar -xzv --strip-components=1 -C /usr/local/bin/
rm -rf etcd-$ETCD_VERSION-linux-amd64*

先决变量

主机名和ip地址,这里的eth0应该与机器的网卡相符

export PEER_NAME=$(hostname)
export PRIVATE_IP=$(ip addr show eth0 | grep -Po 'inet \K[\d.]+')

环境变量写入到/etc/etcd.env

touch /etc/etcd.env
echo "PEER_NAME=$PEER_NAME" >> /etc/etcd.env
echo "PRIVATE_IP=$PRIVATE_IP" >> /etc/etcd.env

启动脚本即将用到的变量,这里标记的是参与etcd集群的三个ip地址

export etcd0_ip_address=172.25.50.16
export etcd1_ip_address=172.25.50.17
export etcd2_ip_address=172.25.50.18

启动脚本

这里面的变量就是上面设置的

cat > /etc/systemd/system/etcd.service << EOL
[Unit]
Description=etcd
Documentation=https://github.com/coreos/etcd
Conflicts=etcd.service
Conflicts=etcd2.service

[Service]
EnvironmentFile=/etc/etcd.env
Type=notify
Restart=always
RestartSec=5s
LimitNOFILE=40000
TimeoutStartSec=0

ExecStart=/usr/local/bin/etcd --name ${PEER_NAME} \
    --data-dir /var/lib/etcd \
    --listen-client-urls https://${PRIVATE_IP}:2379 \
    --advertise-client-urls https://${PRIVATE_IP}:2379 \
    --listen-peer-urls https://${PRIVATE_IP}:2380 \
    --initial-advertise-peer-urls https://${PRIVATE_IP}:2380 \
    --cert-file=/etc/kubernetes/pki/etcd/server.pem \
    --key-file=/etc/kubernetes/pki/etcd/server-key.pem \
    --client-cert-auth \
    --trusted-ca-file=/etc/kubernetes/pki/etcd/ca.pem \
    --peer-cert-file=/etc/kubernetes/pki/etcd/peer.pem \
    --peer-key-file=/etc/kubernetes/pki/etcd/peer-key.pem \
    --peer-client-cert-auth \
    --peer-trusted-ca-file=/etc/kubernetes/pki/etcd/ca.pem \
    --initial-cluster etcd1=https://${etcd0_ip_address}:2380,etcd2=https://${etcd1_ip_address}:2380,etcd3=https://${etcd2_ip_address}:2380 \
    --initial-cluster-state new

[Install]
WantedBy=multi-user.target

EOL
systemctl daemon-reload
systemctl enable etcd.service
systemctl start etcd

IV. 集群状态

查看集群状态是需要证书的,我们配置一个环境变量

CMD='--cacert=/etc/kubernetes/pki/etcd/ca.pem --cert=/etc/kubernetes/pki/etcd/server.pem --key=/etc/kubernetes/pki/etcd/server-key.pem'
CMD1='https://172.25.50.16:2379,https://172.25.50.17:2379,https://172.25.50.18:2379'
CMD2='--ca-file=/etc/kubernetes/pki/etcd/ca.pem --cert-file=/etc/kubernetes/pki/etcd/server.pem --key-file=/etc/kubernetes/pki/etcd/server-key.pem'
[root@linuxea.com-16 /etc/kubernetes/pki/etcd]# for i in 172.25.50.{16,17,18}; do ETCDCTL_API=3 etcdctl --endpoints=https://${i}:2379 $CMD endpoint health; done
https://172.25.50.16:2379 is healthy: successfully committed proposal: took = 1.984026ms
https://172.25.50.17:2379 is healthy: successfully committed proposal: took = 3.357136ms
https://172.25.50.18:2379 is healthy: successfully committed proposal: took = 3.55185ms
[root@linuxea.com-16 /etc/kubernetes/pki/etcd]# ETCDCTL_API=3 etcdctl --endpoints=https://172.25.50.16:2379 $CMD member list
2e70a124f01a4a5, started, etcd3, https://172.25.50.18:2380, https://172.25.50.18:2379
5fba4c5d1e214899, started, etcd2, https://172.25.50.17:2380, https://172.25.50.17:2379
b55bca6849256d2d, started, etcd1, https://172.25.50.16:2380, https://172.25.50.16:2379
[root@linuxea.com-16 /etc/kubernetes/pki/etcd]# 
[root@linuxea.com-16 /etc/kubernetes/pki/etcd]# etcdctl -C $CMD1 $CMD2 cluster-health
member 2e70a124f01a4a5 is healthy: got healthy result from https://172.25.50.18:2379
member 5fba4c5d1e214899 is healthy: got healthy result from https://172.25.50.17:2379
member b55bca6849256d2d is healthy: got healthy result from https://172.25.50.16:2379
cluster is healthy
[root@linuxea.com-16 /etc/kubernetes/pki/etcd]# curl -Lk --cert ./server.pem --key ./server-key.pem  https://172.25.50.16:2379/metrics|grep -v debugging

延伸阅读:https://coreos.com/etcd/docs/latest/metrics.html

docker安装

如果安装此前的配置,docker的配置应该如下

  • 172.25.50.16
docker run --net=host -d -v /etc/kubernetes/pki/etcd/:/etc/kubernetes/pki/etcd/ -p 4001:4001 -p 2380:2380 -p 2379:2379 \
 --name etcd quay.io/coreos/etcd:v3.3.10 \
 etcd -name etcd1 \
 --data-dir /var/lib/etcd \ 
 -advertise-client-urls https://172.25.50.16:2379,https://172.25.50.16:4001 \
 -listen-client-urls https://0.0.0.0:2379,https://0.0.0.0:4001 \
 -initial-advertise-peer-urls https://172.25.50.16:2380 \
 -listen-peer-urls https://0.0.0.0:2380 \
 --cert-file=/etc/kubernetes/pki/etcd/server.pem \
 --key-file=/etc/kubernetes/pki/etcd/server-key.pem \
 --client-cert-auth \
 --trusted-ca-file=/etc/kubernetes/pki/etcd/ca.pem \
 --peer-cert-file=/etc/kubernetes/pki/etcd/peer.pem \
 --peer-key-file=/etc/kubernetes/pki/etcd/peer-key.pem \
 --peer-client-cert-auth \
 --peer-trusted-ca-file=/etc/kubernetes/pki/etcd/ca.pem \
 -initial-cluster-token etcd-cluster \
 -initial-cluster etcd1=https://172.25.50.16:2380,etcd2=https://172.25.50.17:2380,etcd3=https://172.25.50.18:2380 \
 -initial-cluster-state new
  • 172.25.50.17
docker run --net=host -d -v /etc/kubernetes/pki/etcd/:/etc/kubernetes/pki/etcd/ -p 4001:4001 -p 2380:2380 -p 2379:2379 \
 --name etcd quay.io/coreos/etcd:v3.3.10 \
 etcd -name etcd2 \
 --data-dir /var/lib/etcd \
 -advertise-client-urls https://172.25.50.17:2379,https://172.25.50.17:4001 \
 -listen-client-urls https://0.0.0.0:2379,https://0.0.0.0:4001 \
 -initial-advertise-peer-urls https://172.25.50.17:2380 \
 -listen-peer-urls https://0.0.0.0:2380 \
 --cert-file=/etc/kubernetes/pki/etcd/server.pem \
 --key-file=/etc/kubernetes/pki/etcd/server-key.pem \
 --client-cert-auth \
 --trusted-ca-file=/etc/kubernetes/pki/etcd/ca.pem \
 --peer-cert-file=/etc/kubernetes/pki/etcd/peer.pem \
 --peer-key-file=/etc/kubernetes/pki/etcd/peer-key.pem \
 --peer-client-cert-auth \
 --peer-trusted-ca-file=/etc/kubernetes/pki/etcd/ca.pem \
 -initial-cluster-token etcd-cluster \
 -initial-cluster etcd1=https://172.25.50.16:2380,etcd2=https://172.25.50.17:2380,etcd3=https://172.25.50.18:2380 \
 -initial-cluster-state new 
  • 172.25.50.18
docker run --net=host -d -v /etc/kubernetes/pki/etcd/:/etc/kubernetes/pki/etcd/ -p 4001:4001 -p 2380:2380 -p 2379:2379 \
 --name etcd quay.io/coreos/etcd:v3.3.10 \
 etcd -name etcd3 \
 --data-dir /var/lib/etcd \
 -advertise-client-urls https://172.25.50.18:2379,https://172.25.50.18:4001 \
 -listen-client-urls https://0.0.0.0:2379,https://0.0.0.0:4001 \
 -initial-advertise-peer-urls https://172.25.50.18:2380 \
 -listen-peer-urls https://0.0.0.0:2380 \
 --cert-file=/etc/kubernetes/pki/etcd/server.pem \
 --key-file=/etc/kubernetes/pki/etcd/server-key.pem \
 --client-cert-auth \
 --trusted-ca-file=/etc/kubernetes/pki/etcd/ca.pem \
 --peer-cert-file=/etc/kubernetes/pki/etcd/peer.pem \
 --peer-key-file=/etc/kubernetes/pki/etcd/peer-key.pem \
 --peer-client-cert-auth \
 --peer-trusted-ca-file=/etc/kubernetes/pki/etcd/ca.pem \
 -initial-cluster-token etcd-cluster \
 -initial-cluster etcd1=https://172.25.50.16:2380,etcd2=https://172.25.50.17:2380,etcd3=https://172.25.50.18:2380 \
 -initial-cluster-state new  

这样比较麻烦,我们简化一下

  • 先决条件:

配置各个主机的hostname

[root@DT_Node-172_25_50_16 /etc/kubernetes/pki/etcd]# hostnamectl set-hostname etcd1
[root@DT_Node-172_25_50_17 /etc/kubernetes/pki/etcd]# hostnamectl set-hostname etcd2
[root@DT_Node-172_25_50_18 /etc/kubernetes/pki/etcd]# hostnamectl set-hostname etcd3
  • 环境变量
export PEER_NAME=$(hostname)
export PRIVATE_IP=$(ip addr show eth0 | grep -Po 'inet \K[\d.]+')
export etcd0_ip_address=172.25.50.16
export etcd1_ip_address=172.25.50.17
export etcd2_ip_address=172.25.50.18

现在docker的启动命令就如下所示了:

docker run --net=host -d -v /etc/kubernetes/pki/etcd/:/etc/kubernetes/pki/etcd/ -p 4001:4001 -p 2380:2380 -p 2379:2379 \
 -v /data/etcd:/data/etcd \
 --name etcd quay.io/coreos/etcd:v3.3.10 \
 etcd -name ${PEER_NAME} \
 --data-dir /data/etcd \
 -advertise-client-urls https://${PRIVATE_IP}:2379,https://${PRIVATE_IP}:4001 \
 -listen-client-urls https://0.0.0.0:2379,https://0.0.0.0:4001 \
 -initial-advertise-peer-urls https://${PRIVATE_IP}:2380 \
 -listen-peer-urls https://0.0.0.0:2380 \
 --cert-file=/etc/kubernetes/pki/etcd/server.pem \
 --key-file=/etc/kubernetes/pki/etcd/server-key.pem \
 --client-cert-auth \
 --trusted-ca-file=/etc/kubernetes/pki/etcd/ca.pem \
 --peer-cert-file=/etc/kubernetes/pki/etcd/peer.pem \
 --peer-key-file=/etc/kubernetes/pki/etcd/peer-key.pem \
 --peer-client-cert-auth \
 --peer-trusted-ca-file=/etc/kubernetes/pki/etcd/ca.pem \
 -initial-cluster-token etcd-cluster \
 -initial-cluster etcd1=https://${etcd0_ip_address}:2380,etcd2=https://${etcd1_ip_address}:2380,etcd3=https://${etcd2_ip_address}:2380 \
 -initial-cluster-state new

但是这样还是不太方便,我们写成docker-compose即可

V. docker-compose

version: '2.2'
services:
  etcd:
    image: marksugar/coreos-etcd:v3.3.10
    container_name: etcd3
    restart: always
    privileged: true
    network_mode: "host"
    volumes:
      - /data/etcd:/data/etcd
      - /etc/kubernetes/pki/etcd/:/etc/kubernetes/pki/etcd/
    command: "etcd -name ${PEER_NAME} --data-dir /data/etcd -advertise-client-urls https://${PRIVATE_IP}:2379,https://${PRIVATE_IP}:4001 -listen-client-urls https://0.0.0.0:2379,https://0.0.0.0:4001 -initial-advertise-peer-urls https://${PRIVATE_IP}:2380 -listen-peer-urls https://0.0.0.0:2380 --cert-file=/etc/kubernetes/pki/etcd/server.pem --key-file=/etc/kubernetes/pki/etcd/server-key.pem --client-cert-auth --trusted-ca-file=/etc/kubernetes/pki/etcd/ca.pem --peer-cert-file=/etc/kubernetes/pki/etcd/peer.pem --peer-key-file=/etc/kubernetes/pki/etcd/peer-key.pem --peer-client-cert-auth --peer-trusted-ca-file=/etc/kubernetes/pki/etcd/ca.pem -initial-cluster-token etcd-cluster -initial-cluster etcd1=https://${etcd0_ip_address}:2380,etcd2=https://${etcd1_ip_address}:2380,etcd3=https://${etcd2_ip_address}:2380 -initial-cluster-state new "
    cpu_shares: 90
    mem_limit: 2048m
    logging:
      driver: "json-file"
      options:
        max-size: "200M"
    labels:
      SERVICE_TAGS: etcd
  • 必须:-v /data/etcd:/data/etcd

如果丢失数据-dir ==永远丢失成员。

附上三个快速重置的脚本:
脚本运行后会自动删除集群内的数据存储目录,而后重启当前节点的etcd

#!/bin/bash
#########################################################################
# File Name: start.sh
# Author: www.linuxea.com
# Email: usertzc@163.com
# Version:
# Created Time: Wed 02 Jan 2019 11:14:26 AM CST
#########################################################################
for i in 172.25.50.{16,17,18};do ssh $i "docker rm -f etcd && \rm -rf /data/etcd && ls /data";done
hostnamectl set-hostname etcd1
export PEER_NAME=$(hostname)
export PRIVATE_IP=$(ip addr show eth0 | grep -Po 'inet \K[\d.]+')
export etcd0_ip_address=172.25.50.16
export etcd1_ip_address=172.25.50.17
export etcd2_ip_address=172.25.50.18
hostname
docker-compose -f /opt/docker-compose.yaml up -d

#CMD='--cacert=/etc/kubernetes/pki/etcd/ca.pem --cert=/etc/kubernetes/pki/etcd/server.pem --key=/etc/kubernetes/pki/etcd/server-key.pem'
#CMD1='https://172.25.50.16:2379,https://172.25.50.17:2379,https://172.25.50.18:2379'
#CMD2='--ca-file=/etc/kubernetes/pki/etcd/ca.pem --cert-file=/etc/kubernetes/pki/etcd/server.pem --key-file=/etc/kubernetes/pki/etcd/server-key.pem'
#for i in 172.25.50.{16,17,18}; do ETCDCTL_API=3 etcdctl --endpoints=https://${i}:2379 $CMD endpoint health; done
#cd /etc/kubernetes/pki/etcd/ && scp -P22992 ca.pem client.pem client-key.pem 172.25.50.13:/etc/kubernetes/pki/etcd/

脚本2中仅仅设置了环境变量和启动的docker-compose.yaml

#!/bin/bash
#########################################################################
# File Name: start.sh
# Author: www.linuxea.com
# Email: usertzc@163.com
# Version:
# Created Time: Wed 02 Jan 2019 11:15:02 AM CST
#########################################################################
#docker rm -f etcd && \rm -rf /data/etcd
hostnamectl set-hostname etcd2
export PEER_NAME=$(hostname)
export PRIVATE_IP=$(ip addr show eth0 | grep -Po 'inet \K[\d.]+')
export etcd0_ip_address=172.25.50.16
export etcd1_ip_address=172.25.50.17
export etcd2_ip_address=172.25.50.18
hostname
docker-compose -f /opt/docker-compose.yaml up -d

脚本3和2几乎 一样,除了名称外

#!/bin/bash
#########################################################################
# File Name: start.sh
# Author: www.linuxea.com
# Email: usertzc@163.com
# Version:
# Created Time: Wed 02 Jan 2019 11:15:26 AM CST
#########################################################################
#docker rm -f etcd && \rm -rf /data/etcd

hostnamectl set-hostname etcd3
export PEER_NAME=$(hostname)
export PRIVATE_IP=$(ip addr show eth0 | grep -Po 'inet \K[\d.]+')
export etcd0_ip_address=172.25.50.16
export etcd1_ip_address=172.25.50.17
export etcd2_ip_address=172.25.50.18
hostname
docker-compose -f /opt/docker-compose.yaml up -d

延伸阅读:https://coreos.com/etcd/docs/latest/v2/docker_guide.html
https://github.com/etcd-io/etcd/blob/master/Documentation/op-guide/container.md

VI. 监控

https://coreos.com/etcd/docs/latest/metrics.html https://etcd.readthedocs.io/en/latest/operate.html#v3-3

如果要在kubernetes中监控外部etcd,可参考:https://github.com/marksugar/k8s-pgmon

0 分享

您可以选择一种方式赞助本站

支付宝扫码赞助

支付宝扫码赞助

日期: 2019-01-24分类: kubernetes

标签: kubernetes, etcd

发表评论