kubernetes二進位部署 1、環境規劃 軟體 版本 Linux操作系統 CentOS Linux release 7.6.1810 (Core) Kubernetes 1.9 Docker 18.09.3 etcd 3.3.10 角色 IP 組件 推薦配置 k8s_master etcd01 ...
kubernetes二進位部署
1、環境規劃
軟體 |
版本 |
Linux操作系統 |
CentOS Linux release 7.6.1810 (Core) |
Kubernetes |
1.9 |
Docker |
18.09.3 |
etcd |
3.3.10 |
角色 |
IP |
組件 |
推薦配置 |
k8s_master etcd01 |
192.168.1.153 |
kube-apiserver kube-controller-manager kube-scheduler etcd |
CPU 2核+ 2G記憶體+ |
k8s_node01 etcd02 |
192.168.1.154 |
kubelet kube-proxy docker flannel etcd |
|
k8s_node02 etcd03 |
192.168.1.155 |
kubelet kube-proxy docker flannel etcd |
|
2、 單Master集群架構
3、 系統常規參數配置
3.1 關閉selinux
sed -i 's/SELINUX=enforcing/SELINUX=disabled/' /etc/selinux/config
setenforce 0
3.2 文件數調整
sed -i '/* soft nproc 4096/d' /etc/security/limits.d/20-nproc.conf
echo '* - nofile 65536' >> /etc/security/limits.conf
echo '* soft nofile 65535' >> /etc/security/limits.conf
echo '* hard nofile 65535' >> /etc/security/limits.conf
echo 'fs.file-max = 65536' >> /etc/sysctl.conf
3.3 防火牆關閉
systemctl disable firewalld.service
systemctl stop firewalld.service
3.4 常用工具安裝及時間同步
yum -y install vim telnet iotop openssh-clients openssh-server ntp net-tools.x86_64 wget
ntpdate time.windows.com
3.5 hosts文件配置(3個節點)
vim /etc/hosts
192.168.1.153 k8s_master
192.168.1.154 k8s_node01
192.168.1.155 k8s_node02
3.6 伺服器之間免密鑰登錄
ssh-keygen
ssh-copy-id -i ~/.ssh/id_rsa.pub [email protected]
ssh-copy-id -i ~/.ssh/id_rsa.pub [email protected]
4、 自簽ssl證書
4.1 etcd生成證書
cfssl.sh |
etcd-cert.sh |
etcd.sh |
4.1.1 安裝cfssl工具(cfssl.sh)
cd /home/k8s_install/ssl_etcd
chmod +x cfssl.sh
./cfssl.sh
內容如下:
curl -L https://pkg.cfssl.org/R1.2/cfssl_linux-amd64 -o /usr/local/bin/cfssl
curl -L https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64 -o /usr/local/bin/cfssljson
curl -L https://pkg.cfssl.org/R1.2/cfssl-certinfo_linux-amd64 -o /usr/local/bin/cfssl-certinfo
chmod +x /usr/local/bin/cfssl /usr/local/bin/cfssljson /usr/local/bin/cfssl-certinfo
4.1.2 生成etcd 自簽ca證書(etcd-cert.sh)
chmod +x etcd-cert.sh
./etcd-cert.sh
內容如下:
cat > ca-config.json <<EOF
{
"signing": {
"default": {
"expiry": "87600h"
},
"profiles": {
"www": {
"expiry": "87600h",
"usages": [
"signing",
"key encipherment",
"server auth",
"client auth"
]
}
}
}
}
EOF
cat > ca-csr.json <<EOF
{
"CN": "etcd CA",
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"L": "Beijing",
"ST": "Beijing"
}
]
}
EOF
cfssl gencert -initca ca-csr.json | cfssljson -bare ca –
#-----------------------
cat > server-csr.json <<EOF
{
"CN": "etcd",
"hosts": [
"192.168.1.153",
"192.168.1.154",
"192.168.1.155",
"192.168.1.156",
"192.168.1.157"
],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"L": "BeiJing",
"ST": "BeiJing"
}
]
}
EOF
cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=www server-csr.json | cfssljson -bare server
註意:hosts一定要包含所有節點,可以多部署幾個預留節點以便後續擴容,否則還需要重新生成
4.1.3 etcd二進位包安裝
#存放配置文件,可執行文件,證書文件
mkdir /opt/etcd/{cfg,bin,ssl} -p
#ssl 證書切記複製到/opt/etcd/ssl/
cp {ca,server-key,server}.pem /opt/etcd/ssl/
#部署etcd以及增加etcd服務(etcd.sh)
cd /home/k8s_install/soft/
tar -zxvf etcd-v3.3.10-linux-amd64.tar.gz
cd etcd-v3.3.10-linux-amd64
mv etcd etcdctl /opt/etcd/bin/
cd /home/k8s_install/ssl_etcd
chmod +x etcd.sh
參數說明:1.etcd名稱 2.本機ip 3.其他兩個etcd名稱以及地址
./etcd.sh etcd01 192.168.1.153 etcd02=https://192.168.1.154:2380,etcd03=https://192.168.1.155:2380
執行後會卡住實際是在等待其他兩個節點加入
其他兩個node節點部署etcd:
scp -r /opt/etcd/ k8s_node01:/opt/
scp -r /opt/etcd/ k8s_node02:/opt/
scp /usr/lib/systemd/system/etcd.service k8s_node01:/usr/lib/systemd/system/
scp /usr/lib/systemd/system/etcd.service k8s_node02:/usr/lib/systemd/system/
#修改node節點配置文件(2個節點都需要更改)
ssh k8s_node01
vim /opt/etcd/cfg/etcd
ETCD_NAME以及ip地址
systemctl daemon-reload
systemctl start etcd.service
etcd.sh腳本內容如下:
#!/bin/bash
ETCD_NAME=$1
ETCD_IP=$2
ETCD_CLUSTER=$3
WORK_DIR=/opt/etcd
cat <<EOF >$WORK_DIR/cfg/etcd
#[Member]
ETCD_NAME="${ETCD_NAME}"
ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
ETCD_LISTEN_PEER_URLS="https://${ETCD_IP}:2380"
ETCD_LISTEN_CLIENT_URLS="https://${ETCD_IP}:2379"
#[Clustering]
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://${ETCD_IP}:2380"
ETCD_ADVERTISE_CLIENT_URLS="https://${ETCD_IP}:2379"
ETCD_INITIAL_CLUSTER="etcd01=https://${ETCD_IP}:2380,${ETCD_CLUSTER}"
ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"
ETCD_INITIAL_CLUSTER_STATE="new"
EOF
cat <<EOF >/usr/lib/systemd/system/etcd.service
[Unit]
Description=Etcd Server
After=network.target
After=network-online.target
Wants=network-online.target
[Service]
Type=notify
EnvironmentFile=${WORK_DIR}/cfg/etcd
ExecStart=${WORK_DIR}/bin/etcd \
--name=\${ETCD_NAME} \
--data-dir=\${ETCD_DATA_DIR} \
--listen-peer-urls=\${ETCD_LISTEN_PEER_URLS} \
--listen-client-urls=\${ETCD_LISTEN_CLIENT_URLS},http://127.0.0.1:2379 \
--advertise-client-urls=\${ETCD_ADVERTISE_CLIENT_URLS} \
--initial-advertise-peer-urls=\${ETCD_INITIAL_ADVERTISE_PEER_URLS} \
--initial-cluster=\${ETCD_INITIAL_CLUSTER} \
--initial-cluster-token=\${ETCD_INITIAL_CLUSTER_TOKEN} \
--initial-cluster-state=new \
--cert-file=${WORK_DIR}/ssl/server.pem \
--key-file=${WORK_DIR}/ssl/server-key.pem \
--peer-cert-file=${WORK_DIR}/ssl/server.pem \
--peer-key-file=${WORK_DIR}/ssl/server-key.pem \
--trusted-ca-file=${WORK_DIR}/ssl/ca.pem \
--peer-trusted-ca-file=${WORK_DIR}/ssl/ca.pem
Restart=on-failure
LimitNOFILE=65536
[Install]
WantedBy=multi-user.target
EOF
systemctl daemon-reload
systemctl enable etcd
systemctl restart etcd
4.1.4 查看etcd集群健康情況
cd /opt/etcd/ssl
/opt/etcd/bin/etcdctl --ca-file=ca.pem --cert-file=server.pem --key-file=server-key.pem --endpoints="https://192.168.1.153:2379,https://192.168.1.154:2379,https://192.168.1.155:2379" cluster-health
5、 安裝Docker(node 節點)
5.1 安裝依賴包
yum install -y yum-utils \ device-mapper-persistent-data \ lvm2
5.2 配置官方源(替換為阿裡源)
yum-config-manager --add-repo http://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo
5.3 更新並安裝Docker-CE
yum makecache fast
yum install docker-ce -y
5.4 配置docker加速器
curl -sSL https://get.daocloud.io/daotools/set_mirror.sh | sh -s http://f1361db2.m.daocloud.io
5.5 啟動docker
systemctl restart docker.service
systemctl enable docker.service
6、部署Flannel網路
Overlay Network:覆蓋網路,在基礎網路上疊加的一種虛擬網路技術模式,該網路中的主機通過虛擬鏈路連接起來。 VXLAN:將源數據包封裝到UDP中,並使用基礎網路的IP/MAC作為外層報文頭進行封裝,然後在乙太網上傳輸,到達目的地後由隧道端點解封裝並將數據發送給目標地址。 Flannel:是Overlay網路的一種,也是將源數據包封裝在另一種網路包裡面進行路由轉發和通信,目前已經支持UDP、VXLAN、AWS VPC和GCE路由等數據轉發方式。 多主機容器網路通信其他主流方案:隧道方案( Weave、OpenvSwitch ),路由方案(Calico)等。
6.1 寫入分配的子網段到etcd,供flanneld使用(master)
/opt/etcd/bin/etcdctl --ca-file=/opt/etcd/ssl/ca.pem --cert-file=/opt/etcd/ssl/server.pem --key-file=/opt/etcd/ssl/server-key.pem --endpoints=https://192.168.1.153:2379,https://192.168.1.154:2379,https://192.168.1.155:2379 set /coreos.com/network/config '{ "Network": "172.17.0.0/16", "Backend": {"Type": "vxlan"}}'
6.2 二進位包安裝Flannel(node節點 flannel.sh)
下載地址:https://github.com/coreos/flannel/releases/download/
#
mkdir /opt/kubernetes/{bin,cfg,ssl} -p
cd /home/k8s_install/flannel_install/
tar -zxvf flannel-v0.10.0-linux-amd64.tar.gz
mv {flanneld,mk-docker-opts.sh} /opt/kubernetes/bin/
cd /home/k8s_install/flannel_install
chmod +x flannel.sh
chmod +x /opt/kubernetes/bin/{flanneld,mk-docker-opts.sh}
./flannel.sh https://192.168.1.153:2379,https://192.168.1.154:2379,https://192.168.1.155:2379
腳本內容如下:
#!/bin/bash
ETCD_ENDPOINTS=${1:-"http://127.0.0.1:2379"}
cat <<EOF >/opt/kubernetes/cfg/flanneld
FLANNEL_OPTIONS="--etcd-endpoints=${ETCD_ENDPOINTS} \
-etcd-cafile=/opt/etcd/ssl/ca.pem \
-etcd-certfile=/opt/etcd/ssl/server.pem \
-etcd-keyfile=/opt/etcd/ssl/server-key.pem"
EOF
cat <<EOF >/usr/lib/systemd/system/flanneld.service
[Unit]
Description=Flanneld overlay address etcd agent
After=network-online.target network.target
Before=docker.service
[Service]
Type=notify
EnvironmentFile=/opt/kubernetes/cfg/flanneld
ExecStart=/opt/kubernetes/bin/flanneld --ip-masq \$FLANNEL_OPTIONS
ExecStartPost=/opt/kubernetes/bin/mk-docker-opts.sh -k DOCKER_NETWORK_OPTIONS -d /run/flannel/subnet.env
Restart=on-failure
[Install]
WantedBy=multi-user.target
EOF
cat <<EOF >/usr/lib/systemd/system/docker.service
[Unit]
Description=Docker Application Container Engine
Documentation=https://docs.docker.com
After=network-online.target firewalld.service
Wants=network-online.target
[Service]
Type=notify
EnvironmentFile=/run/flannel/subnet.env
ExecStart=/usr/bin/dockerd \$DOCKER_NETWORK_OPTIONS
ExecReload=/bin/kill -s HUP \$MAINPID
LimitNOFILE=infinity
LimitNPROC=infinity
LimitCORE=infinity
TimeoutStartSec=0
Delegate=yes
KillMode=process &nb