使用kubekey离线部署kubesphere v3.4.0 + rook-ceph

服务器规划

IP地址规划

软件版本

端口要求

备注：当使用Calico网络插件并且使用经典网络运行集群时，需要对源地址启用IPENCAP和IPIP协议。

配置服务器环境

Harbor节点服务器配置

安装CentOS 7.9

• 使用光盘镜像文件：CentOS-7-x86_64-DVD-2009.iso
• 最小化安装操作系统，然后安装必要的rpm包。
• 可以不分配swap区
• 因为docker和containerd都将运行数据存放在/var目录下，建议将/var目录配置较大的存储空间（至少大于100GB）

升级内核版本

# 升级内核
yum install kernel-ml-devel-6.0.10-1.el7.elrepo.x86_64.rpm
yum install kernel-ml-6.0.10-1.el7.elrepo.x86_64.rpm

# 设置启动引导使用新内核
grub2-set-default '6.0.10-1.el7.elrepo.x86_64'

# 重启操作系统
reboot

安装必要的软件包

软件包清单：
yum install -y yum-utils device-mapper-persistent-data lvm2 sysstat nfs-utils ntp jq bind-utils telnet curl rsync sshpass wget vim socat conntrack ebtables ipset bash-completion
HA环境的两个软件包：
yum install -y haproxy keepalived
搭建内网yum源或者制作离线包，制作rpm离线包方法网上很多教程可以参考。
或者从这里下载这些软件：rpm离线包下载

上传提前准备好的离线rpm包’k8s-request-rpm.tgz’，并解压缩。

tar zxvf k8s-request-rpm.tgz

drwxr-xr-x monitor/monitor   0 2023-12-01 16:49 ./install_rpm/
-rw-r--r-- monitor/monitor 67624 2014-07-04 08:43 ./install_rpm/autogen-libopts-5.18-5.el7.x86_64.rpm
-rw-r--r-- monitor/monitor 88692 2020-06-24 01:36 ./install_rpm/ntpdate-4.2.6p5-29.el7.centos.2.x86_64.rpm
-rw-r--r-- monitor/monitor 1527972 2019-08-23 05:24 ./install_rpm/GeoIP-1.5.0-14.el7.x86_64.rpm
...
...
...
-rw-r--r-- monitor/monitor  417900 2022-12-20 00:02 ./install_rpm/rsync-3.1.2-12.el7_9.x86_64.rpm
-rw-r--r-- monitor/monitor   46304 2014-07-04 12:25 ./install_rpm/perl-Time-HiRes-1.9725-3.el7.x86_64.rpm
-rw-r--r-- monitor/monitor   19244 2014-07-04 12:15 ./install_rpm/perl-constant-1.27-2.el7.noarch.rpm

切换为root用户安装软件包：

# 切换到root用户
su -
密码：xxxx

# 进入解压好的目录下，安装软件包
cd install_rpm/
yum install *.rpm

已加载插件：fastestmirror
正在检查 autogen-libopts-5.18-5.el7.x86_64.rpm: autogen-libopts-5.18-5.el7.x86_64
正在检查 bind-libs-9.11.4-26.P2.el7_9.15.x86_64.rpm: 32:bind-libs-9.11.4-26.P2.el7_9.15.x86_64
...
...
...
 wget                            x86_64   1.14-18.el7_6.1                    /wget-1.14-18.el7_6.1.x86_64                             2.0 M
 yum-utils                       noarch   1.1.31-54.el7_8                    /yum-utils-1.1.31-54.el7_8.noarch                        337 k

事务概要
============================================================================================================================================
重新安装  73 软件包

总计：92 M
安装大小：92 M
Is this ok [y/d/N]: y

禁止swap区

编辑/etc/fstab注释掉swap那一行

sed -i 's/.*swap.*/#&/' /etc/fstab
swapoff -a

禁止防火墙和selinux

# 关闭禁止selinux
sed -i 's/^SELINUX=enforcing$/SELINUX=disabled/' /etc/selinux/config
setenforce 0

# 关闭禁止防火墙
systemctl disable firewalld && systemctl stop firewalld

配置时区和时间同步

# 设置时区为：Asia/Shanghai
timedatectl set-timezone Asia/Shanghai

# 设置时间同步，每30分钟同步一次
crontab -e
*/30 * * * * /usr/sbin/ntpdate 10.100.48.1 10.100.48.42 10.122.1.6 &> /dev/null

配置文件系统

# 查看硬盘分区情况
lsblk

NAME                MAJ:MIN RM   SIZE RO TYPE MOUNTPOINT
sdb                   8:16   0  10.7T  0 disk
sda                   8:0    0 837.9G  0 disk
├─sda2                8:2    0 837.4G  0 part
│ ├─vg_root-swap_lv 253:1    0    32G  0 lvm
│ ├─vg_root-tmp_lv  253:6    0    10G  0 lvm  /tmp
│ ├─vg_root-nmon_lv 253:4    0    10G  0 lvm  /nmon
│ ├─vg_root-home_lv 253:2    0    10G  0 lvm  /home
│ ├─vg_root-root_lv 253:0    0    50G  0 lvm  /
│ ├─vg_root-app_lv  253:5    0    50G  0 lvm  /app
│ └─vg_root-var_lv  253:3    0    20G  0 lvm  /var
└─sda1                8:1    0   500M  0 part /boot

# 给/dev/sdb进行分区
[root@k8s-harbor1 ~]# parted /dev/sdb
GNU Parted 3.1
使用 /dev/sdb
Welcome to GNU Parted! Type 'help' to view a list of commands.
(parted) mklabel GPT
警告: The existing disk label on /dev/sdb will be destroyed and all data on this disk will be lost. Do you want to continue?
是/Yes/否/No? y
(parted) mkpart
分区名称？  []? sdb1
文件系统类型？  [ext2]?
起始点？ 1
结束点？ 80000

(parted) mkpart
分区名称？  []? sdb2
文件系统类型？  [ext2]?
起始点？ 8000GB
结束点？ 11700GB
错误: The location 11700GB is outside of the device /dev/sdb.
(parted) mkpart
分区名称？  []? sdb2
文件系统类型？  [ext2]?
起始点？ 8000GB
结束点？ 11680GB

(parted) p
Model: AVAGO SAS3108 (scsi)
Disk /dev/sdb: 11.7TB
Sector size (logical/physical): 512B/512B
Partition Table: gpt
Disk Flags:

Number  Start   End     Size    File system  Name  标志
 1      1049kB  8000GB  8000GB               sdb1
 2      8000GB  11.7TB  3680GB               sdb2

(parted) q
信息: You may need to update /etc/fstab.

# 再次查看硬盘分区情况
lsblk

NAME                MAJ:MIN RM   SIZE RO TYPE MOUNTPOINT
sdb                   8:16   0  10.7T  0 disk
├─sdb2                8:18   0   3.4T  0 part
└─sdb1                8:17   0   7.3T  0 part
sda                   8:0    0 837.9G  0 disk
├─sda2                8:2    0 837.4G  0 part
│ ├─vg_root-swap_lv 253:1    0    32G  0 lvm
│ ├─vg_root-tmp_lv  253:6    0    10G  0 lvm  /tmp
│ ├─vg_root-nmon_lv 253:4    0    10G  0 lvm  /nmon
│ ├─vg_root-home_lv 253:2    0    10G  0 lvm  /home
│ ├─vg_root-root_lv 253:0    0    50G  0 lvm  /
│ ├─vg_root-app_lv  253:5    0    50G  0 lvm  /app
│ └─vg_root-var_lv  253:3    0    20G  0 lvm  /var
└─sda1                8:1    0   500M  0 part /boot

# 格式化磁盘
mkfs.xfs /dev/sdb1
meta-data=/dev/sdb1              isize=512    agcount=32, agsize=61035200 blks
         =                       sectsz=512   attr=2, projid32bit=1
         =                       crc=1        finobt=0, sparse=0
data     =                       bsize=4096   blocks=1953124864, imaxpct=5
         =                       sunit=64     swidth=64 blks
naming   =version 2              bsize=4096   ascii-ci=0 ftype=1
log      =internal log           bsize=4096   blocks=521728, version=2
         =                       sectsz=512   sunit=64 blks, lazy-count=1
realtime =none                   extsz=4096   blocks=0, rtextents=0

mkfs.xfs /dev/sdb2
meta-data=/dev/sdb2              isize=512    agcount=32, agsize=28076224 blks
         =                       sectsz=512   attr=2, projid32bit=1
         =                       crc=1        finobt=0, sparse=0
data     =                       bsize=4096   blocks=898437376, imaxpct=5
         =                       sunit=64     swidth=64 blks
naming   =version 2              bsize=4096   ascii-ci=0 ftype=1
log      =internal log           bsize=4096   blocks=438720, version=2
         =                       sectsz=512   sunit=64 blks, lazy-count=1
realtime =none                   extsz=4096   blocks=0, rtextents=0

# 创建挂在目录
mkdir /harbor-data
mkdir /k8s-data

# 编辑/etc/fstab
vim /etc/fstab

#
# /etc/fstab
# Created by anaconda on Thu Nov 30 16:25:28 2023
#
# Accessible filesystems, by reference, are maintained under '/dev/disk'
# See man pages fstab(5), findfs(8), mount(8) and/or blkid(8) for more info
#
/dev/mapper/vg_root-root_lv /                       xfs     defaults        0 0
/dev/mapper/vg_root-app_lv /app                    xfs     defaults        0 0
UUID=0da96d5d-ccb6-4f1a-a8ec-8b552fdf005f /boot                   xfs     defaults        0 0
/dev/mapper/vg_root-home_lv /home                   xfs     defaults        0 0
/dev/mapper/vg_root-nmon_lv /nmon                   xfs     defaults        0 0
/dev/mapper/vg_root-tmp_lv /tmp                    xfs     defaults        0 0
/dev/mapper/vg_root-var_lv /var                    xfs     defaults        0 0
#/dev/mapper/vg_root-swap_lv swap                    swap    defaults        0 0

/dev/sdb1 /harbor-data xfs defaults 0 0
/dev/sdb2 /k8s-data xfs defaults 0 0

#挂载磁盘
mount -a

df -h
文件系统                     容量  已用  可用 已用% 挂载点
devtmpfs                      63G     0   63G    0% /dev
tmpfs                         63G     0   63G    0% /dev/shm
tmpfs                         63G  9.4M   63G    1% /run
tmpfs                         63G     0   63G    0% /sys/fs/cgroup
/dev/mapper/vg_root-root_lv   50G  2.1G   48G    5% /
/dev/sda1                    494M  180M  314M   37% /boot
/dev/mapper/vg_root-nmon_lv   10G   33M   10G    1% /nmon
/dev/mapper/vg_root-tmp_lv    10G   33M   10G    1% /tmp
/dev/mapper/vg_root-app_lv    50G  758M   50G    2% /app
/dev/mapper/vg_root-var_lv    20G  118M   20G    1% /var
/dev/mapper/vg_root-home_lv   10G  2.7G  7.4G   27% /home
tmpfs                         13G     0   13G    0% /run/user/1000
/dev/sdb1                    7.3T   34M  7.3T    1% /harbor-data
/dev/sdb2                    3.4T   34M  3.4T    1% /k8s-data

编辑/etc/hosts文件

vim /etc/hosts

10.122.249.151 k8s-harbor1 harbor.k8s.hebei
10.122.249.152 k8s-harbor2 harhor2.k8s.hebei
10.122.249.153 k8s-ha-vip vip.k8s.hebei

10.122.249.154 k8s-master1
10.122.249.155 k8s-master2
10.122.249.156 k8s-master3

10.122.249.157 k8s-node01
10.122.249.158 k8s-node02
10.122.249.159 k8s-node03
10.122.249.160 k8s-node04
10.122.249.161 k8s-node05

配置dns

# 替换为你自己内网的DNS服务器地址
echo "DNS1=114.114.114.114" >> /etc/sysconfig/network-scripts/ifcfg-ens5f1
systemctl daemon-reload && systemctl restart network

Kubernetes节点配置

参考Harbor节点服务器操作系统配置。

安装部署Harbor服务器

安装Docker环境

docker使用24.0.6版本，上传docker安装rpm包。

# harbor目录规划说明：harbor默认安装目录在/app/harbor，harbor存储的数据目录和日志目录在/harbor-data

# 安装docker
yum install *.rpm

# 配置docker服务私有镜像库信任，限制镜像日志大小尺寸
vim /etc/docker/daemon.json

{
   
        "insecure-registries":["http://10.122.249.151","http://10.122.249.152"],
        "log-driver":"json-file",
        "log-opts":{
   "max-size":"50m","max-file":"3"}
}

# 启动docker服务
systemctl enable --now docker

# 查看docker版本信息
docker version
Client: Docker Engine - Community
 Version:           24.0.7
 API version:       1.43
 Go version:        go1.20.10
 Git commit:        afdd53b
 Built:             Thu Oct 26 09:11:35 2023
 OS/Arch:           linux/amd64
 Context:           default

Server: Docker Engine - Community
 Engine:
  Version:          24.0.6
  API version:      1.43 (minimum version 1.12)
  Go version:       go1.20.7
  Git commit:       1a79695
  Built:            Mon Sep  4 12:34:28 2023
  OS/Arch:          linux/amd64
  Experimental:     false
 containerd:
  Version:          1.6.25
  GitCommit:        d8f198a4ed8892c764191ef7b3b06d8a2eeb5c7f
 runc:
  Version:          1.1.10
  GitCommit:        v1.1.10-0-g18a0cb0
 docker-init:
  Version:          0.19.0
  GitCommit:        de40ad0

# 查看docker服务状态
docker info
Client: Docker Engine - Community
 Version:    24.0.7
 Context:    default
 Debug Mode: false
 Plugins:
  buildx: Docker Buildx (Docker Inc.)
    Version:  v0.11.2
    Path:     /usr/libexec/docker/cli-plugins/docker-buildx
  compose: Docker Compose (Docker Inc.)
    Version:  v2.21.0
    Path:     /usr/libexec/docker/cli-plugins/docker-compose

Server:
 Containers: 0
  Running: 0
  Paused: 0
  Stopped: 0
 Images: 0
 Server Version: 24.0.6
 Storage Driver: overlay2
  Backing Filesystem: xfs
  Supports d_type: true
  Using metacopy: false
  Native Overlay Diff: true
  userxattr: false
 Logging Driver: json-file
 Cgroup Driver: cgroupfs
 Cgroup Version: 1
 Plugins:
  Volume: local
  Network: bridge host ipvlan macvlan null overlay
  Log: awslogs fluentd gcplogs gelf journald json-file local logentries splunk syslog
 Swarm: inactive
 Runtimes: io.containerd.runc.v2 runc
 Default Runtime: runc
 Init Binary: docker-init
 containerd version: d8f198a4ed8892c764191ef7b3b06d8a2eeb5c7f
 runc version: v1.1.10-0-g18a0cb0
 init version: de40ad0
 Security Options:
  seccomp
   Profile: builtin
 Kernel Version: 6.0.10-1.el7.elrepo.x86_64
 Operating System: CentOS Linux 7 (Core)
 OSType: linux
 Architecture: x86_64
 CPUs: 32
 Total Memory: 125.8GiB
 Name: k8s-harbor1
 ID: 51a38654-d8af-4362-9531-d23a397b6a79
 Docker Root Dir: /var/lib/docker
 Debug Mode: false
 Experimental: false
 Insecure Registries:
  10.122.249.151
  127.0.0.0/8
 Live Restore Enabled: false

# 复制docker-compose到/usr/bin/目录下
cp docker-compose-linux-x86_64-v2.10.2 /usr/bin/docker-compose
chmod +x /usr/bin/docker-compose

安装Harbor

# 解压缩离线安装包
cd /app/harbor
tar zxvf harbor-offline-installer-v2.6.0.tgz

# 编辑harbor配置文件
cp harbor.yml.tmpl harbor.yml
vim harbor.yml

hostname: harbor.k8s.hebei

http:
  port: 80

harbor_admin_password: Harbor12345

database:
  password: root123
  max_idle_conns: 100
  max_open_conns: 900

data_volume: /harbor-data

trivy:
  ignore_unfixed: false
  skip_update: false
  offline_scan: false
  insecure: false

jobservice:
  max_job_workers: 10

notification:
  webhook_job_max_retry: 10

chart:
  absolute_url: disabled

log:
  level: info
  local:
    rotate_count: 50
    rotate_size: 200M
    location: /harbor-data/log/harbor

_version: 2.6.0

proxy:
  http_proxy:
  https_proxy:
  no_proxy:
  components:
    - core
    - jobservice
    - trivy

upload_purging:
  enabled: true
  age: 168h
  interval: 24h
  dryrun: false

cache:
  enabled: false
  expire_hours: 24

# 编辑docker-compose.yml文件，修改两处地方，分别是122行和188行，增加一行内容："- /etc/hosts:/etc/hosts:z"，否则主备复制无法解析主机和IP地址，无法建立仓库连接

109   core:
110     image: goharbor/harbor-core:v2.6.0
111     container_name: harbor-core
112     env_file:
113       - ./common/config/core/env
114     restart: always
115     cap_drop:
116       - ALL
117     cap_add:
118       - SETGID
119       - SETUID
120     volumes:
121       - /harbor-data/ca_download/:/etc/core/ca/:z
122       - /etc/hosts:/etc/hosts:z                    # 添加mount /etc/hosts
123       - /harbor-data/:/data/:z
124       - ./common/config/core/certificates/:/etc/core/certificates/:z
125       - type: bind
126         source: ./common/config/core/app.conf
127         target: /etc/core/app.conf
128       - type: bind
129         source: /harbor-data/secret/core/private_key.pem
130         target: /etc/core/private_key.pem
131       - type: bind
132         source: /harbor-data/secret/keys/secretkey
133         target: /etc/core/key
134       - type: bind
135         source: ./common/config/shared/trust-certificates
136         target: /harbor_cust_cert
137     networks:
138       harbor:
139     depends_on:
140       - log
141       - registry
142       - redis
143       - postgresql
144     logging:
145       driver: "syslog"
146       options:
147         syslog-address: "tcp://localhost:1514"
148         tag: "core"

173   jobservice:
174     image: goharbor/harbor-jobservice:v2.6.0
175     container_name: harbor-jobservice
176     env_file:
177       - ./common/config/jobservice/env
178     restart: always
179     cap_drop:
180       - ALL
181     cap_add:
182       - CHOWN
183       - SETGID
184       - SETUID
185     volumes:
186       - /harbor-data/job_logs:/var/log/jobs:z
187       - /harbor-data/scandata_exports:/var/scandata_exports:z
188       - /etc/hosts:/etc/hosts:z                     # 添加mount /etc/hosts
189       - type: bind
190         source: ./common/config/jobservice/config.yml
191         target: /etc/jobservice/config.yml
192       - type: bind
193         source: ./common/config/shared/trust-certificates
194         target: /harbor_cust_cert
195     networks:
196       - harbor
197     depends_on:
198       - core
199     logging:
200       driver: "syslog"
201       options:
202         syslog-address: "tcp://localhost:1514"
203         tag: "jobservice"


# 开始安装并启动harbor
./install.sh

[Step 0]: checking if docker is installed ...
Note: docker version: 24.0.7
[Step 1]: checking docker-compose is installed ...
Note: Docker Compose version v2.21.0

[Step 2]: loading Harbor images ...
915f79eed965: Loading layer [==================================================>]  37.77MB/37.77MB
53e17aa1994a: Loading layer [==================================================>]  8.898MB/8.898MB
82205c155ee7: Loading layer [==================================================>]  3.584kB/3.584kB
7ffa6a408e36: Loading layer [==================================================>]   2.56kB/2.56kB
1a2ed94f447f: Loading layer [==================================================>]  97.91MB/97.91MB
e031eb4548cd: Loading layer [==================================================>]   98.7MB/98.7MB
Loaded image: goharbor/harbor-jobservice:v2.6.0
1ddd239fd081: Loading layer [==================================================>]  5.755MB/5.755MB
51cfe17ad552: Loading layer [==================================================>]  4.096kB/4.096kB
d66b11611927: Loading layer [==================================================>]   17.1MB/17.1MB
95ec06f9ede8: Loading layer [==================================================>]  3.072kB/3.072kB
4915db4c8a75: Loading layer [==================================================>]  29.13MB/29.13MB
de0dd696d1e4: Loading layer [==================================================>]  47.03MB/47.03MB
Loaded image: goharbor/harbor-registryctl:v2.6.0
135ff4cdf210: Loading layer [==================================================>]  119.9MB/119.9MB
971eb518f877: Loading layer [==================================================>]  3.072kB/3.072kB
dca613dfbd94: Loading layer [==================================================>]   59.9kB/59.9kB
86701cd4bbd5: Loading layer [==================================================>]  61.95kB/61.95kB
Loaded image: goharbor/redis-photon:v2.6.0
db777e2b34a6: Loading layer [==================================================>]    119MB/119MB
Loaded image: goharbor/nginx-photon:v2.6.0
e8b623356728: Loading layer [==================================================>]  6.283MB/6.283MB
de97fd65d649: Loading layer [==================================================>]