kubernetes v1.18.6 mongodb集群部署
·
本实验为配置NFS存储类为mongodb提供持久化存储卷，创建3站点mongo集群（1主2从），通过nosqlbooster客户端管理工具连接验证。
1、配置NFS服务
在管理节点（10.0.0.1）上部署NFS服务，也可以用独立的服务器部署NFS服务端，共享的数据目录为/home/pvdata/share（777权限），在此之前需要在每个node上安装 nfs-utils与rpcbind软件（具体可参考https://blog.csdn.net/m0_37939350/article/details/108068192第一章节）
[root@k8s-node01 storageclass]# cat /etc/exports
/home/pvdata/share *(rw,sync,insecure,no_subtree_check,no_root_squash) #PV动态供应
[root@k8s-node01 nginx]#exportfs -r #nfs配置生效
[root@k8s-node01 nginx]#exportfs #查看NFS共享目录
2、创建nfs-provisioner
[root@k8s-node01 storageclass]# cat serviceaccount.yaml #配置nfs-provisioner 的SA账号
apiVersion: v1
kind: ServiceAccount
metadata:
name: nfs-provisioner
[root@k8s-node01 storageclass]# kubectl apply -f serviceaccount.yaml #执行该文件
[root@k8s-node01 storageclass]# cat service-rbac.yaml #对sa账号做rbac授权，RBAC(Role-Based Access control)基于角色的访问控制；
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: nfs-provisioner-runner
rules:
- apiGroups: [""]
resources: ["persistentvolumes"]
verbs: ["get", "list", "watch", "create", "delete"]
- apiGroups: [""]
resources: ["persistentvolumeclaims"]
verbs: ["get", "list", "watch", "update"]
- apiGroups: ["storage.k8s.io"]
resources: ["storageclasses"]
verbs: ["get", "list", "watch"]
- apiGroups: [""]
resources: ["events"]
verbs: ["create", "update", "patch"]
- apiGroups: [""]
resources: ["services", "endpoints"]
verbs: ["get"]
- apiGroups: ["extensions"]
resources: ["podsecuritypolicies"]
resourceNames: ["nfs-provisioner"]
verbs: ["use"]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: run-nfs-provisioner
subjects:
- kind: ServiceAccount
name: nfs-provisioner
namespace: default
roleRef:
kind: ClusterRole
name: nfs-provisioner-runner
apiGroup: rbac.authorization.k8s.io
---
kind: Role
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: leader-locking-nfs-provisioner
rules:
- apiGroups: [""]
resources: ["endpoints"]
verbs: ["get", "list", "watch", "create", "update", "patch"]
---
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: leader-locking-nfs-provisioner
subjects:
- kind: ServiceAccount
name: nfs-provisioner
namespace: default
roleRef:
kind: Role
name: leader-locking-nfs-provisioner
apiGroup: rbac.authorization.k8s.io
[root@k8s-node01 storageclass]# kubectl apply -f service-rbac.yaml #执行该文件
[root@k8s-node01 storageclass]# cat nfs-provisioner-deploy.yaml #创建pod用来运行nfs-provisioner
kind: Deployment
apiVersion: apps/v1
metadata:
name: nfs-provisioner
spec:
selector:
matchLabels:
app: nfs-provisioner
replicas: 1
strategy:
type: Recreate
template:
metadata:
labels:
app: nfs-provisioner
spec:
serviceAccount: nfs-provisioner
containers:
- name: nfs-provisioner
image: registry.cn-hangzhou.aliyuncs.com/open-ali/nfs-client-provisioner:latest
volumeMounts:
- name: nfs-client-root
mountPath: /persistentvolumes
env:
- name: PROVISIONER_NAME
value: jim.com/nfs #可自定义
- name: NFS_SERVER
value: 10.0.0.1 #对应NFS服务器IP地址
- name: NFS_PATH
value: /home/pvdata/share #对应NFS配置好的共享目录
volumes:
- name: nfs-client-root
nfs:
server: 10.0.0.1 #对应NFS服务器IP地址
path: /home/pvdata/share #对应NFS配置好的共享目录
[root@k8s-node01 storageclass]# kubectl apply -f nfs-provisioner-deploy.yaml #执行该文件
[root@k8s-node01 storageclass]# kubectl get pod -l app=nfs-provisioner #检查生成的pod;
NAME READY STATUS RESTARTS AGE
nfs-provisioner-6bbc9fcd47-79m8z 1/1 Running 0 80m
3、配置storageclass存储类
[root@k8s-node01 storageclass]# cat storageclass.yaml
kind: StorageClass
apiVersion: storage.k8s.io/v1
metadata:
name: nfs-storageclass #命名，可自定义，后面PVC需要引用
provisioner: jim.com/nfs #对应环境变量PROVISIONER_NAME的值
[root@k8s-node01 storageclass]# kubectl apply -f storageclass.yaml #执行该文件
[root@k8s-node01 storageclass]# kubectl get storageclasses #检查该存储类是否创建成功
NAME PROVISIONER RECLAIMPOLICY VOLUMEBINDINGMODE ALLOWVOLUMEEXPANSION AGE
nfs-storageclass jim.com/nfs Delete Immediate false 87m
到此NFS StorageClass存储类服务创建成功!!！
4、部署mongoDB资源
[root@k8s-node01 mongodb]# cat mongo_sta_nodeport.yaml
apiVersion: v1
kind: Service
metadata:
name: mongo
labels:
app: mongo
spec:
ports:
- name: mongo
port: 27017
targetPort: 27017
clusterIP: None
selector:
app: mongo
---
apiVersion: v1
kind: Service
metadata:
name: mongo-service #配置外网访问，用于mongo客户端管理工具的管理；
labels:
app: mongo
spec:
ports:
- name: mongo-http
port: 27017
selector:
app: mongo
type: NodePort
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
metadata:
name: mongo-default-view #配置rbac高级访问权限
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: view
subjects:
- kind: ServiceAccount
name: default
namespace: default
---
apiVersion: apps/v1
kind: StatefulSet #通过statefulset部署mongo有状态应用；
metadata:
name: mongo
spec:
selector:
matchLabels:
app: mongo
serviceName: "mongo"
replicas: 3 #3个副本；
template:
metadata:
labels:
app: mongo
role: mongo
environment: test
spec:
terminationGracePeriodSeconds: 10
containers:
- name: mongo
image: mongo:3.4
command:
- mongod
- "--replSet"
- rs0
- "--bind_ip"
- 0.0.0.0
- "--smallfiles"
- "--noprealloc"
ports:
- containerPort: 27017
volumeMounts:
- name: mongo-persistent-storage
mountPath: /data/db
- name: mongo-sidecar #作为MongoDB集群的管理者，将使用此Headless Service来维护各个MongoDB实例之间的集群关系，以及集群规模变化时的自动更新。
image: cvallance/mongo-k8s-sidecar
env:
- name: MONGO_SIDECAR_POD_LABELS
value: "role=mongo,environment=test"
volumeClaimTemplates:
- metadata:
name: mongo-persistent-storage
annotations:
volume.beta.kubernetes.io/storage-class: "nfs-storageclass" #指定上面创建的NFS存储类，用于持久化存储mongoDB数据库文件；
spec:
accessModes: [ "ReadWriteOnce" ]
resources:
requests:
storage: 10Gi
[root@k8s-node01 mongodb]# kubectl apply -f mongo_sta_nodeport.yaml #运行以上yaml文件，创建mongo资源；
[root@k8s-node01 share]# kubectl get pod -l app=mongo -o wide #查看创建好的mongo pod信息；
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
mongo-0 2/2 Running 0 2m40s 10.122.58.240 k8s-node02 <none> <none>
mongo-1 2/2 Running 0 2m20s 10.122.135.145 k8s-node03 <none> <none>
mongo-2 2/2 Running 0 2m 10.122.58.241 k8s-node02 <none> <none>
[root@k8s-node01 share]# kubectl get svc -l app=mongo -o wide #查看mongo的服务资源，记住nodeport自动分配的端口号（32131），后面客户端管理软件连接时需要用到；
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE SELECTOR
mongo ClusterIP None <none> 27017/TCP 3m33s app=mongo
mongo-service NodePort 10.10.233.130 <none> 27017:32131/TCP 3m33s app=mongo
5、验证mongoDB集群
[root@k8s-node01 share]# curl http://172.25.83.130:32131 #验证http访问，172.25.83.130为我的node外网IP地址，可以改成你自己的，32131为nodeport自动生成的端口号；
It looks like you are trying to access MongoDB over HTTP on the native driver port. #集群访问正常；
6、通过mongo shell管理数据库；
[root@k8s-node01 share]# kubectl exec -it mongo-0 -- mongo #进入mongo-0 pod,且mongo命令连接mongoDB数据库
Defaulting container name to mongo.
Use 'kubectl describe pod/mongo-0 -n default' to see all of the containers in this pod.
MongoDB shell version v3.4.24
connecting to: mongodb://127.0.0.1:27017
MongoDB server version: 3.4.24
Welcome to the MongoDB shell.
For interactive help, type "help".
For more comprehensive documentation, see
http://docs.mongodb.org/
Questions? Try the support group
http://groups.google.com/group/mongodb-user
Server has startup warnings:
2020-08-21T21:04:57.887+0000 I CONTROL [initandlisten]
2020-08-21T21:04:57.887+0000 I CONTROL [initandlisten] ** WARNING: Access control is not enabled for the database.
2020-08-21T21:04:57.887+0000 I CONTROL [initandlisten] ** Read and write access to data and configuration is unrestricted.
2020-08-21T21:04:57.887+0000 I CONTROL [initandlisten] ** WARNING: You are running this process as the root user, which is not recommended.
2020-08-21T21:04:57.887+0000 I CONTROL [initandlisten]
2020-08-21T21:04:57.887+0000 I CONTROL [initandlisten]
2020-08-21T21:04:57.887+0000 I CONTROL [initandlisten] ** WARNING: /sys/kernel/mm/transparent_hugepage/enabled is 'always'.
2020-08-21T21:04:57.887+0000 I CONTROL [initandlisten] ** We suggest setting it to 'never'
2020-08-21T21:04:57.887+0000 I CONTROL [initandlisten]
2020-08-21T21:04:57.887+0000 I CONTROL [initandlisten] ** WARNING: /sys/kernel/mm/transparent_hugepage/defrag is 'always'.
2020-08-21T21:04:57.887+0000 I CONTROL [initandlisten] ** We suggest setting it to 'never'
2020-08-21T21:04:57.887+0000 I CONTROL [initandlisten]
rs0:PRIMARY> rs.status() #执行rs.status()命令，查看mongo集群状态，下面信息显示1个PRIMARY节点，2个SECONDARY，集群正常
{
"set" : "rs0",
"date" : ISODate("2020-08-21T21:13:03.617Z"),
"myState" : 1,
"term" : NumberLong(1),
"syncingTo" : "",
"syncSourceHost" : "",
"syncSourceId" : -1,
"heartbeatIntervalMillis" : NumberLong(2000),
"optimes" : {
"lastCommittedOpTime" : {
"ts" : Timestamp(1598044377, 1),
"t" : NumberLong(1)
},
"appliedOpTime" : {
"ts" : Timestamp(1598044377, 1),
"t" : NumberLong(1)
},
"durableOpTime" : {
"ts" : Timestamp(1598044377, 1),
"t" : NumberLong(1)
}
},
"members" : [
{
"_id" : 0,
"name" : "10.122.58.240:27017",
"health" : 1,
"state" : 1,
"stateStr" : "PRIMARY",
"uptime" : 487,
"optime" : {
"ts" : Timestamp(1598044377, 1),
"t" : NumberLong(1)
},
"optimeDate" : ISODate("2020-08-21T21:12:57Z"),
"syncingTo" : "",
"syncSourceHost" : "",
"syncSourceId" : -1,
"infoMessage" : "",
"electionTime" : Timestamp(1598043915, 2),
"electionDate" : ISODate("2020-08-21T21:05:15Z"),
"configVersion" : 4,
"self" : true,
"lastHeartbeatMessage" : ""
},
{
"_id" : 1,
"name" : "10.122.135.145:27017",
"health" : 1,
"state" : 2,
"stateStr" : "SECONDARY",
"uptime" : 446,
"optime" : {
"ts" : Timestamp(1598044377, 1),
"t" : NumberLong(1)
},
"optimeDurable" : {
"ts" : Timestamp(1598044377, 1),
"t" : NumberLong(1)
},
"optimeDate" : ISODate("2020-08-21T21:12:57Z"),
"optimeDurableDate" : ISODate("2020-08-21T21:12:57Z"),
"lastHeartbeat" : ISODate("2020-08-21T21:13:01.901Z"),
"lastHeartbeatRecv" : ISODate("2020-08-21T21:13:02.128Z"),
"pingMs" : NumberLong(1),
"lastHeartbeatMessage" : "",
"syncingTo" : "10.122.58.241:27017",
"syncSourceHost" : "10.122.58.241:27017",
"syncSourceId" : 2,
"infoMessage" : "",
"configVersion" : 4
},
{
"_id" : 2,
"name" : "10.122.58.241:27017",
"health" : 1,
"state" : 2,
"stateStr" : "SECONDARY",
"uptime" : 416,
"optime" : {
"ts" : Timestamp(1598044377, 1),
"t" : NumberLong(1)
},
"optimeDurable" : {
"ts" : Timestamp(1598044377, 1),
"t" : NumberLong(1)
},
"optimeDate" : ISODate("2020-08-21T21:12:57Z"),
"optimeDurableDate" : ISODate("2020-08-21T21:12:57Z"),
"lastHeartbeat" : ISODate("2020-08-21T21:13:03.449Z"),
"lastHeartbeatRecv" : ISODate("2020-08-21T21:13:03.013Z"),
"pingMs" : NumberLong(0),
"lastHeartbeatMessage" : "",
"syncingTo" : "10.122.58.240:27017",
"syncSourceHost" : "10.122.58.240:27017",
"syncSourceId" : 0,
"infoMessage" : "",
"configVersion" : 4
}
],
"ok" : 1
}
rs0:PRIMARY> show dbs; #查数据库，初始配置自动生成了admin与local数据库；
admin 0.000GB
local 0.000GB
rs0:PRIMARY> use test; #创建test数据库；
switched to db test
rs0:PRIMARY> db.users.insertMany( [{ name: "bob", age: 42, status: "A", },{ name: "ahn", age: 22, status: "A", },{ name: "xi", age: 34, status: "D", }]) #为该数据库插入users集合信息
{
"acknowledged" : true,
"insertedIds" : [
ObjectId("5f403b2082cacca4bad16aa8"),
ObjectId("5f403b2082cacca4bad16aa9"),
ObjectId("5f403b2082cacca4bad16aaa")
]
}
rs0:PRIMARY>
rs0:PRIMARY> show collections; #查询集合
users
rs0:PRIMARY> db.users.find() #查询users集合中的数据；
{ "_id" : ObjectId("5f403b2082cacca4bad16aa8"), "name" : "bob", "age" : 42, "status" : "A" }
{ "_id" : ObjectId("5f403b2082cacca4bad16aa9"), "name" : "ahn", "age" : 22, "status" : "A" }
{ "_id" : ObjectId("5f403b2082cacca4bad16aaa"), "name" : "xi", "age" : 34, "status" : "D" }
rs0:PRIMARY>
7、通过nosqlbooster客户端管理工具来管理mongodb;
7.1、因nosqlbooster使用管理员与密码连接，所以为初始安装完成的mongodb添加管理员：
rs0:PRIMARY> use admin #切换到admin数据库
switched to db admin
rs0:PRIMARY> db.createUser({user:"admin",pwd:"password",roles:["root"]}) #创建用户名为admin、密码为password、角色为root的用户；
Successfully added user: { "user" : "admin", "roles" : [ "root" ] }
rs0:PRIMARY> db.getCollectionNames() #新增的用户在system.users；
[ "system.users", "system.version" ]
rs0:PRIMARY> db.auth("admin", "password") #第一个用户添加完成后，便需要认证才能继续添加其他用户；
1
添加数据库用户
为其他数据库添加用户，添加用户前需要切换到该数据库
这里设置其角色为dbOwner
rs0:PRIMARY> use test
switched to db test
rs0:PRIMARY> db.createUser({user: "jim", pwd: "123456", roles: [{ role: "dbOwner", db: "test" }]})
Successfully added user: {
"user" : "jim",
"roles" : [
{
"role" : "dbOwner",
"db" : "test"
}
]
}
查看用户信息：
rs0:PRIMARY> use admin #切换到admin数据库
switched to db admin
rs0:PRIMARY> db.system.users.find() #查询已经创建好的用户信息，如下：
{ "_id" : "admin.admin", "userId" : BinData(4,"DlJ68++lTaq41YaD7O9oYA=="), "user" : "admin", "db" : "admin", "credentials" : { "SCRAM-SHA-1" : { "iterationCount" : 10000, "salt" : "csWUlIt49/gEx7qfuq+xhg==", "storedKey" : "Iycih9aNjGnfpNZEcnQAibAy/5o=", "serverKey" : "GmWQDCunFqJ2gE7bgwIM4mzYp/M=" } }, "roles" : [ { "role" : "root", "db" : "admin" } ] }
{ "_id" : "test.jim", "userId" : BinData(4,"rOgMxHOFQD+KApu7CW8Ozw=="), "user" : "jim", "db" : "test", "credentials" : { "SCRAM-SHA-1" : { "iterationCount" : 10000, "salt" : "M/NWvhyy5X7lhMdwakXjDw==", "storedKey" : "ylPeK5SIQjtrM6kuwFrlQjau9Y0=", "serverKey" : "Zm9L8U1byi5+KE+dS2RLfcfgR7o=" } }, "roles" : [ { "role" : "dbOwner", "db" : "test" } ] }
7.2、nosqlbooster连接mongo数据库配置
7.3、进入nosqlbooster图形化界面，验证已经创建的信息；
8、对Mongo集群进行扩容
[root@k8s-node01 mongodb]# kubectl scale statefulset mongo --replicas=4 #扩展mongo集群到4个节点
statefulset.apps/mongo scaled
[root@k8s-node01 mongodb]# kubectl get pod -l app=mongo -o wide #查询到新生成的pod;
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
mongo-0 2/2 Running 0 62m 10.122.58.240 k8s-node02 <none> <none>
mongo-1 2/2 Running 0 62m 10.122.135.145 k8s-node03 <none> <none>
mongo-2 2/2 Running 0 61m 10.122.58.241 k8s-node02 <none> <none>
mongo-3 2/2 Running 0 3m46s 10.122.135.147 k8s-node03 <none> <none>
刷新nosqlbooster数据库连接，获得新节点信息；
END.
北京人形旗下天工造物具身智能开源社区，聚焦具身天工与慧思开物两大平台
更多推荐
0
0- 0
已为社区贡献1条内容
所有评论(0)