当前位置： 首页 > news >正文

k8s4部署

news 来源：原创 2025/6/9 9:30:28

configMap

configmap概述：数据会存储在etcd数据库，其应用场景主要在应用程序的配置
configmap支持的类型（1）键值对（2）多行数据
pod使用configmap资源有两种常见的方式（1）变量注入（2）数据卷挂载
推荐阅读
https://kubernetes.io/docs/concepts/storage/volumes/#configmap	
https://kubernetes.io/docs/concepts/configuration/configmap/声明式创建cm资源
[root@master231 configmaps]# cat 01-cm-demo.yaml
apiVersion: v1
kind: ConfigMap
metadata:name: game-demo
# 指定cm资源的数据
data:# 类属性键；每一个键都映射到一个简单的值，对应的键值对。player_initial_lives: "3"ui_properties_file_name: "user-interface.properties"school: oldboyeduclass: linux94# 类文件键，对应的是多行数据，注意缩进格式game.properties: |      # |表示换行，一行写不下enemy.types=aliens,monstersplayer.maximum-lives=5    user-interface.properties: |color.good=purplecolor.bad=yellowallow.textmode=true  my.cnf: |[mysqld]datadir=/var/lib/mysqlbasedir=/usr/local/mysqlsocket=/tpm/mysql.sockskip-name-resolve=1port=3306[client]username=adminpassword=oldboyedu 创建
root@ubuntu0:~/manifests/configmap# kubectl apply -f 01-cm-demo.yaml 
configmap/game-demo created查看
root@ubuntu0:~/manifests/configmap# kubectl get cm 
NAME               DATA   AGE
game-demo          7      29s
kube-root-ca.crt   1      23d
root@ubuntu0:~/manifests/configmap# kubectl get cm game-demo 
NAME        DATA   AGE
game-demo   7      33s   #7代表有七个键值对删除
root@ubuntu0:~/manifests/configmap# kubectl delete -f 01-cm-demo.yaml 
configmap "game-demo" deleted响应式创建
root@ubuntu0:~/manifests/configmap# kubectl create configmap xp --from-literal=school=oldboyedu --from-literal=class=linux94 
configmap/xp created
root@ubuntu0:~/manifests/configmap# kubectl get cm 
NAME               DATA   AGE
kube-root-ca.crt   1      23d
xp                 2      6s
root@ubuntu0:~/manifests/configmap# kubectl get cm xp
NAME   DATA   AGE
xp     2      8sroot@ubuntu0:~/manifests/configmap# kubectl describe cm xp 
Name:         xp
Namespace:    default
Labels:       <none>
Annotations:  <none>Data
====
class:
----
linux94
school:
----
oldboyeduBinaryData
====Events:  <none>基于配置文件创建cm[root@master231 configmaps]# ll /root/kube-flannel.yml 
-rw-r--r-- 1 root root 4406 Nov 15 17:40 /root/kube-flannel.yml
[root@master231 configmaps]# 
[root@master231 configmaps]# kubectl create configmap oldboyedu-cni --from-file=cni.yml=/root/kube-flannel.yml
configmap/oldboyedu-cni created3.3 查看cm资源 
[root@master231 configmaps]# kubectl get cm oldboyedu-cni 
NAME            DATA   AGE
oldboyedu-cni   1      8s
[root@master231 configmaps]# 
[root@master231 configmaps]# kubectl describe cm oldboyedu-cni 
[root@master231 configmaps]# 
[root@master231 configmaps]# kubectl get cm oldboyedu-cni -o yaml
[root@master231 configmaps]# 
[root@master231 configmaps]# kubectl get cm oldboyedu-cni -o json3.3 删除cm资源 
[root@master231 configmaps]# kubectl get cm
NAME                DATA   AGE
kube-root-ca.crt    1      4d16h
oldboyedu-cni       1      2m27s
oldboyedu-linux94   2      4m38s
[root@master231 configmaps]# 
[root@master231 configmaps]# kubectl delete cm oldboyedu-cni 
configmap "oldboyedu-cni" deleted
[root@master231 configmaps]# 
[root@master231 configmaps]# kubectl get cm
NAME                DATA   AGE
kube-root-ca.crt    1      4d16h
oldboyedu-linux94   2      5m9s
[root@master231 configmaps]# root@ubuntu0:~/manifests/configmap# kubectl get cm xp -o yaml
apiVersion: v1
data:class: linux94school: oldboyedu
kind: ConfigMap
metadata:creationTimestamp: "2025-05-02T10:49:39Z"name: xpnamespace: defaultresourceVersion: "670826"uid: b129e625-7733-4b80-9d9f-55227b473f51那如何在声明式中引用这个key：vlaue呢
root@ubuntu0:~/manifests/configmap# cat 01-cm-demo.yaml  
apiVersion: v1
kind: ReplicationController 
metadata:name: xp-configmap
spec:replicas: 1selector:apps: v1template:metadata:labels:apps: v1spec:nodeName: ubuntu1containers:- name: xiuxian-v1image: registry.cn-hangzhou.aliyuncs.com/yinzhengjie-k8s/apps:v1env:- name: SCHOOL# 指定值从哪里来，一点定义了valueFrom字段，则不能定义value字段。# 换句话说，定义了valueFrom字段，则value字段必须为空，省略不写!valueFrom:# 表示数据从一个cm资源引用configMapKeyRef:# 指定cm的名称name: "xp"# 指定引用cm的KEYkey: "school"- name: CLassvalueFrom:configMapKeyRef:name: "xp"key: "class"
root@ubuntu0:~/manifests/configmap# kubectl apply -f 01-cm-demo.yaml 
replicationcontroller/xp-configmap created
root@ubuntu0:~/manifests/configmap# kubectl exec -it xp-configmap-vvtch -- env|grep -Ei 'school|class'
SCHOOL=oldboyedu
CLass=linux94

Pod基于存储卷引用cm资源

root@ubuntu0:~/manifests/configmap# cat 02-rc-configmaps-volumes.yaml 
apiVersion: v1
kind: ReplicationController 
metadata:name: xpxp
spec:replicas: 1selector: apps: xpxp-v1template:metadata:labels:apps: xpxp-v1spec:nodeName: ubuntu1volumes:- name: data# 指定存储卷类型是cm资源configMap:# 指定cm的名称name: "xp"# 定义需要引用具体的KEY，若不定义，则默认引用所有的KEYitems:# 表示引用指定的KEY- key: school # 可以暂时理解为将来的文件名称path: school.txtcontainers:- name: nginximage: registry.cn-hangzhou.aliyuncs.com/yinzhengjie-k8s/apps:v1volumeMounts:- name: datamountPath: /oldboyeduroot@ubuntu0:~/manifests/configmap# kubectl apply -f 02-rc-configmaps-volumes.yaml 
replicationcontroller/xpxp created
root@ubuntu0:~/manifests/configmap# kubectl get pods -o wide
NAME         READY   STATUS    RESTARTS   AGE   IP            NODE      NOMINATED NODE   READINESS GATES
xpxp-25pwf   1/1     Running   0          25m   10.100.2.19   ubuntu1   <none>           <none>
root@ubuntu0:~/manifests/configmap# kubectl exec xpxp-25pwf -- ls /oldboyedu
school.txt
root@ubuntu0:~/manifests/configmap# kubectl exec xpxp-25pwf -- more /oldboyedu/school.txt
oldboyeduroot@ubuntu0:~/manifests/configmap# 将"registry.cn-hangzhou.aliyuncs.com/yinzhengjie-k8s/apps:v1"镜像的80端口修改为81端口，要求在不重新打镜像的情况下，使用cm存储卷的方式挂载。1.找到nginx的配置文件
root@ubuntu0:~/manifests/configmap# kubectl exec -it xp-configmap-pgc6z -- sh
/ # vi /etc/nginx/nginx.conf 
/ # ls /etc/nginx/conf.d/default.conf 
/etc/nginx/conf.d/default.conf
/ # vi /etc/nginx/conf.d/default.conf 
server {                   listen       80;       listen  [::]:80;                                  server_name  localhost;                                                               location / {                     root   /usr/share/nginx/html;index  index.html index.htm;        }                                              error_page   500 502 503 504  /50x.html;location = /50x.html {           root   /usr/share/nginx/html;                          }  }  
2.编写资源清单root@ubuntu0:~/manifests/configmap# cat 02-rc-configmaps-volumes.yaml 
apiVersion: v1
kind: ConfigMap
metadata:name: game-cm
data:port.conf: |server {                   listen       81;       listen  [::]:81;                                  server_name  localhost;                                                               location / {                     root   /usr/share/nginx/html;index  index.html index.htm;        }                                              error_page   500 502 503 504  /50x.html;location = /50x.html {           root   /usr/share/nginx/html;                          }  }  
---
apiVersion: v1
kind: ReplicationController 
metadata:name: xiuxian-cm
spec:replicas: 1selector: apps: xpxp-v1template:metadata:labels:apps: xpxp-v1spec:nodeName: ubuntu1volumes:- name: data# 指定存储卷类型是cm资源configMap:# 指定cm的名称name: "game-cm"# 定义需要引用具体的KEY，若不定义，则默认引用所有的KEYitems:# 表示引用指定的KEY- key: port.conf# 可以暂时理解为将来的文件名称path: default.confcontainers:- name: nginximage: registry.cn-hangzhou.aliyuncs.com/yinzhengjie-k8s/apps:v1volumeMounts:- name: datamountPath: /etc/nginx/conf.d/
---
apiVersion: v1
kind: Service
metadata:name: svc-mysql
spec:type: NodePortselector:apps: xpxp-v1ports:- port: 80targetPort: 81nodePort: 30081
root@ubuntu0:~/manifests/configmap# kubectl apply -f 02-rc-configmaps-volumes.yaml 
configmap/game-cm created
replicationcontroller/xiuxian-cm created
service/svc-mysql created
root@ubuntu0:~/manifests/configmap# kubectl get pods -o wide
NAME               READY   STATUS    RESTARTS   AGE   IP            NODE      NOMINATED NODE   READINESS GATES
xiuxian-cm-b52m2   1/1     Running   0          7s    10.100.2.21   ubuntu1   <none>           <none>
root@ubuntu0:~/manifests/configmap# kubectl describe svc 
kubernetes  svc-mysql   
root@ubuntu0:~/manifests/configmap# kubectl describe svc svc-mysql 
Name:                     svc-mysql
Namespace:                default
Labels:                   <none>
Annotations:              <none>
Selector:                 apps=xpxp-v1
Type:                     NodePort
IP Family Policy:         SingleStack
IP Families:              IPv4
IP:                       192.168.116.228
IPs:                      192.168.116.228
Port:                     <unset>  80/TCP
TargetPort:               81/TCP
NodePort:                 <unset>  30081/TCP
Endpoints:                10.100.2.21:81
Session Affinity:         None
External Traffic Policy:  Cluster
Events:                   <none>
root@ubuntu0:~/manifests/configmap# curl 10.100.2.21:81
<!DOCTYPE html>
<html><head><meta charset="utf-8"/><title>yinzhengjie apps v1</title><style>div img {width: 900px;height: 600px;margin: 0;}</style></head><body><h1 style="color: green">凡人修仙传 v1 </h1><div><img src="1.jpg"><div></body></html>在进入容器内，已经发生改变了
root@ubuntu0:~/manifests/configmap# kubectl get pods -o wide
NAME               READY   STATUS    RESTARTS   AGE     IP            NODE      NOMINATED NODE   READINESS GATES
xiuxian-cm-b52m2   1/1     Running   0          2m38s   10.100.2.21   ubuntu1   <none>           <none>
root@ubuntu0:~/manifests/configmap# kubectl exec -it xiuxian-cm-b52m2 -- sh
/ # cat /etc/nginx/conf.d/default.conf 
server {                   listen       81;       listen  [::]:81;                                  server_name  localhost;                                                               location / {                     root   /usr/share/nginx/html;index  index.html index.htm;        }                                              error_page   500 502 503 504  /50x.html;location = /50x.html {           root   /usr/share/nginx/html;                          }  }

kubectl logs查看Pod日志

	1.实时查看日志
[root@master231 ~]# kubectl get pods -o wide
NAME                         READY   STATUS    RESTARTS   AGE     IP            NODE        NOMINATED NODE   READINESS GATES
oldboyedu-xiuxian-cm-844zl   1/1     Running   0          4m47s   10.100.1.55   worker232   <none>           <none>
[root@master231 ~]# 
[root@master231 ~]# kubectl logs -f oldboyedu-xiuxian-cm-844zl 
/docker-entrypoint.sh: /docker-entrypoint.d/ is not empty, will attempt to perform configuration
/docker-entrypoint.sh: Looking for shell scripts in /docker-entrypoint.d/
/docker-entrypoint.sh: Launching /docker-entrypoint.d/10-listen-on-ipv6-by-default.sh
10-listen-on-ipv6-by-default.sh: info: can not modify /etc/nginx/conf.d/default.conf (read-only file system?)
/docker-entrypoint.sh: Launching /docker-entrypoint.d/20-envsubst-on-templates.sh
/docker-entrypoint.sh: Launching /docker-entrypoint.d/30-tune-worker-processes.sh
/docker-entrypoint.sh: Configuration complete; ready for start up
2024/11/20 03:10:00 [notice] 1#1: using the "epoll" event method
2024/11/20 03:10:00 [notice] 1#1: nginx/1.20.1
2024/11/20 03:10:00 [notice] 1#1: built by gcc 10.2.1 20201203 (Alpine 10.2.1_pre1) 
2024/11/20 03:10:00 [notice] 1#1: OS: Linux 5.15.0-119-generic
2024/11/20 03:10:00 [notice] 1#1: getrlimit(RLIMIT_NOFILE): 524288:524288
2024/11/20 03:10:00 [notice] 1#1: start worker processes
2024/11/20 03:10:00 [notice] 1#1: start worker process 23
2024/11/20 03:10:00 [notice] 1#1: start worker process 24
10.100.0.0 - - [20/Nov/2024:03:10:08 +0000] "GET / HTTP/1.1" 200 357 "-" "curl/7.81.0" "-"
10.100.0.0 - - [20/Nov/2024:03:10:13 +0000] "GET / HTTP/1.1" 200 357 "-" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36" "-"
10.100.0.0 - - [20/Nov/2024:03:10:13 +0000] "GET /1.jpg HTTP/1.1" 200 233472 "http://10.0.0.231:30080/" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36" "-"
2024/11/20 03:10:13 [error] 24#24: *2 open() "/usr/share/nginx/html/favicon.ico" failed (2: No such file or directory), client: 10.100.0.0, server: localhost, request: "GET /favicon.ico HTTP/1.1", host: "10.0.0.231:30080", referrer: "http://10.0.0.231:30080/"
10.100.0.0 - - [20/Nov/2024:03:10:13 +0000] "GET /favicon.ico HTTP/1.1" 404 555 "http://10.0.0.231:30080/" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36" "-"2.查看最近5min的日志
[root@master231 ~]# kubectl logs -f --since=5m oldboyedu-xiuxian-cm-844zl 
2024/11/20 03:16:22 [error] 24#24: *4 open() "/usr/share/nginx/html/oldboyedu.html" failed (2: No such file or directory), client: 10.100.0.0, server: localhost, request: "GET /oldboyedu.html HTTP/1.1", host: "10.100.1.55:81"
10.100.0.0 - - [20/Nov/2024:03:16:22 +0000] "GET /oldboyedu.html HTTP/1.1" 404 153 "-" "curl/7.81.0" "-"3.查看指定容器的日志（一般情况下是一个Pod有多个容器时才会使用）
[root@master231 ~]# kubectl logs -c c1 -f --since=5m oldboyedu-xiuxian-cm-844zl 
2024/11/20 03:16:22 [error] 24#24: *4 open() "/usr/share/nginx/html/oldboyedu.html" failed (2: No such file or directory), client: 10.100.0.0, server: localhost, request: "GET /oldboyedu.html HTTP/1.1", host: "10.100.1.55:81"
10.100.0.0 - - [20/Nov/2024:03:16:22 +0000] "GET /oldboyedu.html HTTP/1.1" 404 153 "-" "curl/7.81.0" "-"

cm资源存储nginx主配置文件值subPath案例

root@ubuntu0:~/manifests/configmap# cat 02-rc-configmaps-volumes.yaml 
apiVersion: v1
kind: ConfigMap
metadata:name: game-cm
data:main.conf: |user  nginx;worker_processes  auto;error_log  /var/log/nginx/error.log notice;pid        /var/run/nginx.pid;events {worker_connections  1024;}http {include       /etc/nginx/mime.types;default_type  application/octet-stream;log_format oldboyedu_nginx_json '{"@timestamp":"$time_iso8601",''"host":"$server_addr",''"clientip":"$remote_addr",''"SendBytes":$body_bytes_sent,''"responsetime":$request_time,''"upstreamtime":"$upstream_response_time",''"upstreamhost":"$upstream_addr",''"http_host":"$host",''"uri":"$uri",''"domain":"$host",''"xff":"$http_x_forwarded_for",''"referer":"$http_referer",''"tcp_xff":"$proxy_protocol_addr",''"http_user_agent":"$http_user_agent",''"status":"$status"}';access_log  /var/log/nginx/access.log  oldboyedu_nginx_json;sendfile        on;keepalive_timeout  65;include /etc/nginx/conf.d/*.conf;}port.conf: |server {                   listen       81;       listen  [::]:81;                                  server_name  localhost;                                                               location / {                     root   /usr/share/nginx/html;index  index.html index.htm;        }                                              error_page   500 502 503 504  /50x.html;location = /50x.html {           root   /usr/share/nginx/html;                          }  }  
---
apiVersion: v1
kind: ReplicationController 
metadata:name: xiuxian-cm
spec:replicas: 1selector: apps: xpxp-v1template:metadata:labels:apps: xpxp-v1spec:nodeName: ubuntu1volumes:- name: data# 指定存储卷类型是cm资源configMap:# 指定cm的名称name: "game-cm"# 定义需要引用具体的KEY，若不定义，则默认引用所有的KEYitems:# 表示引用指定的KEY- key: port.conf# 可以暂时理解为将来的文件名称path: default.conf- name: data1configMap:name: "game-cm"items:- key: main.confpath: nginx.conf containers:- name: nginximage: registry.cn-hangzhou.aliyuncs.com/yinzhengjie-k8s/apps:v1volumeMounts:- name: datamountPath: /etc/nginx/conf.d/- name: data1mountPath: /etc/nginx/nginx.conf# 当subPath的值和cm的items的path值相同时，则mountPath表示的是文件而不是目录subPath: nginx.conf
---
apiVersion: v1
kind: Service
metadata:name: svc-mysql
spec:type: NodePortselector:apps: xpxp-v1ports:- port: 80targetPort: 81nodePort: 30081root@ubuntu0:~/manifests/configmap# kubectl get pods -o wide
NAME               READY   STATUS    RESTARTS   AGE   IP            NODE      NOMINATED NODE   READINESS GATES
xiuxian-cm-8q45n   1/1     Running   0          7s    10.100.2.22   ubuntu1   <none>           <none>
root@ubuntu0:~/manifests/configmap# curl 10.100.2.22:81
<!DOCTYPE html>
<html><head><meta charset="utf-8"/><title>yinzhengjie apps v1</title><style>div img {width: 900px;height: 600px;margin: 0;}</style></head><body><h1 style="color: green">凡人修仙传 v1 </h1><div><img src="1.jpg"><div></body></html>

k8s部署mysql主从

root@ubuntu0:~/manifests/ReplicationController# cat 05-nfs-mysql.yaml 
apiVersion: v1
kind: ConfigMap
metadata:name: game-demo
# 指定cm资源的数据
data:master.cnf: |[mysqld]# 二进制日志log-bin=mysqllog-binserver_id=111skip-host-cacheskip-name-resolvedatadir=/var/lib/mysqlsocket=/var/run/mysqld/mysqld.socksecure-file-priv=/var/lib/mysql-filesuser=mysqlpid-file=/var/run/mysqld/mysqld.pid[client]socket=/var/run/mysqld/mysqld.sock !includedir /etc/mysql/conf.d/slave.cnf: |[mysqld]log-bin=mysqllog-binserver_id=222skip-host-cacheskip-name-resolvedatadir=/var/lib/mysqlsocket=/var/run/mysqld/mysqld.socksecure-file-priv=/var/lib/mysql-filesuser=mysqlpid-file=/var/run/mysqld/mysqld.pid[client]socket=/var/run/mysqld/mysqld.sock !includedir /etc/mysql/conf.d/
---
apiVersion: v1
kind: ReplicationController 
metadata:name: mysql-master
spec:replicas: 1selector:apps: v1 template:spec:nodeName: ubuntu1volumes:- name: datanfs:server: ubuntu0path: /oldboyedu/data/nfs-server/master-lib - name: data1configMap:name: "game-demo"items:- key: master.cnfpath: my.cnf containers:- name: mysql-v1image: mysql:5.7.29ports:- containerPort: 3306name: mysqlportenv:- name: MYSQL_ALLOW_EMPTY_PASSWORDvalue: "yes"- name: MYSQL_USERvalue: linux94- name: MYSQL_PASSWORDvalue: 'oldboyedu'volumeMounts:- name: datamountPath: /var/lib/mysql- name: data1mountPath: /etc/my.cnfsubPath: my.cnfargs:- --character-set-server=utf8- --collation-server=utf8_bin- --default-authentication-plugin=mysql_native_passwordmetadata:labels:apps: v1  
---
apiVersion: v1
kind: Service
metadata:name: svc-mysql
spec:selector:apps: v1ports:- port: 3306 name: mysqlport
---apiVersion: v1
kind: ReplicationController 
metadata:name: mysql-slave
spec:replicas: 1selector:apps: v2template:spec:nodeName: ubuntu1volumes:- name: datanfs:server: ubuntu0path: /oldboyedu/data/nfs-server/slave-lib - name: data1configMap:name: "game-demo"items:- key: slave.cnfpath: my.cnf containers:- name: mysql-v2image: mysql:5.7.29env:- name: MYSQL_ALLOW_EMPTY_PASSWORDvalue: "yes"- name: MYSQL_MASTER_HOSTvalue: 'svc-mysql'ports:- containerPort: 3306volumeMounts:- name: datamountPath: /var/lib/mysql- name: data1mountPath: /etc/my.cnfsubPath: my.cnfmetadata:labels:apps: v2  
---
apiVersion: v1
kind: Service
metadata:name: svc-slave
spec:selector:apps: v2ports:- port: 3306进入主数据库查看
root@ubuntu0:~/manifests/ReplicationController# kubectl exec -it mysql-master-4zxp6 -- mysql
Welcome to the MySQL monitor.  Commands end with ; or \g.
Your MySQL connection id is 2
Server version: 5.7.29-log MySQL Community Server (GPL)Copyright (c) 2000, 2020, Oracle and/or its affiliates. All rights reserved.Oracle is a registered trademark of Oracle Corporation and/or its
affiliates. Other names may be trademarks of their respective
owners.Type 'help;' or '\h' for help. Type '\c' to clear the current input statement.授权用户访问
mysql> GRANT Replication slave ON *.* TO linux94;
Query OK, 0 rows affected (0.00 sec)mysql> SHOW MASTER STATUS\G
*************************** 1. row ***************************File: mysqllog-bin.000004Position: 353Binlog_Do_DB: Binlog_Ignore_DB: 
Executed_Gtid_Set: 
1 row in set (0.00 sec)mysql> SHOW GRANTS FOR linux94;
+-------------------------------------------------+
| Grants for linux94@%                            |
+-------------------------------------------------+
| GRANT REPLICATION SLAVE ON *.* TO 'linux94'@'%' |
+-------------------------------------------------+从库配置
mysql> CHANGE MASTER TO MASTER_HOST='svc-mysql',MASTER_USER='linux94',MASTER_PASSWORD='oldboyedu',MASTER_PORT=3306,MASTER_LOG_FILE='mysqllog-bin.000004',MASTER_LOG_POS=353,MASTER_CONNECT_RETRY=3;
Query OK, 0 rows affected, 2 warnings (0.04 sec)mysql> STOP SLAVE;
Query OK, 0 rows affected, 1 warning (0.00 sec)mysql> START SLAVE;
Query OK, 0 rows affected (0.00 sec)mysql> SHOW SLAVE STATUS\G
*************************** 1. row ***************************Slave_IO_State: Waiting for master to send eventMaster_Host: svc-mysqlMaster_User: linux94Master_Port: 3306Connect_Retry: 3Master_Log_File: mysqllog-bin.000004Read_Master_Log_Pos: 353Relay_Log_File: mysql-slave-2pgsd-relay-bin.000002Relay_Log_Pos: 323Relay_Master_Log_File: mysqllog-bin.000004Slave_IO_Running: YesSlave_SQL_Running: Yes

secret概述

与ConfigMap类似，区别在于secret存储敏感数据，所有的数据都需要经过base64进行编码。
使用secret主要存储的是凭据信息。参考链接:https://kubernetes.io/zh/docs/concepts/configuration/secret/#secret-typessecret资源声明式两种创建方式2.1 方式一: 基于stringData方式（推荐）
root@ubuntu0:~/manifests/secret# cat 01-secrets-stringData.yaml 
apiVersion: v1
kind: Secret
metadata:name: user-info
stringData:username: adminpassword: "1"my.cnf: |[mysqld]basedir=/oldboyedu/softwares/mysql80port=3306datadir=/oldboyedu/data/mysql80socket=/tmp/mysql80.sock
root@ubuntu0:~/manifests/secret# kubectl apply -f 01-secrets-stringData.yaml 
secret/user-info created
root@ubuntu0:~/manifests/secret# kubectl get secrets -o wideNAME                  TYPE                                  DATA   AGE
default-token-hszqs   kubernetes.io/service-account-token   3      26d
user-info             Opaque                                3      7s
root@ubuntu0:~/manifests/secret# kubectl get secrets -o wide user-info
NAME        TYPE     DATA   AGE
user-info   Opaque   3      46s
root@ubuntu0:~/manifests/secret# kubectl describe secrets  user-info 
Name:         user-info
Namespace:    default
Labels:       <none>
Annotations:  <none>Type:  OpaqueData
====
my.cnf:    113 bytes
password:  1 bytes
username:  5 bytes
root@ubuntu0:~/manifests/secret#  kubectl get secrets user-info -o yaml
apiVersion: v1
data:my.cnf: W215c3FsZF0KYmFzZWRpcj0vb2xkYm95ZWR1L3NvZnR3YXJlcy9teXNxbDgwCnBvcnQ9MzMwNgpkYXRhZGlyPS9vbGRib3llZHUvZGF0YS9teXNxbDgwCnNvY2tldD0vdG1wL215c3FsODAuc29jawo=password: MQ==username: YWRtaW4=
kind: Secret
metadata:annotations:kubectl.kubernetes.io/last-applied-configuration: |{"apiVersion":"v1","kind":"Secret","metadata":{"annotations":{},"name":"user-info","namespace":"default"},"stringData":{"my.cnf":"[mysqld]\nbasedir=/oldboyedu/softwares/mysql80\nport=3306\ndatadir=/oldboyedu/data/mysql80\nsocket=/tmp/mysql80.sock\n","password":"1","username":"admin"}}creationTimestamp: "2025-05-05T02:16:24Z"name: user-infonamespace: defaultresourceVersion: "815355"uid: 90653274-1030-4208-a555-032c6484029f
type: Opaque解密：
root@ubuntu0:~/manifests/secret# echo 'W215c3FsZF0KYmFzZWRpcj0vb2xkYm95ZWR1L3NvZnR3YXJlcy9teXNxbDgwCnBvcnQ9MzMwNgpkYXRhZGlyPS9vbGRib3llZHUvZGF0YS9teXNxbDgwCnNvY2tldD0vdG1wL215c3FsO
DAuc29jawo='|base64 -d
[mysqld]
basedir=/oldboyedu/softwares/mysql80
port=3306
datadir=/oldboyedu/data/mysql80
socket=/tmp/mysql80.sock
root@ubuntu0:~/manifests/secret# echo 'MQ=='|base64 -d
1root@ubuntu0:~/manifests/secret# echo 'MQ=='|base64 -d|more
1方式二: 基于方式（不推荐，编写时容易出错）
root@ubuntu0:~/manifests/secret# echo linux94 | base64 
bGludXg5NAo=
root@ubuntu0:~/manifests/secret#  echo oldboyedu | base64 
b2xkYm95ZWR1Cg==
root@ubuntu0:~/manifests/secret# cat 02-secrets-data.yaml 
apiVersion: v1
kind: Secret
metadata:name: admin
data:# KEY无需做任何操作，VALUE进行BASE64手动编码username: bGludXg5NAo=password: b2xkYm95ZWR1Cg==
root@ubuntu0:~/manifests/secret# echo linux94 | base64 
bGludXg5NAo=
root@ubuntu0:~/manifests/secret#  echo oldboyedu | base64 
b2xkYm95ZWR1Cg==
root@ubuntu0:~/manifests/secret# cat 02-secrets-data.yaml 
apiVersion: v1
kind: Secret
metadata:name: admin
data:# KEY无需做任何操作，VALUE进行BASE64手动编码username: bGludXg5NAo=password: b2xkYm95ZWR1Cg==
root@ubuntu0:~/manifests/secret# kubectl apply -f 02-secrets-data.yaml 
secret/admin created
root@ubuntu0:~/manifests/secret# kubectl get secrets admin 
NAME    TYPE     DATA   AGE
admin   Opaque   2      6s
root@ubuntu0:~/manifests/secret# kubectl get secrets admin -o yaml
apiVersion: v1
data:password: b2xkYm95ZWR1Cg==username: bGludXg5NAo=
kind: Secret
metadata:annotations:kubectl.kubernetes.io/last-applied-configuration: |{"apiVersion":"v1","data":{"password":"b2xkYm95ZWR1Cg==","username":"bGludXg5NAo="},"kind":"Secret","metadata":{"annotations":{},"name":"admin","namespace":"default"}}creationTimestamp: "2025-05-05T02:21:09Z"name: adminnamespace: defaultresourceVersion: "815756"uid: 4e62f0f6-1940-43f7-b81c-16278cda898d
type: Opaque

响应式创建secrets常用选项

root@ubuntu0:~/manifests/secret# kubectl create secret generic test01 --from-literal=SCHOOL=oldboyedu --from-literal=class=linux94
secret/test01 created
root@ubuntu0:~/manifests/secret# kubectl create secret generic test02 --from-file=stringData=01-secrets-stringData.yaml --from-file=Data=02-secrets-data.yaml
secret/test02 created
root@ubuntu0:~/manifests/secret#  kubectl get secrets test01  test02 
NAME     TYPE     DATA   AGE
test01   Opaque   2      20s
test02   Opaque   2      9s
root@ubuntu0:~/manifests/secret# kubectl get secrets test01  test02  -o yaml
apiVersion: v1
items:
- apiVersion: v1data:SCHOOL: b2xkYm95ZWR1class: bGludXg5NA==kind: Secretmetadata:creationTimestamp: "2025-05-05T02:39:39Z"name: test01namespace: defaultresourceVersion: "817317"uid: 9255edec-24db-4490-a8b3-d0951b21c470type: Opaque
- apiVersion: v1data:Data: YXBpVmVyc2lvbjogdjEKa2luZDogU2VjcmV0Cm1ldGFkYXRhOgogIG5hbWU6IGFkbWluCmRhdGE6CiAgIyBLRVnml6DpnIDlgZrku7vkvZXmk43kvZzvvIxWQUxVRei/m+ihjEJBU0U2NOaJi+WKqOe8lueggQogIHVzZXJuYW1lOiBiR2x1ZFhnNU5Bbz0KICBwYXNzd29yZDogYjJ4a1ltOTVaV1IxQ2c9PQo=stringData: YXBpVmVyc2lvbjogdjEKa2luZDogU2VjcmV0Cm1ldGFkYXRhOgogIG5hbWU6IHVzZXItaW5mbwpzdHJpbmdEYXRhOgogIHVzZXJuYW1lOiBhZG1pbgogIHBhc3N3b3JkOiAiMSIKCiAgbXkuY25mOiB8CiAgICBbbXlzcWxkXQogICAgYmFzZWRpcj0vb2xkYm95ZWR1L3NvZnR3YXJlcy9teXNxbDgwCiAgICBwb3J0PTMzMDYKICAgIGRhdGFkaXI9L29sZGJveWVkdS9kYXRhL215c3FsODAKICAgIHNvY2tldD0vdG1wL215c3FsODAuc29jawo=kind: Secretmetadata:creationTimestamp: "2025-05-05T02:39:50Z"name: test02namespace: defaultresourceVersion: "817332"uid: b1476fdd-5182-426d-a784-7ee5b16342c1type: Opaque
kind: List
metadata:resourceVersion: ""selfLink: ""
root@ubuntu0:~/manifests/secret#  kubectl delete secrets test01  test02 
secret "test01" deleted
secret "test02" deleted

Pod引用secrets的两种方式

基于环境变量引入1.先查看一下user-info的变量
root@ubuntu0:~/manifests/secret# kubectl get secrets user-info 
NAME        TYPE     DATA   AGE
user-info   Opaque   3      4d21h
root@ubuntu0:~/manifests/secret# kubectl get secrets user-info -o yaml
apiVersion: v1
data:my.cnf: W215c3FsZF0KYmFzZWRpcj0vb2xkYm95ZWR1L3NvZnR3YXJlcy9teXNxbDgwCnBvcnQ9MzMwNgpkYXRhZGlyPS9vbGRib3llZHUvZGF0YS9teXNxbDgwCnNvY2tldD0vdG1wL215c3FsODAuc29jawo=password: MQ==username: YWRtaW4=
kind: Secret2.基于环境变量引入root@ubuntu0:~/manifests/secret# cat 04-secret-env.yaml 
apiVersion: v1
kind: ReplicationController
metadata:name: secret-env
spec:replicas: 1selector:apps: v1template:metadata:labels:apps: v1spec:nodeName: ubuntu1containers:- name: xpimage: registry.cn-hangzhou.aliyuncs.com/yinzhengjie-k8s/apps:v1env:- name: env_usernamevalueFrom:#表示值从一个secrets资源中引用secretKeyRef:# 指定secrets资源的名称name: user-info# 引用secrets的keykey: username- name: env_mycnfvalueFrom:#表示值从一个secrets资源中引用secretKeyRef:# 指定secrets资源的名称name: user-info# 引用secrets的keykey: my.cnf
root@ubuntu0:~/manifests/secret# kubectl apply -f 04-secret-env.yaml 
replicationcontroller/secret-env created
root@ubuntu0:~/manifests/secret# kubectl get pods -o wide
NAME                 READY   STATUS    RESTARTS   AGE   IP            NODE      NOMINATED NODE   READINESS GATES
mysql-master-4zxp6   1/1     Running   0          5d    10.100.2.30   ubuntu1   <none>           <none>
mysql-slave-2pgsd    1/1     Running   0          5d    10.100.2.29   ubuntu1   <none>           <none>
secret-env-78xjs     1/1     Running   0          29s   10.100.2.32   ubuntu1   <none>           <none>
查看环境变量，他会自动的解密
root@ubuntu0:~/manifests/secret# kubectl exec -it secret-env-78xjs -- env
env_username=admin
env_mycnf=[mysqld]
basedir=/oldboyedu/softwares/mysql80
port=3306
datadir=/oldboyedu/data/mysql80
socket=/tmp/mysql80.sock基于存储卷的方式引用
root@ubuntu0:~/manifests/secret# cat 03-secret.yaml 
apiVersion: v1
kind: ReplicationController
metadata:name: oldboyedu-rc-nfs-v1
spec:replicas: 1selector:apps: v1template:metadata:labels:apps: v1spec:nodeName: ubuntu1volumes:- name: data#表示存储卷的类型是secretsecret: # 指定secret的名称secretName: user-info# 指定要引用的键值对items:# 指定secrets的KEY - key: username# 暂时理解为将来在Pod容器挂载时的文件名称path: username.txt- key: passwordpath: password.txt- key: my.cnfpath: my.cnfcontainers:- name: nginximage: registry.cn-hangzhou.aliyuncs.com/yinzhengjie-k8s/apps:v1 volumeMounts:- name: datamountPath: /oldboyeduroot@ubuntu0:~/manifests/secret# kubectl get pods -o wide
NAME                        READY   STATUS    RESTARTS   AGE   IP            NODE      NOMINATED NODE   READINESS GATES
mysql-master-4zxp6          1/1     Running   0          5d    10.100.2.30   ubuntu1   <none>           <none>
mysql-slave-2pgsd           1/1     Running   0          5d    10.100.2.29   ubuntu1   <none>           <none>
oldboyedu-rc-nfs-v1-5cx5d   1/1     Running   0          8s    10.100.2.33   ubuntu1   <none>           <none>
root@ubuntu0:~/manifests/secret# kubectl exec -it oldboyedu-rc-nfs-v1-5cx5d -- sh
/ # ls -l /oldboyedu/
total 0
lrwxrwxrwx    1 root     root            13 May 10 00:42 my.cnf -> ..data/my.cnf
lrwxrwxrwx    1 root     root            19 May 10 00:42 password.txt -> ..data/password.txt
lrwxrwxrwx    1 root     root            19 May 10 00:42 username.txt -> ..data/username.txt
/ # cat /oldboyedu/my.cnf 
[mysqld]
basedir=/oldboyedu/softwares/mysql80
port=3306
datadir=/oldboyedu/data/mysql80
socket=/tmp/mysql80.sock

基于响应式secret实现harbor登录认证案例

1.响应式创建harbor的认证信息
[root@master231 case-demo]# kubectl create secret docker-registry oldboyedu-harbor --docker-username=admin --docker-password=1 --docker-email=admin@oldboyedu.com --docker-server=harbor.oldboyedu.com
secret/oldboyedu-harbor created
[root@master231 case-demo]# kubectl get secrets oldboyedu-harbor 
NAME               TYPE                             DATA   AGE
oldboyedu-harbor   kubernetes.io/dockerconfigjson   1      9s2.创建测试 
[root@master231 case-demo]# cat 16-rc-secrets-private-harbor-registry.yaml 
apiVersion: v1
kind: ReplicationController
metadata:name: oldboyedu-private-harbor
spec:replicas: 3selector:apps: linuxtemplate:spec:# 镜像拉取的认证凭据imagePullSecrets:# 指定访问harbor的认证信息- name: oldboyedu-harborcontainers:- name: c1image: harbor.oldboyedu.com/oldboyedu-linux/alpine:latestimagePullPolicy: Alwaysstdin: truemetadata:labels:apps: linux
[root@master231 case-demo]# 
[root@master231 case-demo]# kubectl apply -f 16-rc-secrets-private-harbor-registry.yaml 
replicationcontroller/oldboyedu-private-harbor created
[root@master231 case-demo]# 
[root@master231 case-demo]# kubectl get pods -o wide
NAME                             READY   STATUS        RESTARTS   AGE   IP             NODE        NOMINATED NODE   READINESS GATES
oldboyedu-private-harbor-f7hmj   1/1     Running       0          4s    10.100.1.64    worker232   <none>           <none>
oldboyedu-private-harbor-gkmtm   1/1     Running       0          4s    10.100.2.120   worker233   <none>           <none>
oldboyedu-private-harbor-pmf5q   1/1     Running       0          4s    10.100.2.119   worker233   <none>           <none>
[root@master231 case-demo]#

基于声明式secret实现harbor登录认证案例

1.harbor创建用户名和密码 用户名称: linux94 密码: Linux@2024邮箱: linux94@oldboyedu.com 2.对认证信息进行base64编码[root@master231 case-demo]# echo -n linux94:Linux@2024 | base64 bGludXg5NDpMaW51eEAyMDI0[root@master231 case-demo]#
3.得到最终的认证信息:{"auths":{"harbor.oldboyedu.com":{"username":"linux94","password":"Linux@2024","email":"linux94@oldboyedu.com","auth":"bGludXg5NDpMaW51eEAyMDI0"}}} 4.编写资源清单
[root@master231 case-demo]# cat 16-rc-secrets-private-harbor-registry.yaml 
apiVersion: v1
kind: Secret
metadata:name: linux94-harbor
stringData:.dockerconfigjson: '{"auths":{"harbor.oldboyedu.com":{"username":"linux94","password":"Linux@2024","email":"linux94@oldboyedu.com","auth":"bGludXg5NDpMaW51eEAyMDI0"}}}'
type: kubernetes.io/dockerconfigjson---apiVersion: v1
kind: ReplicationController
metadata:name: oldboyedu-private-harbor
spec:replicas: 3selector:apps: linuxtemplate:spec:# 镜像拉取的认证凭据imagePullSecrets:# 指定访问harbor的认证信息- name: linux94-harborcontainers:- name: c1image: harbor.oldboyedu.com/oldboyedu-linux/alpine:latestimagePullPolicy: Alwaysstdin: truemetadata:labels:apps: linux
[root@master231 case-demo]# 
[root@master231 case-demo]# kubectl apply -f 16-rc-secrets-private-harbor-registry.yaml
secret/linux94-harbor created
replicationcontroller/oldboyedu-private-harbor created
[root@master231 case-demo]# 
[root@master231 case-demo]# kubectl get pods -o wide
NAME                             READY   STATUS    RESTARTS   AGE   IP             NODE        NOMINATED NODE   READINESS GATES
oldboyedu-private-harbor-6kf6t   1/1     Running   0          3s    10.100.2.124   worker233   <none>           <none>
oldboyedu-private-harbor-prqnv   1/1     Running   0          3s    10.100.2.125   worker233   <none>           <none>
oldboyedu-private-harbor-tcp27   1/1     Running   0          3s    10.100.1.68    worker232   <none>           <none>
[root@master231 case-demo]#

基于serviceaccounts绑定secret实现harbor认证

root@ubuntu0:/oldboyedu/softwares/harbor# kubectl api-resources |grep -w sa
serviceaccounts                   sa           v1                                     true         ServiceAccount1.响应式创建账号
root@ubuntu0:/oldboyedu/softwares/harbor# kubectl create sa xixi
serviceaccount/xixi created
root@ubuntu0:/oldboyedu/softwares/harbor# kubectl get sa xixi        
NAME   SECRETS   AGE
xixi   1         61s
root@ubuntu0:/oldboyedu/softwares/harbor# kubectl get sa xixi -o yaml
apiVersion: v1
kind: ServiceAccount
metadata:creationTimestamp: "2025-05-18T12:03:44Z"name: xixinamespace: defaultresourceVersion: "1617334"uid: 8597121b-be87-4e33-a25b-d7f84a2fc43d
secrets:
- name: xixi-token-f4d9p2.查看账号后端的Image pull secrets
root@ubuntu0:/oldboyedu/softwares/harbor# kubectl describe sa xixi 
Name:                xixi
Namespace:           default
Labels:              <none>
Annotations:         <none>
Image pull secrets:  <none>
Mountable secrets:   xixi-token-f4d9p
Tokens:              xixi-token-f4d9p
Events:              <none>
由于上面的镜像拉去策略为空，所以更新账号绑定的信息
[root@master231 serviceaccounts]# kubectl patch sa xixi -p '{"imagePullSecrets":[{"name":"oldboyedu-harbor"}]}'
serviceaccount/xixi patched
[root@master231 serviceaccounts]# 
[root@master231 serviceaccounts]# kubectl describe sa xixi 
Name:                xixi
Namespace:           default
Labels:              <none>
Annotations:         <none>
Image pull secrets:  oldboyedu-harbor
Mountable secrets:   xixi-token-nk9z4
Tokens:              xixi-token-nk9z4
Events:              <none>4.响应式更新账号的信息
[root@master231 serviceaccounts]# kubectl get secrets oldboyedu-harbor 
NAME               TYPE                             DATA   AGE
oldboyedu-harbor   kubernetes.io/dockerconfigjson   1      66m
[root@master231 serviceaccounts]# 
[root@master231 serviceaccounts]# kubectl describe sa xixi 
Name:                xixi
Namespace:           default
Labels:              <none>
Annotations:         <none>
Image pull secrets:  <none>
Mountable secrets:   xixi-token-nk9z4
Tokens:              xixi-token-nk9z4
Events:              <none>
[root@master231 serviceaccounts]# 
[root@master231 serviceaccounts]# kubectl patch sa xixi -p '{"imagePullSecrets":[{"name":"oldboyedu-harbor"}]}'
serviceaccount/xixi patched
[root@master231 serviceaccounts]# 
[root@master231 serviceaccounts]# kubectl describe sa xixi 
Name:                xixi
Namespace:           default
Labels:              <none>
Annotations:         <none>
Image pull secrets:  oldboyedu-harbor
Mountable secrets:   xixi-token-nk9z4
Tokens:              xixi-token-nk9z4
Events:              <none>
[root@master231 serviceaccounts]# 5.Pod使用sa账号拉取镜像
[root@master231 case-demo]# cat 17-rc-secrets-sa-harbor.yaml 
apiVersion: v1
kind: Secret
metadata:name: linux94-harbor
stringData:.dockerconfigjson: '{"auths":{"harbor.oldboyedu.com":{"username":"linux94","password":"Linux@2024","email":"linux94@oldboyedu.com","auth":"bGludXg5NDpMaW51eEAyMDI0"}}}'
type: kubernetes.io/dockerconfigjson---apiVersion: v1
# 将sa账号绑定secret的拉取镜像认证信息
imagePullSecrets:
- name: linux94-harbor
kind: ServiceAccount
metadata:name: linux94namespace: default---apiVersion: v1
kind: ReplicationController
metadata:name: oldboyedu-private-harbor
spec:replicas: 3selector:apps: linuxtemplate:spec:# 指定服务账号，基于该账号拉取镜像serviceAccount: linux94containers:- name: c1image: harbor.oldboyedu.com/oldboyedu-linux/alpine:latestimagePullPolicy: Alwaysstdin: truemetadata:labels:apps: linux
[root@master231 case-demo]#

pod创建流程

- Pod创建流程：Pod的创建，删除，修改流程:1.执行kubectl命令时会加载"~/.kube/config"，从而识别到apiserver的地址，端口及认证证书;2.apiserver进行证书认证，鉴权，语法检查，若成功则可以进行数据的读取或者写入;3.若用户是写入操作(创建，修改，删除)则需要修改etcd数据库的信息;4.如果创建Pod，此时scheduler负责Pod调度，将Pod调度到合适的worker节点，并将结果返回给ApiServer存储到etcd中;5.kubelet组件会周期性上报给apiServer节点，包括Pod内的容器资源(cpu,memory,disk,gpu,...)及worker宿主机节点状态，apiServer并将结果存储到etcd中，若有该节点的任务也会直接返回给该节点进行调度;6.kubelet开始调用CRI接口创建容器(依次创建pause，initContainers，containers);7.在运行过程中，若Pod容器，正常或者异常退出时，kubelet会根据重启策略是否重启容器(Never,Always,OnFailure);8.若一个节点怪掉，则需要controller manager介入维护，比如Pod副本数量缺失，则需要创建watch事件，要求控制器的副本数要达到标准，从而要创建新的Pod，此过程重复步骤4-6。

k8s部署jenkins

apiVersion: v1
kind: Namespace
metadata:name: devops---apiVersion: v1
kind: ReplicationController
metadata:name: oldboyedu-jenkinsnamespace: devops
spec:replicas: 1selector:apps: jenkinstemplate:spec:nodeName: worker233volumes:- name: datanfs:server: 10.0.0.231path: /oldboyedu/data/nfs-server/volumes/devops/jenkinscontainers:- name: c1# image: jenkins/jenkins:2.479.1-alpine-jdk21image: harbor.oldboyedu.com/oldboyedu-devops/jenkins:2.479.1-alpine-jdk21#command: #- tail#- -f#- /etc/hostsports:- containerPort: 8080volumeMounts:- name: datamountPath: /var/jenkins_home/metadata:labels:apps: jenkins---apiVersion: v1
kind: Service
metadata:name: svc-jenkinsnamespace: devops
spec:type: NodePortselector:apps: jenkinsports:- port: 8080nodePort: 30083
[root@master231 case-demo]# 3.温馨提示:- 在使用资源清单之前，应该先将Jenkins运行起来，安装常用的插件;- 再将/var/jenkins_home/数据拷贝到"/oldboyedu/data/nfs-server/volumes/devops/jenkins"中。

相关文章：

  • ESP12E/F 参数对比
  • (LeetCode 动态规划（基础版）)96. 不同的二叉搜索树 （递推 || 递归）
  • 设备驱动与文件系统：04 从生磁盘到文件
  • 软件测试—学习Day11
  • 经典算法：排序链表
  • 【第九篇】 SpringBoot测试补充篇
  • leetcode238-除自身以外数组的乘积
  • 权限一览表
  • AUTOSAR实战教程--标准协议栈实现DoIP转DoCAN的方法
  • 第三讲 Linux进程概念
  • java_dependencies_父子模块引入依赖关系
  • CompletableFuture+线程池使用案列
  • @Import原理与实战
  • 自定义protoc-gen-go生成Go结构体，统一字段命名与JSON标签风格
  • Go语言系统监控实战：gopsutil库全面解析与应用
  • 75Qt窗口_Qt窗口概览
  • Redis集群模式之主从复制模式（2）
  • 轻量级的Windows系统优化与个性化解决方案
  • 汽车车载软件平台化项目规模颗粒度选择的一些探讨
  • JS Day05
  • 阿里巴巴国际站靠谱吗/百度快速查询
  • 网上做石材去哪个网站/做电商如何起步
  • 做正版电子书下载网站/seo一个月工资一般多少
  • 有关做甜点的网站/怎么把产品放到网上销售
  • 邯郸建移动网站费用/乐事薯片软文推广
  • wordpress网站怎么百度的到/杭州关键词推广优化方案