# 说明 ```bash 192.168.33.11 etcd集群节点1，安装etcd。 192.168.33.12 etcd集群节点2，安装etcd。 192.168.33.13 etcd集群节点3，安装etcd。 192.168.33.21 apiserver集群节点1，安装kube-apiserver、kube-controller-manager、kube-scheduler并同时安装kubelet、kube-proxy和dockerd并label为master。安装keepalived+haproxy，设置KUBE_MASTER_VIP="https://192.168.33.100:8443"。 192.168.33.22 apiserver集群节点2，安装kube-apiserver、kube-controller-manager、kube-scheduler并同时安装kubelet、kube-proxy和dockerd并label为master。安装keepalived+haproxy，设置KUBE_MASTER_VIP="https://192.168.33.100:8443"。 192.168.33.23 apiserver集群节点3，安装kube-apiserver、kube-controller-manager、kube-scheduler并同时安装kubelet、kube-proxy和dockerd并label为master。安装keepalived+haproxy，设置KUBE_MASTER_VIP="https://192.168.33.100:8443"。 192.168.33.26 仅安装kubelet、kube-proxy和dockerd并label为worker。 192.168.33.27 仅安装kubelet、kube-proxy和dockerd并label为worker。 ``` # 系统 ## Vagrantfile ```ruby config.ssh.username = "vagrant" config.ssh.password = "vagrant" #config.ssh.insert_key = false [11,12,13,21,22,23,26,27].each do |i| config.vm.define "n#{i}" do |node| node.vm.network "private_network", ip: "192.168.33.#{i}" node.vm.synced_folder "/data/vagrant/shell", "/shell" node.vm.network :forwarded_port, guest: 22, host: "2#{i}22", host_ip: "0.0.0.0" node.vm.provider "virtualbox" do |vb| vb.memory = "2048" vb.cpus = 2 end node.vm.provision "shell", inline: <<-SHELL echo "vagrant:vagrant" | sudo chpasswd mkdir -p /data echo IP=\"192.168.33.#{i}\" >>/data/env.sh echo ETCD_IP=\"192.168.33.#{i}\" >>/data/env.sh echo ETCD_NAME=\"etcd-#{i}\" >>/data/env.sh echo KUBE_NODENAME=\"n#{i}\" >>/data/env.sh echo KUBE_NODE_HOSTNAME=\"n#{i}.dev\" >>/data/env.sh echo HOSTNAME_=\"n#{i}\" >>/data/env.sh chown -R vagrant:vagrant /data hostnamectl set-hostname n#{i} timedatectl set-timezone Asia/Shanghai SHELL end end ``` ## centos7 ```bash # 所有的机器安装执行 #!/bin/bash mv /etc/yum.repos.d/CentOS-Base.repo /etc/yum.repos.d/CentOS-Base.repo.bak wget -O /etc/yum.repos.d/CentOS-Base.repo https://mirrors.163.com/.help/CentOS7-Base-163.repo rm -rf /etc/yum.repos.d/epel.repo yum clean all yum makecache yum install -y epel-release yum clean all yum makecache #yum repolist enabled #yum repolist all yum update yum upgrade yum install -y net-tools curl wget git vim jq socat conntrack ipvsadm ipset sysstat libseccomp gcc gcc-c++ cmake make bzip2 automake autoconf libtool flex bison pcre-devel zlib-devel openssl openssl-devel lsof htop patch lsof yum install -y libnfnetlink-devel libnl3 libnl3-devel systemd-devel yum install -y bridge-utils bind-utils psmisc yum install -y device-mapper-persistent-data lvm2 #yum install kernel-devel-$(uname -r) kernel-headers-$(uname -r) ``` ```bash sudo timedatectl set-timezone Asia/Shanghai cat > /tmp/mysysctl.conf < /tmp/etcd.service < value: "192.168.0.0/16" ``` ```bash kubectl apply -f calico_deploy.yaml # 部署calico后cni之bridge自动失效 # calicoctl vi calicoctl.yaml kubectl apply -f calicoctl.yaml ``` ```yaml apiVersion: v1 kind: Pod metadata: name: calicoctl namespace: kube-system spec: hostNetwork: true serviceAccountName: calico-node containers: - name: calicoctl image: quay.io/calico/ctl:v3.12.3 command: ["tail", "-f", "/dev/null"] env: - name: DATASTORE_TYPE value: kubernetes ``` ```bash kubectl -n kube-system get pod -l k8s-app=calico-node -o wide kubectl -n kube-system logs $(kubectl get po -o jsonpath={.items[0].metadata.name} -l k8s-app=calico-node -n kube-system) kubectl run --image=nginx:1.16.1-alpine --replicas=3 example kubectl run nginx --image=nginx:1.16.1 --replicas=5 kubectl get pod -l run=example --show-labels -o wide kubectl exec -ti -n kube-system calicoctl -- /calicoctl get workloadEndpoint -o wide kubectl exec -ti -n kube-system calicoctl -- /calicoctl get node -o wide kubectl exec -ti -n kube-system calicoctl -- /calicoctl get ipPool kubectl exec -ti -n kube-system calicoctl -- /calicoctl get ipPool -o json kubectl exec -ti -n kube-system calicoctl -- /calicoctl ipam show --ip=172.20.1.1 kubectl exec -ti -n kube-system calicoctl -- /calicoctl get bgpConfiguration ``` # 部署coredns `//部署` ```bash #!/bin/bash #export http_proxy= #export https_proxy= #10.254.0.2 为cluster-ip,根据情况部署写死指定 #cluster.local. 为域名 wget https://raw.githubusercontent.com/coredns/deployment/master/kubernetes/coredns.yaml.sed wget https://raw.githubusercontent.com/coredns/deployment/master/kubernetes/deploy.sh chmod +x deploy.sh ./deploy.sh -i 10.254.0.2 -d cluster.local. > dns.yaml kubectl apply -f dns.yaml; ## 升级coredns curl -Ls -o coredns_1.5.0.yaml.sed https://raw.githubusercontent.com/coredns/deployment/master/kubernetes/coredns.yaml.sed ``` `//测试` ```bash # --service-cluster-ip-range (kube-apiserver) # --cluster-cidr (kube-controller-manager、kube-proxy） # --cluster-dns (kubelet） kubectl config view -o jsonpath="{.clusters[?(@.name == \"kubernetes\")].cluster.server}"; docker run -d -p 9000:9000 --name portainer --restart always -v /var/run/docker.sock:/var/run/docker.sock -v /data/portainer_data:/data portainer/portainer ##### kubectl get no,svc,ep,pod,deployment -o wide -n kube-system kubectl get no,svc,ep,pod,deployment -o wide -n default kubectl get svc,ep,po -n kube-system -l k8s-app=kube-dns -o wide --show-labels kubectl edit configmap coredns -n kube-system kubectl get ds,rs,sts,cm --all-namespaces kubectl get ev --all-namespaces kubectl get pod -n kube-system -o wide --show-labels ##### kubectl get svc,ep,po -n kube-system -l k8s-app=kubernetes-dashboard -o wide --show-labels kubectl logs $(kubectl get po -o jsonpath={.items[0].metadata.name} -l k8s-app=kubernetes-dashboard -n kube-system) -n kube-system kubectl logs $(kubectl get po --field-selector=status.phase=Running -o jsonpath={.items[0].metadata.name} -l app=helm -n kube-system) -n kube-system kubectl describe pod $(kubectl get po -o jsonpath={.items[0].metadata.name} -l k8s-app=kubernetes-dashboard -n kube-system) -n kube-system kubectl describe svc kubernetes-dashboard -n kube-system ##### kubectl logs $(kubectl get pod -o json -l k8s-app=kube-dns -n kube-system | jq -r '.items | .[0] | .metadata | .name') -n kube-system kubectl logs $(kubectl get pod -o jsonpath={.items[0].metadata.name} -l k8s-app=kube-dns -n kube-system) -n kube-system kubectl describe pod $(kubectl get pod -o jsonpath={.items[0].metadata.name} -l k8s-app=kube-dns -n kube-system) -n kube-system ######### kubectl get svc,ep,pod,cm -n ingress-nginx -l app.kubernetes.io/name=ingress-nginx -o wide --show-labels kubectl logs $(kubectl get pod -n ingress-nginx -l app.kubernetes.io/name=ingress-nginx -o jsonpath={.items[0].metadata.name}) -n ingress-nginx kubectl describe pod $(kubectl get pod -n ingress-nginx -l app.kubernetes.io/name=ingress-nginx -o jsonpath={.items[0].metadata.name}) -n ingress-nginx ## 查看coredns kubectl run -it --rm --generator=run-pod/v1 --image=tutum/dnsutils dns -- /bin/sh kubectl run -it --rm --generator=run-pod/v1 --image=nginx:1.15.11-alpine nginx1 -- /bin/sh kubectl run -it --rm --generator=run-pod/v1 --image=nginx:1.15.11-perl nginx2 -- /bin/sh kubectl run -it --rm --generator=run-pod/v1 --image=infoblox/dnstools dns-client ############ sudo ipvsadm -Ln -t 10.254.0.2:53 --stats sudo ipvsadm --list -n --tcp-service 10.254.0.2:53 sudo ipvsadm --list -n --tcp-service 10.254.0.1:443 ping 10.254.0.1 ping 10.254.0.2 ## apt-get install dnsutils inetutils-ping ## 查看53端口 nc -nvz 10.254.0.2 53 dig -t A -p 53 @10.254.0.2 kubernetes.default.svc.cluster.local. +short dig kubernetes.default.svc.cluster.local. -t A +short ## 执行命令 nslookup -type=A kubernetes ## 返回如下信息 Server: 10.254.0.2 Address: 10.254.0.2#53 Name: kubernetes.default.svc.cluster.local Address: 10.254.0.1 ``` # 部署metrics-server `//开启apiserver聚合层` ```bash ## 开启聚合层 ## https://kubernetes.io/docs/tasks/access-kubernetes-api/configure-aggregation-layer/#enable-kubernetes-apiserver-flags --requestheader-client-ca-file= --requestheader-allowed-names=front-proxy-client --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --proxy-client-cert-file= --proxy-client-key-file= ``` `//部署` ```bash mkdir metrics-server cd metrics-server # https://github.com/kubernetes-incubator/metrics-server/tree/master/deploy url=https://raw.githubusercontent.com/kubernetes-incubator/metrics-server/master/deploy/1.8%2B wget --no-check-certificate $url/aggregated-metrics-reader.yaml wget --no-check-certificate $url/auth-delegator.yaml wget --no-check-certificate $url/auth-reader.yaml wget --no-check-certificate $url/metrics-apiservice.yaml wget --no-check-certificate $url/metrics-server-deployment.yaml wget --no-check-certificate $url/metrics-server-service.yaml wget --no-check-certificate $url/resource-reader.yaml # k8s.gcr.io/metrics-server-amd64:v0.3.1 # 修改metrics-server-deployment.yaml，添加证书 # command: # - /metrics-server # - --metric-resolution=20s # - --requestheader-client-ca-file=/etc/kubernetes/pki/kubernetes-front-proxy-ca.pem # - --requestheader-allowed-names=aggregator # - --kubelet-insecure-tls # - --kubelet-preferred-address-types=InternalIP,Hostname,InternalDNS,ExternalDNS,ExternalIP # --metric-resolution=30s：从 kubelet 采集数据的周期； # --requestheader-allowed-names=aggregator：允许请求 metrics-server API 的用户名，该名称与 kube-apiserver 的 --proxy-client-cert-file 指定的证书 CN 一致； # --kubelet-preferred-address-types：优先使用 InternalIP 来访问 kubelet，这样可以避免节点名称没有 DNS 解析记录时，通过节点名称调用节点 kubelet API 失败的情况（未配置时默认的情况）； kubectl create -f . # https://github.com/kubernetes-incubator/metrics-server/issues/45 # W0312 01:45:37.102313 1 x509.go:172] x509: subject with cn=front-proxy-client is not in the allowed list: [aggregator] # E0312 01:45:37.102354 1 authentication.go:62] Unable to authenticate the request due to an error: [x509: subject with cn=front-proxy-client is not allowed, x509: certificate signed by unknown author # https://github.com/kubernetes-incubator/metrics-server/issues/58 # error: Metrics not available for pod default # https://github.com/kubernetes-incubator/metrics-server/issues/150 # unable to fetch metrics from Kubelet # https://github.com/kubernetes-incubator/metrics-server/issues/146 # E0311 16:43:38.416358 1 manager.go:102] unable to fully collect metrics: [unable to fully scrape metrics from source kubelet_summary:n22.dev: unable to fetch metrics from Kubelet n22.dev (192.168.33.22): Get https://192.168.33.22:10250/stats/summary/: x509: cannot validate certificate for 192.168.33.22 because it doesn't contain any IP SANs, unable to fully scrape metrics from source kubelet_summary:n26.dev: unable to fetch metrics from Kubelet n26.dev (192.168.33.26): Get https://192.168.33.26:10250/stats/summary/: x509: cannot validate certificate for 192.168.33.26 because it doesn't contain any IP SANs, unable to fully scrape metrics from source kubelet_summary:n23.dev: unable to fetch metrics from Kubelet n23.dev (192.168.33.23): Get https://192.168.33.23:10250/stats/summary/: x509: cannot validate certificate for 192.168.33.23 because it doesn't contain any IP SANs, unable to fully scrape metrics from source kubelet_summary:n21.dev: unable to fetch metrics from Kubelet n21.dev (192.168.33.21): Get https://192.168.33.21:10250/stats/summary/: x509: cannot validate certificate for 192.168.33.21 because it doesn't contain any IP SANs, unable to fully scrape metrics from source kubelet_summary:n27.dev: unable to fetch metrics from Kubelet n27.dev (192.168.33.27): Get https://192.168.33.27:10250/stats/summary/: x509: cannot validate certificate for 192.168.33.27 because it doesn't contain any IP SANs] # Error from server (Forbidden): nodes.metrics.k8s.io is forbidden: User “system:kube-proxy” cannot list nodes.metrics.k8s.io at the cluster scope # Forbidden (user=system:anonymous, verb=get, resource=nodes, subresource=proxy) kubectl apply -f https://raw.githubusercontent.com/crosscloudci/cross-cloud/master/rbac/apiserver-to-kubelet-rbac.yml kubectl create clusterrolebinding system:anonymous --clusterrole=cluster-admin --user=system:anonymous kubectl create clusterrolebinding kubelet-admin --clusterrole=system:kubelet-api-admin --user=kubelet-client openssl x509 -in /var/lib/kubelet/pki/kubelet-client-current.pem -noout -text -certopt no_version,no_pubkey,no_sigdump -nameopt multiline ``` ```yaml --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: view-metrics rules: - apiGroups: - metrics.k8s.io resources: - pods - nodes verbs: - get - list - watch --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: view-metrics roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: view-metrics subjects: - apiGroup: rbac.authorization.k8s.io kind: User name: system:kube-proxy --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: anonymous-logs-role rules: - apiGroups: [""] resources: ["nodes/proxy"] verbs: ["create", "get"] --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: anonymous-logs-binding roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: anonymous-logs-role subjects: - apiGroup: rbac.authorization.k8s.io kind: User name: system:anonymous ``` ```bash kubectl get apiservice v1beta1.metrics.k8s.io -o json kubectl get pods -n kube-system -l k8s-app=metrics-server -o wide kubectl get --raw "/apis/metrics.k8s.io" # 查看node指标 kubectl get --raw "/apis/metrics.k8s.io/v1beta1/nodes" # 查看pod指标 kubectl get --raw "/apis/metrics.k8s.io/v1beta1/pods" kubectl get --raw "/apis/metrics.k8s.io/v1beta1" | jq . kubectl logs $(kubectl get po -o json -n kube-system -l k8s-app=metrics-server | jq -r '.items | .[] | .metadata | .name') -n kube-system # kubectl get --raw "/apis/metrics.k8s.io/v1beta1" | jq . # error: You must be logged in to the server (Unauthorized) # front-proxy-ca.pem证书问题,--requestheader-client-ca-file参数 # error: metrics not available yet # 刚搭建完会没有数据，过几分钟就能收集到数据了 # '/sys/fs/cgroup/cpuset': operation not permitted # Could not configure a source for OOM detection, disabling OOM events: open /dev/kmsg: no such file or directory ``` `//测试` ```bash kubectl top node kubectl top pod ``` # 部署dashboard `//部署` ```bash # [Release](https://github.com/kubernetes/dashboard) # [Dashboard arguments](https://github.com/kubernetes/dashboard/wiki/Dashboard-arguments) kubectl apply -f https://raw.githubusercontent.com/kubernetes/dashboard/v1.10.1/src/deploy/recommended/kubernetes-dashboard.yaml ``` `//创建账户` ```yaml --- apiVersion: v1 kind: ServiceAccount metadata: name: admin-user namespace: kube-system --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: admin-user roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: cluster-admin subjects: - kind: ServiceAccount name: admin-user namespace: kube-system --- kind: Service apiVersion: v1 metadata: labels: k8s-app: kubernetes-dashboard name: kubernetes-dashboard2 namespace: kube-system spec: type: NodePort ports: - port: 443 targetPort: 8443 nodePort: 30014 selector: k8s-app: kubernetes-dashboard ``` ```bash ## 获取登录token #kubectl create sa dashboard-admin -n kube-system #kubectl create clusterrolebinding dashboard-admin --clusterrole=cluster-admin --serviceaccount=kube-system:dashboard-admin kubectl -n kube-system describe secret $(kubectl -n kube-system get secret | grep admin-user | awk '{print $1}') | grep -E '^token' | awk '{print $2}' ``` # 部署ingress-nginx `//部署` ```bash #kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/master/deploy/mandatory.yaml #kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/master/deploy/static/mandatory.yaml kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/master/deploy/static/provider/baremetal/deploy.yaml ## service-nodeport.yaml #kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/master/deploy/baremetal/service-nodeport.yaml ``` ```bash ## pod固定 hostNetwork: true dnsPolicy: ClusterFirstWithHostNet ``` ```yaml apiVersion: v1 kind: Service metadata: name: ingress-nginx namespace: ingress-nginx labels: app.kubernetes.io/name: ingress-nginx app.kubernetes.io/part-of: ingress-nginx spec: type: NodePort ports: - name: http port: 80 targetPort: 80 protocol: TCP - name: https port: 443 targetPort: 443 protocol: TCP - name: proxied-tcp-3306 port: 3306 targetPort: 3306 protocol: TCP nodePort: 30016 selector: app.kubernetes.io/name: ingress-nginx app.kubernetes.io/part-of: ingress-nginx ``` `//L4转发` ```yaml kind: ConfigMap apiVersion: v1 metadata: name: tcp-services namespace: ingress-nginx data: 3306: "default/mysql-svc:3306" ``` `//L7转发` ```yaml apiVersion: extensions/v1beta1 kind: Ingress metadata: name: my-ingress annotations: nginx.ingress.kubernetes.io/use-regex: "true" spec: rules: - host: demo.mydomain.com http: paths: - backend: serviceName: api servicePort: 80 ``` `//测试` ```bash curl -H "host:demo.mydomain.com" http://192.168.33.27:31025 ``` # 部署traefik-ingress `//部署` ```bash # https://github.com/containous/traefik/tree/v1.7/examples/k8s kubectl apply -f https://raw.githubusercontent.com/containous/traefik/v1.7/examples/k8s/traefik-rbac.yaml # examples/k8s/traefik-deployment.yaml kubectl apply -f https://raw.githubusercontent.com/containous/traefik/v1.7/examples/k8s/traefik-deployment.yaml # examples/k8s/traefik-ds.yaml #kubectl apply -f https://raw.githubusercontent.com/containous/traefik/v1.7/examples/k8s/traefik-ds.yaml kubectl --namespace=kube-system get pod,svc -o wide # examples/k8s/ui.yaml kubectl apply -f https://raw.githubusercontent.com/containous/traefik/v1.7/examples/k8s/ui.yaml ``` `//Ingress` ```toml --- apiVersion: v1 kind: Service metadata: name: traefik-web-ui namespace: kube-system spec: selector: k8s-app: traefik-ingress-lb ports: - name: web port: 80 targetPort: 8080 --- apiVersion: extensions/v1beta1 kind: Ingress metadata: name: traefik-web-ui namespace: kube-system annotations: kubernetes.io/ingress.class: traefik spec: rules: - host: traefik-ui.minikube http: paths: - path: / backend: serviceName: traefik-web-ui servicePort: 80 ``` `//测试` ```bash curl -fsSL http://$(kubectl get svc traefik-web-ui -n kube-system -o jsonpath='{.spec.clusterIP}'):80/api/providers |jq ``` # 部署helm ```bash # 下载 # curl -sSLk https://get.helm.sh/helm-v3.1.1-linux-amd64.tar.gz curl -sSL https://get.helm.sh/helm-v2.16.3-linux-amd64.tar.gz | tar -xvz -C /tmp/ sudo cp /tmp/linux-amd64/helm /usr/local/bin/ # 初始化SA kubectl create serviceaccount --namespace kube-system tiller kubectl create clusterrolebinding tiller-cluster-rule --clusterrole=cluster-admin --serviceaccount=kube-system:tiller # 部署tiller helm init --service-account tiller --upgrade --tiller-image jessestuart/tiller:v2.16.5 --stable-repo-url https://mirror.azure.cn/kubernetes/charts/ # --tiller-image gcr.io/kubernetes-helm/tiller:v2.16.3 \ # --tiller-tls-cert /etc/kubernetes/ssl/tiller001.pem \ # --tiller-tls-key /etc/kubernetes/ssl/tiller001-key.pem \ # --tls-ca-cert /etc/kubernetes/ssl/ca.pem \ # --tiller-namespace kube-system # helm换源 helm repo remove stable helm repo add stable https://mirror.azure.cn/kubernetes/charts/ helm repo update # 升级tiller kubectl -n kube-system set image deployment/tiller-deploy tiller=jessestuart/tiller:v2.16.5 ``` ```bash Creating /home/vagrant/.helm Creating /home/vagrant/.helm/repository Creating /home/vagrant/.helm/repository/cache Creating /home/vagrant/.helm/repository/local Creating /home/vagrant/.helm/plugins Creating /home/vagrant/.helm/starters Creating /home/vagrant/.helm/cache/archive Creating /home/vagrant/.helm/repository/repositories.yaml Adding stable repo with URL: https://kubernetes-charts.storage.googleapis.com Adding local repo with URL: http://127.0.0.1:8879/charts $HELM_HOME has been configured at /home/vagrant/.helm. Tiller (the Helm server-side component) has been installed into your Kubernetes Cluster. Please note: by default, Tiller is deployed with an insecure 'allow unauthenticated users' policy. To prevent this, run `helm init` with the --tiller-tls-verify flag. For more information on securing your installation see: https://docs.helm.sh/using_helm/#securing-your-helm-installation ``` ```bash kubectl get deploy -n kube-system tiller-deploy --output yaml|grep serviceAccount kubectl describe deploy -n kube-system tiller-deploy kubectl get pod -n kube-system -o wide -l app=helm kubectl -n kube-system get pods|grep tiller ## helm error installing the server could not find the requested resource ## gcr.io/kubernetes-helm/tiller:v2.16.1 helm init --debug --dry-run \ --service-account tiller \ --tiller-namespace kube-system \ --upgrade \ --tiller-image jessestuart/tiller:v2.16.5 \ --stable-repo-url https://mirror.azure.cn/kubernetes/charts/ \ |sed -e "s/apiVersion: extensions\/v1beta1/apiVersion: apps\/v1/g" \ |sed '/^\./d' \ |sed -e "/replicas: 1/a \ selector:\n\ matchLabels:\n\ app: helm\n\ name: tiller" \ |kubectl apply -f - ## #https://github.com/helm/helm/issues/6374 helm init --service-account tiller --override spec.selector.matchLabels.'name'='tiller',spec.selector.matchLabels.'app'='helm' --output yaml | sed 's@apiVersion: extensions/v1beta1@apiVersion: apps/v1@' | kubectl apply -f - ## ## 查看tiller kubectl get deployments,pod -n kube-system -o wide -l app=helm -l name=tiller # 添加fabric8库 helm repo add fabric8 https://fabric8.io/helm helm search prometheus helm fetch stable/prometheus helm search jenkins helm install --name my-release --set Persistence.StorageClass=slow stable/jenkins helm get my-release helm install stable/nginx-ingress --set controller.hostNetwork=true，rbac.create=true helm repo add incubator http://storage.googleapis.com/kubernetes-charts-incubator helm install incubator/sparkoperator --namespace spark-operator # 更换仓库 helm repo remove stable helm repo add stable https://mirror.azure.cn/kubernetes/charts/ helm repo update helm repo list helm search fluentd # 产生yaml helm install stable/fluentd --version 2.0.0 --dry-run --debug |sed -n '/^---/,$p' > fluentd.yaml ``` # 部署operators //TODO # 部署glusterfs集群 `//准备机器` ```bash 192.168.33.50 安装heketi服务器 192.168.33.55 glusterfs集群server 192.168.33.56 glusterfs集群server 192.168.33.57 glusterfs集群server # 配置 /etc/hosts 192.168.33.50 n50.dev 192.168.33.55 n55.dev 192.168.33.56 n56.dev 192.168.33.57 n57.dev ``` `//格式化磁盘` ```bash # 格式化磁盘 # 查看CentOS 7支持的文件系统格式 cat /etc/filesystems # 查看mount查看分区的文件系统格式 mount # 查看磁盘 fdisk -l # 挂载磁盘 # /dev/sda # /dev/sdb # 格式化磁盘并挂载到/data/brick1 mkfs.xfs -i size=512 /dev/sdb mkdir -p /data/brick1 echo '/dev/sdb /data/brick1 xfs defaults 1 2' >> /etc/fstab mount -a && mount ################### # 查看挂载情况 df |grep /data/brick1 # 参考 # https://docs.gluster.org/en/latest/Quick-Start-Guide/Quickstart/ # https://kubernetes.feisky.xyz/cha-jian-kuo-zhan/volume/glusterfs # https://www.linuxidc.com/Linux/2017-02/140517.htm # https://github.com/gluster/gluster-kubernetes/blob/master/docs/setup-guide.md ``` `//编译安装` ```bash # 编译源代码 # https://docs.gluster.org/en/latest/Developer-guide/Building-GlusterFS/#building-from-source sudo yum remove epel-release-7-11.noarch && sudo yum install -y epel-release sudo yum install centos-release-gluster -y sudo yum clean all && sudo yum makecache sudo yum install autoconf automake bison cmockery2-devel dos2unix flex fuse-devel glib2-devel libacl-devel libaio-devel libattr-devel libcurl-devel libibverbs-devel librdmacm-devel libtirpc-devel libtool libxml2-devel lvm2-devel make openssl-devel pkgconfig pyliblzma python-devel python-eventlet python-netifaces python-paste-deploy python-simplejson python-sphinx python-webob pyxattr readline-devel rpm-build sqlite-devel systemtap-sdt-devel tar userspace-rcu-devel ## 下载源代码 wget https://github.com/gluster/glusterfs/archive/v6.5.tar.gz tar -xvf v6.5.tar.gz cd glusterfs-6.5 ./autogen.sh ./configure --sysconfdir=/etc --without-libtirpc --enable-gnfs ################################ GlusterFS configure summary =========================== FUSE client : yes Infiniband verbs : yes epoll IO multiplex : yes fusermount : yes readline : yes georeplication : yes Linux-AIO : yes Enable Debug : no Enable ASAN : no Enable TSAN : no Use syslog : yes XML output : yes Unit Tests : no Track priv ports : yes POSIX ACLs : yes SELinux features : yes firewalld-config : no Events : yes EC dynamic support : x64 sse avx Use memory pools : yes Nanosecond m/atimes : yes Server components : yes Legacy gNFS server : yes IPV6 default : no Use TIRPC : no With Python : 2.7 Cloudsync : yes ################################ make sudo make install ############# ls /usr/local/sbin/ |grep gluster && ls /usr/local/bin/ |grep gluster ls -l /etc/glusterfs/ cat /usr/local/lib/systemd/system/glusterd.service # 修改 glusterd 目录 sed -i 's/var\/lib/opt/g' /etc/glusterfs/glusterd.vol sudo yum install -y rpcbind attr psmisc # 设置 glusterd.service 开机启动 sudo systemctl enable glusterd.service # 启动 glusterd.service sudo systemctl start glusterd.service # 查看 glusterd.service systemctl status glusterd.service ``` ```bash ● glusterd.service - GlusterFS, a clustered file-system server Loaded: loaded (/usr/local/lib/systemd/system/glusterd.service; enabled; vendor preset: disabled) Active: active (running) since 五 2019-08-09 21:58:51 CST; 7s ago Docs: man:glusterd(8) Process: 3717 ExecStart=/usr/local/sbin/glusterd -p /var/run/glusterd.pid --log-level $LOG_LEVEL $GLUSTERD_OPTIONS (code=exited, status=0/SUCCESS) Main PID: 3718 (glusterd) Tasks: 9 Memory: 10.2M CGroup: /system.slice/glusterd.service └─3718 /usr/local/sbin/glusterd -p /var/run/glusterd.pid --log-level INFO 8月 09 21:58:50 n55 systemd[1]: Starting GlusterFS, a clustered file-system server... 8月 09 21:58:51 n55 systemd[1]: Started GlusterFS, a clustered file-system server. ``` `//创建测试卷` ```bash # 加载内核模块 sudo modprobe dm_snapshot sudo modprobe dm_thin_pool sudo modprobe dm_mirror # 确认模块已加载 lsmod |egrep "dm_snapshot|dm_mirror|dm_thin_pool" # 分区、格式化 vgcreate v_gluster /dev/sdb lvcreate -n lv_gluster -L 2G vg_gluster mkfs.ext4 /dev/v_gluster/lv_gluster # 添加节点到集群 sudo /usr/local/sbin/gluster peer probe n56.dev sudo /usr/local/sbin/gluster peer probe n57.dev # 查看集群状态 sudo /usr/local/sbin/gluster peer status # 创建test-volume sudo /usr/local/sbin/gluster volume create test-volume transport rdma \ n55.dev:/data/brick1/gv0 \ n56.dev:/data/brick1/gv0 \ n57.dev:/data/brick1/gv0 # volume create: test-volume: failed: The brick n55.dev:/data/brick1 is a mount point. Please create a sub-directory under the mount point and use that as the brick directory. Or use 'force' at the end of the command if you want to override this behavior. # volume create: test-volume: failed: The brick n55.dev:/opt/gfs_data is being created in the root partition. It is recommended that you don't use the system's root partition for storage backend. Or use 'force' at the end of the command if you want to override this behavior. sudo /usr/local/sbin/gluster volume start test-volume # volume start: test-volume: failed: Commit failed on localhost. Please check log file for details. # 好像不支持 rdma 协议 sudo /usr/local/sbin/gluster volume set test-volume config.transport tcp sudo /usr/local/sbin/gluster volume info test-volume Volume Name: test-volume Type: Distribute Volume ID: 5cf6bcd6-d2b3-495e-aa6c-601d54c23707 Status: Created Snapshot Count: 0 Number of Bricks: 3 Transport-type: rdma Bricks: Brick1: n55.dev:/data/brick1/gv0 Brick2: n56.dev:/data/brick1/gv0 Brick3: n57.dev:/data/brick1/gv0 Options Reconfigured: nfs.disable: on sudo /usr/local/sbin/gluster volume status test-volume Volume test-volume is not started sudo /usr/local/sbin/gluster volume status test-volume Status of volume: test-volume Gluster process TCP Port RDMA Port Online Pid ------------------------------------------------------------------------------ Brick n55.dev:/data/brick1/gv0 49153 0 Y 4086 Brick n56.dev:/data/brick1/gv0 49152 0 Y 7782 Brick n57.dev:/data/brick1/gv0 49152 0 Y 6744 Task Status of Volume test-volume ------------------------------------------------------------------------------ There are no active volume tasks # 查询端口 sudo netstat -tunlp |grep gluster tcp 0 0 0.0.0.0:49153 0.0.0.0:* LISTEN 4086/glusterfsd tcp 0 0 0.0.0.0:24007 0.0.0.0:* LISTEN 3718/glusterd # Unmount the volume on all the clients using the following command: umount mount-point # Stop the volumes using the following command: gluster volume stop test-volume # Mount the volume on all the clients. mount -t glusterfs -o transport=rdma server1:/test-volume /mnt/glusterfs # Change the transport type. gluster volume set test-volume config.transport tcp,rdma ######################### # 默认模式，既DHT, 也叫 分布卷: 将文件已hash算法随机分布到 一台服务器节点中存储。 gluster volume create test-volume server1:/exp1 server2:/exp2 # 复制模式，既AFR, 创建volume 时带 replica x 数量: 将文件复制到 replica x 个节点中。 gluster volume create test-volume replica 2 transport tcp server1:/exp1 server2:/exp2 # 条带模式，既Striped, 创建volume 时带 stripe x 数量： 将文件切割成数据块，分别存储到 stripe x 个节点中 ( 类似raid 0 )。 gluster volume create test-volume stripe 2 transport tcp server1:/exp1 server2:/exp2 # 分布式条带模式（组合型），最少需要4台服务器才能创建。 创建volume 时 stripe 2 server = 4 个节点： 是DHT 与 Striped 的组合型。 gluster volume create test-volume stripe 2 transport tcp server1:/exp1 server2:/exp2 server3:/exp3 server4:/exp4 # 分布式复制模式（组合型）, 最少需要4台服务器才能创建。 创建volume 时 replica 2 server = 4 个节点：是DHT 与 AFR 的组合型。 gluster volume create test-volume replica 2 transport tcp server1:/exp1 server2:/exp2　server3:/exp3 server4:/exp4 # 条带复制卷模式（组合型）, 最少需要4台服务器才能创建。 创建volume 时 stripe 2 replica 2 server = 4 个节点： 是 Striped 与 AFR 的组合型。 gluster volume create test-volume stripe 2 replica 2 transport tcp server1:/exp1 server2:/exp2 server3:/exp3 server4:/exp4 # 三种模式混合, 至少需要8台 服务器才能创建。 stripe 2 replica 2 , 每4个节点 组成一个 组。 gluster volume create test-volume stripe 2 replica 2 transport tcp server1:/exp1 server2:/exp2 server3:/exp3 server4:/exp4 server5:/exp5 server6:/exp6 server7:/exp7 server8:/exp8 # 验证可信存储池的状态 gluster peer status gluster pool list ``` `//部署heketi（物理器二进制安装）` ```bash wget https://github.com/heketi/heketi/releases/download/v9.0.0/heketi-v9.0.0.linux.amd64.tar.gz tar -xvf heketi-v9.0.0.linux.amd64.tar.gz cd heketi; sudo mv heketi-cli /usr/local/bin sudo mv heketi /usr/local/bin heketi-cli --version heketi-cli v9.0.0 sudo mkdir -p /etc/heketi/ sudo mkdir -p /var/lib/heketi ssh-keygen -f /etc/heketi/heketi_key # ssh-keygen -t rsa -q -f /etc/heketi/heketi_key -N '' chmod 600 /etc/heketi/heketi_key.pub ssh-copy-id -i /etc/heketi/heketi_key.pub vagrant@n55.dev ssh-copy-id -i /etc/heketi/heketi_key.pub vagrant@n56.dev ssh-copy-id -i /etc/heketi/heketi_key.pub vagrant@n57.dev ssh vagrant@57.dev -i /etc/heketi/heketi_key wget https://raw.githubusercontent.com/heketi/heketi/master/extras/systemd/heketi.env # https://github.com/heketi/heketi/blob/master/extras/systemd/heketi.service cd /lib/systemd/system/ sudo wget https://raw.githubusercontent.com/heketi/heketi/master/extras/systemd/heketi.service sudo ln -s /usr/local/bin/heketi /usr/bin/heketi sudo ln -s /usr/local/bin/heketi-cli /usr/bin/heketi-cli heketi-cli -s http://localhost:7070 --user admin --secret '' topology load --json=/shell/deploy/glusterfs/topology.json Creating cluster ... ID: 606bbee818c0e23c8d09d793824c23ba Allowing file volumes on cluster. Allowing block volumes on cluster. Creating node n55.dev ... ID: 9121797acba88cdff782253669041503 Adding device /dev/sdb ... Unable to add device: Setup of device /dev/sdb failed (already initialized or contains data?): /bin/bash: pvcreate: command not found Creating node n56.dev ... Unable to create node: /bin/bash: gluster: command not found Creating node n57.dev ... Unable to create node: /bin/bash: gluster: command not found heketi-cli -s http://localhost:7070 --user admin --secret '' topology load --json=/shell/deploy/glusterfs/topology.json Found node n55.dev on cluster 606bbee818c0e23c8d09d793824c23ba Adding device /dev/sdb ... Unable to add device: Setup of device /dev/sdb failed (already initialized or contains data?): Can't open /dev/sdb exclusively. Mounted filesystem? Can't open /dev/sdb exclusively. Mounted filesystem? Creating node n56.dev ... Unable to create node: sudo: gluster: command not found Creating node n57.dev ... Unable to create node: sudo: gluster: command not found sudo vi /etc/sudoers #Defaults secure_path = /sbin:/bin:/usr/sbin:/usr/bin Defaults secure_path = /sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin [vagrant@n55 ~]$ sudo /usr/local/sbin/gluster volume start test-volume volume start: test-volume: failed: Volume test-volume already started [vagrant@n55 ~]$ sudo /usr/local/sbin/gluster volume stop test-volume Stopping volume will make its data inaccessible. Do you want to continue? (y/n) y volume stop: test-volume: success [vagrant@n55 ~]$ sudo /usr/local/sbin/gluster volume delete test-volume Deleting volume will erase all information about the volume. Do you want to continue? (y/n) y volume delete: test-volume: success sudo umount /data/brick1/ Found node n55.dev on cluster 606bbee818c0e23c8d09d793824c23ba Adding device /dev/sdb ... Unable to add device: Setup of device /dev/sdb failed (already initialized or contains data?): Can't open /dev/sdb exclusively. Mounted filesystem? Can't open /dev/sdb exclusively. Mounted filesystem? Found node n56.dev on cluster 606bbee818c0e23c8d09d793824c23ba Adding device /dev/sdb ... Unable to add device: Setup of device /dev/sdb failed (already initialized or contains data?): Can't open /dev/sdb exclusively. Mounted filesystem? Can't open /dev/sdb exclusively. Mounted filesystem? Found node n57.dev on cluster 606bbee818c0e23c8d09d793824c23ba Adding device /dev/sdb ... Unable to add device: Setup of device /dev/sdb failed (already initialized or contains data?): Can't open /dev/sdb exclusively. Mounted filesystem? Can't open /dev/sdb exclusively. Mounted filesystem? [vagrant@n50 heketi]$ heketi-cli -s http://localhost:7070 --user admin --secret '' topology load --json=/shell/deploy/glusterfs/topology.json Found node n55.dev on cluster 606bbee818c0e23c8d09d793824c23ba Adding device /dev/sdb ... Unable to add device: Setup of device /dev/sdb failed (already initialized or contains data?): WARNING: xfs signature detected on /dev/sdb at offset 0. Wipe it? [y/n]: [n] Aborted wiping of xfs. 1 existing signature left on the device. Found node n56.dev on cluster 606bbee818c0e23c8d09d793824c23ba Adding device /dev/sdb ... Unable to add device: Setup of device /dev/sdb failed (already initialized or contains data?): WARNING: xfs signature detected on /dev/sdb at offset 0. Wipe it? [y/n]: [n] Aborted wiping of xfs. 1 existing signature left on the device. Found node n57.dev on cluster 606bbee818c0e23c8d09d793824c23ba Adding device /dev/sdb ... Unable to add device: Setup of device /dev/sdb failed (already initialized or contains data?): WARNING: xfs signature detected on /dev/sdb at offset 0. Wipe it? [y/n]: [n] Aborted wiping of xfs. 1 existing signature left on the device. # Heketi需要空块设备，如果您已经在这些块设备上创建了lvm pvs或文件系统或分区等，则无法使用。如果您知道这些设备未使用，您可以在尝试初始化heketi之前使用像wipefs -a这样的命令来清除它们。 sudo wipefs -a /dev/sdb heketi-cli -s http://localhost:7070 --user admin --secret '' topology load --json=/shell/deploy/glusterfs/topology.json Found node n55.dev on cluster 606bbee818c0e23c8d09d793824c23ba Adding device /dev/sdb ... OK Found node n56.dev on cluster 606bbee818c0e23c8d09d793824c23ba Adding device /dev/sdb ... OK Found node n57.dev on cluster 606bbee818c0e23c8d09d793824c23ba Adding device /dev/sdb ... OK heketi-cli -s http://localhost:7070 --user admin --secret '' topology info Cluster Id: 606bbee818c0e23c8d09d793824c23ba File: true Block: true Volumes: Nodes: Node Id: 2315d10e5bf974a1c9069a7181e3b8a3 State: online Cluster Id: 606bbee818c0e23c8d09d793824c23ba Zone: 1 Management Hostnames: n56.dev Storage Hostnames: 192.168.33.56 Devices: Id:ff84b184c51e3ef7f86b38c387bb9ed5 Name:/dev/sdb State:online Size (GiB):249 Used (GiB):0 Free (GiB):249 Bricks: Node Id: 9121797acba88cdff782253669041503 State: online Cluster Id: 606bbee818c0e23c8d09d793824c23ba Zone: 1 Management Hostnames: n55.dev Storage Hostnames: 192.168.33.55 Devices: Id:67c05c37402ba2eaf85223966401cf05 Name:/dev/sdb State:online Size (GiB):249 Used (GiB):0 Free (GiB):249 Bricks: Node Id: ece7a447bd1987ad6fd862db418dfcef State: online Cluster Id: 606bbee818c0e23c8d09d793824c23ba Zone: 1 Management Hostnames: n57.dev Storage Hostnames: 192.168.33.57 Devices: Id:5d312fe9a5ab47c0b2384d5be5053eaf Name:/dev/sdb State:online Size (GiB):249 Used (GiB):0 Free (GiB):249 Bricks: # 创建一个大小为3G，副本为2的volume heketi-cli -s http://localhost:7070 --user admin --secret '' volume create --size 3 --replica 2 Name: vol_58ebc1e318fdabbf3c905cc9ffd2f6f8 Size: 3 Volume Id: 58ebc1e318fdabbf3c905cc9ffd2f6f8 Cluster Id: 606bbee818c0e23c8d09d793824c23ba Mount: 192.168.33.56:vol_58ebc1e318fdabbf3c905cc9ffd2f6f8 Mount Options: backup-volfile-servers=192.168.33.55,192.168.33.57 Block: false Free Size: 0 Reserved Size: 0 Block Hosting Restriction: (none) Block Volumes: [] Durability Type: replicate Distribute Count: 1 Replica Count: 2 ``` `//部署heketi（k8s方式）` ```bash wget https://raw.githubusercontent.com/gluster/gluster-kubernetes/master/deploy/kube-templates/heketi-service-account.yaml wget https://raw.githubusercontent.com/gluster/gluster-kubernetes/master/deploy/kube-templates/deploy-heketi-deployment.yaml wget https://raw.githubusercontent.com/gluster/gluster-kubernetes/master/docs/examples/hello_world/gluster-storage-class.yaml diff deploy-heketi-deployment.yaml deploy-heketi-deployment1.yaml 11a12 > clusterIP: 10.254.8.2 36a38 > nodeName: n26.dev 77a80,81 > hostPath: > path: /data/heketi-data diff gluster-storage-class.yaml gluster-storage-class1.yaml 1c1 < apiVersion: storage.k8s.io/v1beta1 --- > apiVersion: storage.k8s.io/v1 8,10c8,11 < resturl: "http://10.42.0.0:8080" < restuser: "joe" < restuserkey: "My Secret Life" --- > resturl: "http://10.254.8.2:8080" > restauthenabled: "false" > #restuser: "joe" > #restuserkey: "My Secret Life" ## 创建资源 curl -L -o heketi.json https://raw.githubusercontent.com/gluster/gluster-kubernetes/master/deploy/heketi.json.template # curl -L -o heketi.json https://raw.githubusercontent.com/heketi/heketi/master/etc/heketi.json curl -L -o topology.json https://raw.githubusercontent.com/gluster/gluster-kubernetes/master/deploy/topology.json.sample kubectl -n default create secret generic heketi-config-secret --from-file=private_key=/dev/null --from-file=./heketi.json --from-file=topology.json=topology.json sed \ -e 's/\${HEKETI_EXECUTOR}/kubernetes/' \ -e 's/\${SSH_PORT}/22/' \ -e 's/\${SSH_USER}/vagrant/' \ -e 's/\${SSH_SUDO}/true/' \ ./heketi.json sed \ -e 's/\${HEKETI_EXECUTOR}/kubernetes/' \ -e 's#\${HEKETI_FSTAB}#/var/lib/heketi/fstab#' \ -e 's/\${HEKETI_ADMIN_KEY}//' \ -e 's/\${HEKETI_USER_KEY}//' ./deploy-heketi-deployment1.yaml | kubectl -n default create -f - kubectl apply -f heketi-service-account.yaml kubectl apply -f deploy-heketi-deployment1.yaml kubectl apply -f gluster-storage-class1.yaml # ERROR: Unable to parse configuration: json: cannot unmarshal string into Go struct field SshConfig.xfs_sw of type int kubectl logs $(kubectl get pod -l glusterfs=heketi-pod -o jsonpath={.items[0].metadata.name}) kubectl -n default exec -i $(kubectl get pod -l glusterfs=heketi-pod -o jsonpath={.items[0].metadata.name}) -- heketi-cli -s http://localhost:8080 --user admin --secret '' topology load --json=/etc/heketi/topology.json kubectl -n default exec -i $(kubectl get pod -l glusterfs=heketi-pod -o jsonpath={.items[0].metadata.name}) cat /etc/heketi/topology.json kubectl -n default exec -i $(kubectl get pod -l glusterfs=heketi-pod -o jsonpath={.items[0].metadata.name}) -- heketi-cli --server http://localhost:8080 --user admin --secret "" topology load --json=/etc/heketi/topology.json kubectl -n default delete secret heketi-config-secret && \ kubectl -n default create secret generic heketi-config-secret --from-file=private_key=/dev/null --from-file=heketi.json=heketi2.json --from-file=topology.json=topology.json && \ kubectl delete -f deploy-heketi-deployment1.yaml && \ sed \ -e 's/\${HEKETI_EXECUTOR}/kubernetes/' \ -e 's#\${HEKETI_FSTAB}#/var/lib/heketi/fstab#' \ -e 's/\${HEKETI_ADMIN_KEY}//' \ -e 's/\${HEKETI_USER_KEY}//' ./deploy-heketi-deployment1.yaml | kubectl -n default create -f - # https://blog.fleeto.us/post/glusterfs-and-heketi-on-physical-server/ # https://www.jianshu.com/p/f70d370582ef # https://fkpwolf.net/2018/09/11/glusterfs.html # https://lichi6174.github.io/glusterfs-heketi ``` `//pvc测试` `cat gluster-storage-class1.yaml` ```yaml apiVersion: storage.k8s.io/v1 kind: StorageClass metadata: name: gluster-heketi provisioner: kubernetes.io/glusterfs parameters: #endpoint: "heketi-storage-endpoints" resturl: "http://192.168.33.50:7070" restauthenabled: "false" volumetype: "replicate:2" #restuser: "joe" #restuserkey: "My Secret Life" ``` `cat gluster-pvc.yaml` ```yaml apiVersion: v1 kind: PersistentVolumeClaim metadata: name: gluster1 annotations: volume.beta.kubernetes.io/storage-class: gluster-heketi spec: accessModes: - ReadWriteMany resources: requests: storage: 2Gi ``` `cat nginx-pod.yaml` ```yaml apiVersion: v1 kind: Pod metadata: name: gluster-pod1 labels: name: gluster-pod1 spec: containers: - name: gluster-pod1 image: nginx:1.17.2-alpine ports: - name: web containerPort: 80 securityContext: privileged: true volumeMounts: - name: gluster-vol1 mountPath: /usr/share/nginx/html volumes: - name: gluster-vol1 persistentVolumeClaim: claimName: gluster1 ``` ```bash ## 查看 kubectl get pv,pvc,ep NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE persistentvolume/pvc-17535729-c745-11e9-837f-08002763a99f 2Gi RWX Delete Bound default/gluster1 gluster-heketi 2m NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE persistentvolumeclaim/gluster1 Bound pvc-17535729-c745-11e9-837f-08002763a99f 2Gi RWX gluster-heketi 2m20s NAME ENDPOINTS AGE endpoints/glusterfs-dynamic-17535729-c745-11e9-837f-08002763a99f 192.168.33.55:1,192.168.33.56:1,192.168.33.57:1 2m1s ## 一些错误 Mounting command: systemd-run Mounting arguments: --description=Kubernetes transient mount for /var/lib/kubelet/pods/064e4421-bc43-11e9-9644-08002763a99f/volumes/kubernetes.io~glusterfs/pvc-0649fb80-bc43-11e9-9644-08002763a99f --scope -- mount -t glusterfs -o log-file=/var/lib/kubelet/plugins/kubernetes.io/glusterfs/pvc-0649fb80-bc43-11e9-9644-08002763a99f/gluster-pod1-glusterfs.log,log-level=ERROR,backup-volfile-servers=192.168.33.55:192.168.33.56:192.168.33.57,auto_unmount 192.168.33.55:vol_fe3d76a59b5805d057eda0f2f24d4a66 /var/lib/kubelet/pods/064e4421-bc43-11e9-9644-08002763a99f/volumes/kubernetes.io~glusterfs/pvc-0649fb80-bc43-11e9-9644-08002763a99f Output: Running scope as unit run-15406.scope. mount: unknown filesystem type 'glusterfs' Warning FailedMount 2m27s (x25 over 2m35s) kubelet, n27.dev MountVolume.NewMounter initialization failed for volume "pvc-10c304b0-bc48-11e9-9644-08002763a99f" : endpoints "glusterfs-dynamic-gluster1" not found # 安装glusterfs-fuse包 the following error information was pulled from the glusterfs log to help diagnose this issue: could not open log file for pod gluster-pod1 Warning FailedMount 2s (x7 over 15m) kubelet, n27.dev Unable to mount volumes for pod "gluster-pod1_default(064e4421-bc43-11e9-9644-08002763a99f)": timeout expired waiting for volumes to attach or mount for pod "default"/"gluster-pod1". list of unmounted volumes=[gluster-vol1]. list of unattached volumes=[gluster-vol1 default-token-4vjc4] the following error information was pulled from the glusterfs log to help diagnose this issue: [2019-08-11 14:41:42.236008] E [MSGID: 100026] [glusterfsd.c:2351:glusterfs_process_volfp] 0-: failed to construct the graph [2019-08-11 14:41:42.236212] E [graph.c:1142:glusterfs_graph_destroy] (-->/usr/sbin/glusterfs(mgmt_getspec_cbk+0x532) [0x559227d11332] -->/usr/sbin/glusterfs(glusterfs_process_volfp+0x150) [0x559227d0ae60] -->/lib64/libglusterfs.so.0(glusterfs_graph_destroy+0x84) [0x7f2dedacb1e4] ) 0-graph: invalid argument: graph [Invalid argument] Warning FailedMount 49s kubelet, n27.dev MountVolume.SetUp failed for volume "pvc-111c0388-bc46-11e9-9644-08002763a99f" : mount failed: mount failed: exit status 1 Mounting command: systemd-run Mounting arguments: --description=Kubernetes transient mount for /var/lib/kubelet/pods/1123119f-bc46-11e9-9644-08002763a99f/volumes/kubernetes.io~glusterfs/pvc-111c0388-bc46-11e9-9644-08002763a99f --scope -- mount -t glusterfs -o log-file=/var/lib/kubelet/plugins/kubernetes.io/glusterfs/pvc-111c0388-bc46-11e9-9644-08002763a99f/gluster-pod1-glusterfs.log,log-level=ERROR,backup-volfile-servers=192.168.33.55:192.168.33.56:192.168.33.57,auto_unmount 192.168.33.55:vol_050b5c59f74b1ca53e838b9544866dd2 /var/lib/kubelet/pods/1123119f-bc46-11e9-9644-08002763a99f/volumes/kubernetes.io~glusterfs/pvc-111c0388-bc46-11e9-9644-08002763a99f Output: Running scope as unit run-24918.scope. Mount failed. Please check the log file for more details. the following error information was pulled from the glusterfs log to help diagnose this issue: [2019-08-11 14:41:50.612078] E [MSGID: 100026] [glusterfsd.c:2351:glusterfs_process_volfp] 0-: failed to construct the graph [2019-08-11 14:41:50.612277] E [graph.c:1142:glusterfs_graph_destroy] (-->/usr/sbin/glusterfs(mgmt_getspec_cbk+0x532) [0x558767795332] -->/usr/sbin/glusterfs(glusterfs_process_volfp+0x150) [0x55876778ee60] -->/lib64/libglusterfs.so.0(glusterfs_graph_destroy+0x84) [0x7fe047c8f1e4] ) 0-graph: invalid argument: graph [Invalid argument] Warning FailedMount 33s kubelet, n27.dev MountVolume.SetUp failed for volume "pvc-111c0388-bc46-11e9-9644-08002763a99f" : mount failed: mount failed: exit status 1 Mounting command: systemd-run Mounting arguments: --description=Kubernetes transient mount for /var/lib/kubelet/pods/1123119f-bc46-11e9-9644-08002763a99f/volumes/kubernetes.io~glusterfs/pvc-111c0388-bc46-11e9-9644-08002763a99f --scope -- mount -t glusterfs -o log-level=ERROR,backup-volfile-servers=192.168.33.55:192.168.33.56:192.168.33.57,auto_unmount,log-file=/var/lib/kubelet/plugins/kubernetes.io/glusterfs/pvc-111c0388-bc46-11e9-9644-08002763a99f/gluster-pod1-glusterfs.log 192.168.33.55:vol_050b5c59f74b1ca53e838b9544866dd2 /var/lib/kubelet/pods/1123119f-bc46-11e9-9644-08002763a99f/volumes/kubernetes.io~glusterfs/pvc-111c0388-bc46-11e9-9644-08002763a99f Output: Running scope as unit run-25208.scope. Mount failed. Please check the log file for more details. the following error information was pulled from the glusterfs log to help diagnose this issue: [2019-08-11 14:42:06.929857] E [MSGID: 100026] [glusterfsd.c:2351:glusterfs_process_volfp] 0-: failed to construct the graph [2019-08-11 14:42:06.930063] E [graph.c:1142:glusterfs_graph_destroy] (-->/usr/sbin/glusterfs(mgmt_getspec_cbk+0x532) [0x55a84024d332] -->/usr/sbin/glusterfs(glusterfs_process_volfp+0x150) [0x55a840246e60] -->/lib64/libglusterfs.so.0(glusterfs_graph_destroy+0x84) [0x7f8a74e401e4] ) 0-graph: invalid argument: graph [Invalid argument] Warning FailedMount 0s kubelet, n27.dev MountVolume.SetUp failed for volume "pvc-111c0388-bc46-11e9-9644-08002763a99f" : mount failed: mount failed: exit status 1 Mounting command: systemd-run Mounting arguments: --description=Kubernetes transient mount for /var/lib/kubelet/pods/1123119f-bc46-11e9-9644-08002763a99f/volumes/kubernetes.io~glusterfs/pvc-111c0388-bc46-11e9-9644-08002763a99f --scope -- mount -t glusterfs -o auto_unmount,log-file=/var/lib/kubelet/plugins/kubernetes.io/glusterfs/pvc-111c0388-bc46-11e9-9644-08002763a99f/gluster-pod1-glusterfs.log,log-level=ERROR,backup-volfile-servers=192.168.33.55:192.168.33.56:192.168.33.57 192.168.33.55:vol_050b5c59f74b1ca53e838b9544866dd2 /var/lib/kubelet/pods/1123119f-bc46-11e9-9644-08002763a99f/volumes/kubernetes.io~glusterfs/pvc-111c0388-bc46-11e9-9644-08002763a99f Output: Running scope as unit run-25563.scope. Mount failed. Please check the log file for more details. the following error information was pulled from the glusterfs log to help diagnose this issue: [2019-08-11 14:42:39.344185] E [MSGID: 100026] [glusterfsd.c:2351:glusterfs_process_volfp] 0-: failed to construct the graph [2019-08-11 14:42:39.344398] E [graph.c:1142:glusterfs_graph_destroy] (-->/usr/sbin/glusterfs(mgmt_getspec_cbk+0x532) [0x5639cfa58332] -->/usr/sbin/glusterfs(glusterfs_process_volfp+0x150) [0x5639cfa51e60] -->/lib64/libglusterfs.so.0(glusterfs_graph_destroy+0x84) [0x7f93ffa2c1e4] ) 0-graph: invalid argument: graph [Invalid argument] # 服务端和客户端版本不一致 sudo yum list installed |grep gluster sudo rpm -qa |grep gluster sudo yum install centos-release-gluster -y && sudo yum clean all && sudo yum makecache sudo yum remove glusterfs-fuse -y && sudo yum autoremove -y && sudo yum install glusterfs-fuse ``` # 部署istio ```bash curl -L https://git.io/getLatestIstio | ISTIO_VERSION=1.2.4 sh - kubectl create namespace istio-system helm template install/kubernetes/helm/istio-init --name istio-init --namespace istio-system | kubectl apply -f - kubectl get crds | grep 'istio.io\|certmanager.k8s.io' | wc -l # 23 个crd If cert-manager is enabled, then the CRD count will be 28 instead. helm template install/kubernetes/helm/istio --name istio --namespace istio-system | kubectl apply -f - kubectl -n istio-system get svc,ep,pod,deployments,job -o wide kubectl patch svc istio-ingressgateway -n istio-system -p '{"spec":{"externalIPs":["your_external_ip"]}}' #### kubectl run -it --rm --restart=Never --generator=run-pod/v1 --image=praqma/network-multitool pod-$RANDOM -- /bin/sh -c 'nslookup istio-sidecar-injector.istio-system.svc' kubectl run -it --rm --restart=Never --generator=run-pod/v1 --image=praqma/network-multitool pod-$RANDOM -- /bin/sh -c 'nslookup istio-sidecar-injector.istio-system.svc && curl https://istio-sidecar-injector.istio-system.svc:443/inject?timeout=30s -k -i -v && curl -ivk --connect-timeout 3 -m 5 -H "Authorization: Bearer $(cat /var/run/secrets/kubernetes.io/serviceaccount/token)" https://10.254.0.1:443/apis/admissionregistration.k8s.io/v1beta1/mutatingwebhookconfigurations' # /apis/admissionregistration.k8s.io/v1beta1/mutatingwebhookconfigurations kubectl label namespace MYNAMESPACE istio-injection=enabled kubectl get mutatingwebhookconfiguration istio-sidecar-injector -o yaml -o jsonpath='{.webhooks[0].clientConfig.caBundle}' | md5sum kubectl -n istio-system get secret istio.istio-sidecar-injector-service-account -o jsonpath='{.data.root-cert\.pem}' | md5sum kubectl -n istio-system logs $(kubectl get pod -n istio-system -l istio=sidecar-injector -o jsonpath={.items[0].metadata.name}) kubectl get validatingwebhookconfiguration istio-galley -o yaml kubectl -n istio-system get configmap istio-galley-configuration -o jsonpath='{.data}' kubectl get ValidatingWebhookConfiguration istio-galley -oyaml kubectl get ValidatingWebhookConfiguration istio-galley -oyaml |grep -C3 'admitpilot\|admitmixer' kubectl -n istio-system get pod -listio=galley -o wide kubectl -n istio-system get ep istio-galley kubectl get clusterrole istio-galley-istio-system -o yaml for pod in $(kubectl -n istio-system get pod -listio=galley -o jsonpath='{.items[*].metadata.name}'); do kubectl -n istio-system logs ${pod} done # https://istio.io/docs/ops/setup/validation/ # https://istio.io/docs/ops/setup/webhook/ # 验证webhook服务的连通性 kubectl get pod,svc -o wide -n istio-system -l app=sidecarInjectorWebhook kubectl get pod,svc -o wide -n istio-system -l istio=galley sudo ipvsadm --list -n |grep -A2 10.254.98.23 nc -nvz 10.254.98.23 443 kubectl get svc -o wide -n istio-system -l istio=galley -o jsonpath='{.items[0].spec.clusterIP}' kubectl get svc -o wide -n istio-system -l app=sidecarInjectorWebhook -o jsonpath='{.items[0].spec.clusterIP}' curl -ikv https://`kubectl get svc -o wide -n istio-system -l app=sidecarInjectorWebhook -o jsonpath='{.items[0].spec.clusterIP}'`:443 curl -ikv https://`kubectl get svc -o wide -n istio-system -l istio=galley -o jsonpath='{.items[0].spec.clusterIP}'`:443 ## 部署demo kubectl label namespace default istio-injection=enabled kubectl apply -f samples/bookinfo/platform/kube/bookinfo.yaml kubectl apply -f samples/bookinfo/networking/bookinfo-gateway.yaml kubectl apply -f samples/helloworld/helloworld-gateway.yaml kubectl apply -f samples/helloworld/helloworld.yaml curl http://192.168.33.180/hello curl http://192.168.33.180/productpage kubectl run -it --rm --restart=Never --generator=run-pod/v1 \ --image=raisiqueira/mtr pod-$RANDOM \ -- /bin/sh -c 'sleep 3s; \ echo "################################################################" \ && ip a && mtr -c2 -r 192.168.33.21'; ### kubectl exec -n istio-system $(kubectl get pod --namespace istio-system --selector istio=ingressgateway --output jsonpath='{.items[0].metadata.name}') -- netstat -tunlp Active Internet connections (only servers) Proto Recv-Q Send-Q Local Address Foreign Address State PID/Program name tcp 0 0 0.0.0.0:80 0.0.0.0:* LISTEN 27/envoy tcp 0 0 0.0.0.0:15090 0.0.0.0:* LISTEN 27/envoy tcp 0 0 127.0.0.1:15000 0.0.0.0:* LISTEN 27/envoy tcp6 0 0 :::15020 :::* LISTEN 1/pilot-agent kubectl exec -it helloworld-v1-b6c7975c7-77xvl -c helloworld -- curl http://helloworld.default.svc:5000/hello kubectl -n istio-system describe deployments istio-policy kubectl -n istio-system logs $(kubectl -n istio-system get pods -lapp=policy -o jsonpath='{.items[0].metadata.name}') -cmixer kubectl -n istio-system logs $(kubectl -n istio-system get pods -lapp=policy -o jsonpath='{.items[0].metadata.name}') -cistio-proxy kubectl -n istio-system logs istio-pilot-78998f5bc4-s8kdj discovery -f nnection error: desc = "transport: Error while dialing dial tcp 10.254.248.13:9901: connect: connection refused" 2019-09-27T02:35:21.354124Z info mcp (re)trying to establish new MCP sink stream 2019-09-27T02:35:21.354202Z error mcp Failed to create a new MCP sink stream: rpc error: code = Unavailable desc = all SubConns are in TransientFailure, latest connection error: connection error: desc = "transport: Error while dialing dial tcp 10.254.248.13:9901: connect: connection refused" 2019-09-27T02:35:22.354985Z info mcp (re)trying to establish new MCP sink stream 2019-09-27T02:35:22.355068Z error mcp Failed to create a new MCP sink stream: rpc error: code = Unavailable desc = all SubConns are in TransientFailure, latest connection error: connection error: desc = "transport: Error while dialing dial tcp 10.254.248.13:9901: connect: connection refused" 2019-09-27T02:35:23.360141Z info mcp (re)trying to establish new MCP sink stream 2019-09-27T02:35:23.360231Z error mcp Failed to create a new MCP sink stream: rpc error: code = Unavailable desc = all SubConns are in TransientFailure, latest connection error: connection error: desc = "transport: Error while dialing dial tcp 10.254.248.13:9901: connect: connection refused" kubectl -n istio-system logs istio-galley-677bf9cff7-22sr7 2019-09-27T02:40:37.791750Z error istio.io/istio/galley/pkg/crd/validation/endpoint.go:103: Failed to list *v1.Endpoints: Get https://10.254.0.1:443/api/v1/namespaces/istio-system/endpoints?fieldSelector=metadata.name%3Distio-galley&limit=500&resourceVersion=0: dial tcp 10.254.0.1:443: connect: no route to host 2019-09-27T02:40:40.803000Z error istio.io/istio/galley/pkg/crd/validation/endpoint.go:103: Failed to list *v1.Endpoints: Get https://10.254.0.1:443/api/v1/namespaces/istio-system/endpoints?fieldSelector=metadata.name%3Distio-galley&limit=500&resourceVersion=0: dial tcp 10.254.0.1:443: connect: no route to host 2019-09-27T02:40:43.808687Z error istio.io/istio/galley/pkg/crd/validation/endpoint.go:103: Failed to list *v1.Endpoints: Get https://10.254.0.1:443/api/v1/namespaces/istio-system/endpoints?fieldSelector=metadata.name%3Distio-galley&limit=500&resourceVersion=0: dial tcp 10.254.0.1:443: connect: no route to host 2019-09-27T02:40:46.819131Z error istio.io/istio/galley/pkg/crd/validation/endpoint.go:103: Failed to list *v1.Endpoints: Get https://10.254.0.1:443/api/v1/namespaces/istio-system/endpoints?fieldSelector=metadata.name%3Distio-galley&limit=500&resourceVersion=0: dial tcp 10.254.0.1:443: connect: no route to host 2019-09-27T02:40:49.830907Z error istio.io/istio/galley/pkg/crd/validation/endpoint.go:103: Failed to list *v1.Endpoints: Get https://10.254.0.1:443/api/v1/namespaces/istio-system/endpoints?fieldSelector=metadata.name%3Distio-galley&limit=500&resourceVersion=0: dial tcp 10.254.0.1:443: connect: no route to host 2019-09-27T02:41:46.992999Z error kube Expected resources (CRDs) not found: [template QuotaSpec rule VirtualService ServiceEntry ClusterRbacConfig QuotaSpecBinding ServiceRoleBinding attributemanifest HTTPAPISpecBinding Sidecar adapter Ingress Gateway Policy instance DestinationRule EnvoyFilter MeshPolicy RbacConfig handler ServiceRole HTTPAPISpec] 2019-09-27T02:41:46.993023Z error kube To stop Galley from waiting for these resources (CRDs), consider using the --excludedResourceKinds flag 2019-09-27T02:41:46.993379Z fatal Error creating server: 54 errors occurred: * could not find config.istio.io/v1alpha2: Get https://10.254.0.1:443/apis/config.istio.io/v1alpha2?timeout=32s: dial tcp 10.254.0.1:443: connect: no route to host * could not find config.istio.io/v1alpha2: Get https://10.254.0.1:443/apis/config.istio.io/v1alpha2?timeout=32s: dial tcp 10.254.0.1:443: connect: no route to host * could not find config.istio.io/v1alpha2: Get https://10.254.0.1:443/apis/config.istio.io/v1alpha2?timeout=32s: dial tcp 10.254.0.1:443: connect: no route to host * could not find config.istio.io/v1alpha2: Get https://10.254.0.1:443/apis/config.istio.io/v1alpha2?timeout=32s: dial tcp 10.254.0.1:443: connect: no route to host * could not find config.istio.io/v1alpha2: Get https://10.254.0.1:443/apis/config.istio.io/v1alpha2?timeout=32s: dial tcp 10.254.0.1:443: connect: no route to host * could not find config.istio.io/v1alpha2: Get https://10.254.0.1:443/apis/config.istio.io/v1alpha2?timeout=32s: dial tcp 10.254.0.1:443: connect: no route to host * could not find config.istio.io/v1alpha2: Get https://10.254.0.1:443/apis/config.istio.io/v1alpha2?timeout=32s: dial tcp 10.254.0.1:443: connect: no route to host * could not find networking.istio.io/v1alpha3: Get https://10.254.0.1:443/apis/networking.istio.io/v1alpha3?timeout=32s: dial tcp 10.254.0.1:443: connect: no route to host * could not find config.istio.io/v1alpha2: Get https://10.254.0.1:443/apis/config.istio.io/v1alpha2?timeout=32s: dial tcp 10.254.0.1:443: connect: no route to host * could not find config.istio.io/v1alpha2: Get https://10.254.0.1:443/apis/config.istio.io/v1alpha2?timeout=32s: dial tcp 10.254.0.1:443: connect: no route to host * could not find config.istio.io/v1alpha2: Get https://10.254.0.1:443/apis/config.istio.io/v1alpha2?timeout=32s: dial tcp 10.254.0.1:443: connect: no route to host * could not find config.istio.io/v1alpha2: Get https://10.254.0.1:443/apis/config.istio.io/v1alpha2?timeout=32s: dial tcp 10.254.0.1:443: connect: no route to host * could not find config.istio.io/v1alpha2: Get https://10.254.0.1:443/apis/config.istio.io/v1alpha2?timeout=32s: dial tcp 10.254.0.1:443: connect: no route to host * could not find networking.istio.io/v1alpha3: Get https://10.254.0.1:443/apis/networking.istio.io/v1alpha3?timeout=32s: dial tcp 10.254.0.1:443: connect: no route to host * could not find networking.istio.io/v1alpha3: Get https://10.254.0.1:443/apis/networking.istio.io/v1alpha3?timeout=32s: dial tcp 10.254.0.1:443: connect: no route to host * could not find config.istio.io/v1alpha2: Get https://10.254.0.1:443/apis/config.istio.io/v1alpha2?timeout=32s: dial tcp 10.254.0.1:443: connect: no route to host * could not find config.istio.io/v1alpha2: Get https://10.254.0.1:443/apis/config.istio.io/v1alpha2?timeout=32s: dial tcp 10.254.0.1:443: connect: no route to host * could not find config.istio.io/v1alpha2: Get https://10.254.0.1:443/apis/config.istio.io/v1alpha2?timeout=32s: dial tcp 10.254.0.1:443: connect: no route to host * could not find networking.istio.io/v1alpha3: Get https://10.254.0.1:443/apis/networking.istio.io/v1alpha3?timeout=32s: dial tcp 10.254.0.1:443: connect: no route to host * could not find config.istio.io/v1alpha2: Get https://10.254.0.1:443/apis/config.istio.io/v1alpha2?timeout=32s: dial tcp 10.254.0.1:443: connect: no route to host * could not find config.istio.io/v1alpha2: Get https://10.254.0.1:443/apis/config.istio.io/v1alpha2?timeout=32s: dial tcp 10.254.0.1:443: connect: no route to host * could not find networking.istio.io/v1alpha3: Get https://10.254.0.1:443/apis/networking.istio.io/v1alpha3?timeout=32s: dial tcp 10.254.0.1:443: connect: no route to host * could not find config.istio.io/v1alpha2: Get https://10.254.0.1:443/apis/config.istio.io/v1alpha2?timeout=32s: dial tcp 10.254.0.1:443: connect: no route to host * could not find config.istio.io/v1alpha2: Get https://10.254.0.1:443/apis/config.istio.io/v1alpha2?timeout=32s: dial tcp 10.254.0.1:443: connect: no route to host * could not find authentication.istio.io/v1alpha1: Get https://10.254.0.1:443/apis/authentication.istio.io/v1alpha1?timeout=32s: dial tcp 10.254.0.1:443: connect: no route to host * could not find config.istio.io/v1alpha2: Get https://10.254.0.1:443/apis/config.istio.io/v1alpha2?timeout=32s: dial tcp 10.254.0.1:443: connect: no route to host * could not find config.istio.io/v1alpha2: Get https://10.254.0.1:443/apis/config.istio.io/v1alpha2?timeout=32s: dial tcp 10.254.0.1:443: connect: no route to host * could not find config.istio.io/v1alpha2: Get https://10.254.0.1:443/apis/config.istio.io/v1alpha2?timeout=32s: dial tcp 10.254.0.1:443: connect: no route to host * could not find rbac.istio.io/v1alpha1: Get https://10.254.0.1:443/apis/rbac.istio.io/v1alpha1?timeout=32s: dial tcp 10.254.0.1:443: connect: no route to host * could not find config.istio.io/v1alpha2: Get https://10.254.0.1:443/apis/config.istio.io/v1alpha2?timeout=32s: dial tcp 10.254.0.1:443: connect: no route to host * could not find config.istio.io/v1alpha2: Get https://10.254.0.1:443/apis/config.istio.io/v1alpha2?timeout=32s: dial tcp 10.254.0.1:443: connect: no route to host * could not find config.istio.io/v1alpha2: Get https://10.254.0.1:443/apis/config.istio.io/v1alpha2?timeout=32s: dial tcp 10.254.0.1:443: connect: no route to host * could not find extensions/v1beta1: Get https://10.254.0.1:443/apis/extensions/v1beta1?timeout=32s: dial tcp 10.254.0.1:443: connect: no route to host * could not find config.istio.io/v1alpha2: Get https://10.254.0.1:443/apis/config.istio.io/v1alpha2?timeout=32s: dial tcp 10.254.0.1:443: connect: no route to host * could not find rbac.istio.io/v1alpha1: Get https://10.254.0.1:443/apis/rbac.istio.io/v1alpha1?timeout=32s: dial tcp 10.254.0.1:443: connect: no route to host * could not find config.istio.io/v1alpha2: Get https://10.254.0.1:443/apis/config.istio.io/v1alpha2?timeout=32s: dial tcp 10.254.0.1:443: connect: no route to host * could not find config.istio.io/v1alpha2: Get https://10.254.0.1:443/apis/config.istio.io/v1alpha2?timeout=32s: dial tcp 10.254.0.1:443: connect: no route to host * could not find config.istio.io/v1alpha2: Get https://10.254.0.1:443/apis/config.istio.io/v1alpha2?timeout=32s: dial tcp 10.254.0.1:443: connect: no route to host * could not find rbac.istio.io/v1alpha1: Get https://10.254.0.1:443/apis/rbac.istio.io/v1alpha1?timeout=32s: dial tcp 10.254.0.1:443: connect: no route to host * could not find rbac.istio.io/v1alpha1: Get https://10.254.0.1:443/apis/rbac.istio.io/v1alpha1?timeout=32s: dial tcp 10.254.0.1:443: connect: no route to host * could not find config.istio.io/v1alpha2: Get https://10.254.0.1:443/apis/config.istio.io/v1alpha2?timeout=32s: dial tcp 10.254.0.1:443: connect: no route to host * could not find config.istio.io/v1alpha2: Get https://10.254.0.1:443/apis/config.istio.io/v1alpha2?timeout=32s: dial tcp 10.254.0.1:443: connect: no route to host * could not find config.istio.io/v1alpha2: Get https://10.254.0.1:443/apis/config.istio.io/v1alpha2?timeout=32s: dial tcp 10.254.0.1:443: connect: no route to host * could not find config.istio.io/v1alpha2: Get https://10.254.0.1:443/apis/config.istio.io/v1alpha2?timeout=32s: dial tcp 10.254.0.1:443: connect: no route to host * could not find config.istio.io/v1alpha2: Get https://10.254.0.1:443/apis/config.istio.io/v1alpha2?timeout=32s: dial tcp 10.254.0.1:443: connect: no route to host * could not find config.istio.io/v1alpha2: Get https://10.254.0.1:443/apis/config.istio.io/v1alpha2?timeout=32s: dial tcp 10.254.0.1:443: connect: no route to host * could not find config.istio.io/v1alpha2: Get https://10.254.0.1:443/apis/config.istio.io/v1alpha2?timeout=32s: dial tcp 10.254.0.1:443: connect: no route to host * could not find config.istio.io/v1alpha2: Get https://10.254.0.1:443/apis/config.istio.io/v1alpha2?timeout=32s: dial tcp 10.254.0.1:443: connect: no route to host * could not find config.istio.io/v1alpha2: Get https://10.254.0.1:443/apis/config.istio.io/v1alpha2?timeout=32s: dial tcp 10.254.0.1:443: connect: no route to host * could not find config.istio.io/v1alpha2: Get https://10.254.0.1:443/apis/config.istio.io/v1alpha2?timeout=32s: dial tcp 10.254.0.1:443: connect: no route to host * could not find authentication.istio.io/v1alpha1: Get https://10.254.0.1:443/apis/authentication.istio.io/v1alpha1?timeout=32s: dial tcp 10.254.0.1:443: connect: no route to host * could not find config.istio.io/v1alpha2: Get https://10.254.0.1:443/apis/config.istio.io/v1alpha2?timeout=32s: dial tcp 10.254.0.1:443: connect: no route to host * could not find networking.istio.io/v1alpha3: Get https://10.254.0.1:443/apis/networking.istio.io/v1alpha3?timeout=32s: dial tcp 10.254.0.1:443: connect: no route to host * could not find config.istio.io/v1alpha2: Get https://10.254.0.1:443/apis/config.istio.io/v1alpha2?timeout=32s: dial tcp 10.254.0.1:443: connect: no route to host : the following resource type(s) were not found: [template QuotaSpec rule VirtualService ServiceEntry ClusterRbacConfig QuotaSpecBinding ServiceRoleBinding attributemanifest HTTPAPISpecBinding Sidecar adapter Ingress Gateway Policy instance DestinationRule EnvoyFilter MeshPolicy RbacConfig handler ServiceRole HTTPAPISpec] ## 升级 curl -L https://git.io/getLatestIstio | ISTIO_VERSION=1.3.0 sh - sudo cp istio-1.3.0/bin/istioctl /usr/bin/istioctl helm template install/kubernetes/helm/istio-init --name istio-init --namespace istio-system > istio-init.yaml helm template install/kubernetes/helm/istio --name istio --namespace istio-system > istio.yaml kubectl patch service istio-ingressgateway -n istio-system -p '{"spec":{"type":"NodePort"}}' kubectl patch service istio-ingressgateway -n istio-system -p '{"spec":{"type":"LoadBalancer"}}' kubectl -n istio-system get service istio-ingressgateway -o jsonpath='{.spec.ports[?(@.name=="http2")].nodePort}' kubectl -n istio-system get pod -l istio=ingressgateway -o jsonpath={.items[0].status.hostIP} ## 部署metallb kubectl apply -f https://raw.githubusercontent.com/metallb/metallb/main/manifests/metallb.yaml #### cat < docker container inspect -f "{{.State.Pid}}" docker container inspect -f "{{.Id}}" docker container ps --format='table {{.Ports}}\t{{.Names}}' ##### docker container run -d --restart always --name centos77 centos:centos7.6.1810 /bin/sh -c "sleep 365d;" docker container exec -it centos77 /bin/bash docker container stop centos77 docker container rm centos77 ##### docker container run -d --restart always --name ddd debian:stretch /bin/sh -c "while true; do sleep 1s;echo \$HOSTNAME; done;" docker container exec -it ddd /bin/bash docker container logs -f ddd docker container stop ddd && docker container rm ddd ######### # 导入导出镜像 docker image save registry.access.redhat.com/rhel7/pod-infrastructure > /shell/pod-infrastructure.tar docker image load < pod-infrastructure.tar ##### kubectl api-resources kubectl api-versions kubectl cluster-info kubectl get all --all-namespaces kubectl get apiservices kubectl get replicaset kubectl get --raw "/apis" | jq kubectl get svc,ep,deploy,po,no,rs,ns,sa --all-namespaces -o wide kubectl get ds kubectl get cs kubectl get csr kubectl certificate approve kubectl certificate approve `kubectl get csr --no-headers |awk '{print $1}'` kubectl logs calico-node-2nct7 --namespace kube-system # --field-selector status.phase=Running kubectl logs --namespace kube-system $(kubectl get pod -n kube-system -l k8s-app=calico-node -o jsonpath={.items[0].metadata.name}) ## 强制删除 pod kubectl delete -n istio-system pod istio-policy-5dd5cb9645-l5wfd --force --grace-period=0 ## 强制删除 namespace kubectl delete namespace NAMESPACENAME --force --grace-period=0 ######### kubectl get pod -n kube-system -l k8s-app=calico-node -o wide kubectl get pod -n kube-system -l k8s-app=calico-kube-controllers -o wide kubectl describe cm calico-config -n kube-system cat /etc/cni/net.d/10-calico.conflist ls -l /etc/cni/net.d/ # 滚动跟新 kubectl set image deployment/my-nginx nginx=nginx:1.9.1 kubectl rollout status deployment/my-nginx kubectl rollout history deployment/my-nginx kubectl apply -f --record # spec.revisionHistoryLimit # Mark node as unschedulable or schedulable # SchedulingDisabled kubectl cordon kubectl uncordon # 污点 kubectl taint # 驱赶node上的pod kubectl drain kubectl run -it --rm --generator=run-pod/v1 --image=busybox:1.30.1-musl busy11 -- /bin/sh kubectl run -it --rm --generator=run-pod/v1 --image=radial/busyboxplus:curl busyboxplus -- /bin/sh kubectl run -it --rm --generator=run-pod/v1 --image=alpine:3.9.4 alpine -- /bin/sh kubectl run -it --rm --generator=run-pod/v1 --image=infoblox/dnstools dns-client kubectl run -it --rm --generator=run-pod/v1 --image=tutum/dnsutils dns -- /bin/sh kubectl run -it --rm --generator=run-pod/v1 --image=sjourdan/toolbox t111 -- /bin/sh kubectl run -it --rm --generator=run-pod/v1 --image=praqma/network-multitool t222 -- /bin/sh kubectl run -it --rm --generator=run-pod/v1 --image=buildpack-deps:18.04-curl ubuntu -- /bin/sh kubectl run -it --rm --generator=run-pod/v1 --image=buildpack-deps:18.04-curl ubuntu1 -- /bin/bash kubectl run -it --rm --restart=Never --generator=run-pod/v1 --image=praqma/network-multitool pod-$RANDOM -- /bin/sh -c 'nc -nvz 10.254.98.23 443' ############## kubectl run --generator=deployment/apps.v1beta1 --image=nginx:1.16.0-alpine --replicas=2 nginx11 kubectl exec -it $(kubectl get pod -l run=nginx11 -o jsonpath={.items[0].metadata.name}) -- /bin/sh -c "ping -c 200 10.254.0.1" ############## kubectl run -it --rm --restart=Never --generator=run-pod/v1 --image=busybox:1.31.1-glibc busybox-$RANDOM -- /bin/sh -c 'nslookup kubernetes.default.svc.cluster.local'; kubectl run --generator=run-pod/v1 --image=busybox:1.30.1-musl busybox11 -- /bin/sh -c 'sleep 3699d'; kubectl exec -it busybox11 -- /bin/sh -c "nslookup kubernetes.default.svc.cluster.local." kubectl exec -it busybox11 -- nslookup kubernetes.default.svc.cluster.local. kubectl run --generator=run-pod/v1 --image=sjourdan/toolbox t111 -- /bin/sh -c 'sleep 3699d' kubectl run --generator=run-pod/v1 --image=praqma/network-multitool t222 -- /bin/sh -c 'sleep 3699d' ######### kubectl run --generator=run-pod/v1 --image=tutum/dnsutils dns11 -- /bin/sh -c 'sleep 3699d' kubectl exec -it dns11 -- nslookup kubernetes.default ############ kubectl run --generator=run-pod/v1 --image=ubuntu:18.04 ubu -- /bin/sh -c "sleep 36500d;" kubectl run --generator=run-pod/v1 --image=centos:centos7.6.1810 centos7 -- /bin/sh -c "sleep 36500d;" ########### kubectl run --generator=run-pod/v1 --image=containous/whoami who333 kubectl expose pod who333 --port=80 --target-port=80 --type=NodePort curl http://$(kubectl get node -o jsonpath='{.items[0].status.addresses[?(@.type=="InternalIP")].address}'):$(kubectl get svc who333 -o jsonpath='{.spec.ports..nodePort}') ######## kubectl run --generator=deployment/apps.v1beta1 --image=nginx --replicas=5 nginx009 kubectl expose deployments nginx009 --port=80 --target-port=80 --type=NodePort curl http://$(kubectl get node -o jsonpath='{.items[0].status.addresses[?(@.type=="InternalIP")].address}'):$(kubectl get svc nginx009 -o jsonpath='{.spec.ports..nodePort}') kubectl scale --replicas=1 deployments/nginx009 ################# kubectl run --restart=Never --generator=run-pod/v1 --image=gcr.io/kuar-demo/kuard-amd64:blue kuard kubectl expose pod kuard --port=8080 --target-port=8080 --type=NodePort ### mtr -r -c 1 10.254.0.1 traceroute -n -q1 10.254.0.1 nc -nvz 10.254.0.2 53 nslookup -type=A kubernetes nslookup -type=A www.baidu.com ### kubectl get service kubernetes --output jsonpath='{.spec.ports[?(@.name=="https")].targetPort}' kubectl get node -o jsonpath='{.items[0].status.addresses[?(@.type=="InternalIP")].address}' kubectl get po -l k8s-app=nginx-demo -o jsonpath={.items..metadata.name} kubectl exec `kubectl get po -l run=ubuntu -o=name|cut -d "/" -f2` env kubectl get po -o=custom-columns=NAME:.metadata.name,RSRC:.metadata.resourceVersion kubectl get po --all-namespaces -o=custom-columns=NAME:.metadata.name,USER:.metadata.user,VERSION:.metadata.version kubectl get po -o go-template --template '{{range .items}}{{.metadata.name}}{{"\t"}}{{range .spec.containers}}{{.image}}{{","}}{{end}}{{"\n"}}{{end}}' ################## for ns in $(kubectl get ns --no-headers | cut -d ' ' -f1); do for po in $(kubectl -n $ns get po --no-headers --ignore-not-found | grep Terminating | cut -d ' ' -f1); do kubectl -n $ns delete po $po --force --grace-period 0; done; done; ################## kubectl -n kube-system edit cm kubeadm-config ################## # alpine更换阿里云repo sed -i 's/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g' /etc/apk/repositories # alpine更换科大repo sed -i 's/dl-cdn.alpinelinux.org/mirrors.ustc.edu.cn/g' /etc/apk/repositories kubectl exec -it /bin/sh kubectl exec -it $(kubectl get pods -o jsonpath='{.items[0].metadata.name}') /bin/sh kubectl exec -it $(kubectl get po -o json | jq -r '.items | sort_by(.spec.nodeName)[] | [.metadata.name] | @tsv' |grep test) /bin/sh # pod 中通过访问api curl -ik -H "Authorization: Bearer $(cat /var/run/secrets/kubernetes.io/serviceaccount/token)" https://kubernetes/api/v1/namespaces/default/pods kubectl get secret --namespace default dokuwiki-dokuwiki -o jsonpath="{.data.dokuwiki-password}" | base64 --decode kubectl get pods -o go-template --template '{{range .items}}{{.metadata.name}}{{"\n"}}{{end}}' kubectl get secret -o jsonpath='{range .items[?(@.metadata.annotations.kubernetes\.io/service-account\.name=="default")].data}{"token: "}{.token}{"\n\n"}{"ca.crt: "}{.ca\.crt}{"\n"}{end}' ### kubectl get no -o json | jq -r '.items[].status.addresses[] | select(.type=="InternalIP") | .address' kubectl delete pod $(kubectl get pods -o go-template='{{range .items}}{{if eq .status.phase "Failed"}}{{.metadata.name}}{{"\n"}}{{end}}{{end}}') ### export jsonpath='{range .items[*]}{"\n"}{@.metadata.name}{range @.status.conditions[*]}{"\t"}{@.type}={@.status}{end}{end}' kubectl get po -ao jsonpath="$jsonpath" && echo ### kubectl get --all-namespaces svc -o json | jq -r '.items[] | [.metadata.name,([.spec.ports[]?.nodePort | tostring ] | join("|"))] | @csv' kubectl get svc --all-namespaces -o go-template='{{range .items}}{{.metadata.name}}{{"\t"}}{{range.spec.ports}}{{if .nodePort}}{{.nodePort}}{{","}}{{end}}{{end}}{{"\n"}}{{end}}' kubectl get no -o jsonpath="{.items[?(@.spec.unschedulable)].metadata.name}" kubectl get pv -o jsonpath='{range.items[?(@.status.phase=="Released")]}{.metadata.name}{"\n"}{end}' kubectl get pv -o json | jq -r '.items | sort_by(.spec.capacity.storage)[]|[.metadata.name,.spec.capacity.storage]| @tsv' kubectl get po -o json | jq -r '.items | sort_by(.spec.nodeName)[]|select(.spec.nodeName=="node3")|[.metadata.name,.spec.nodeName]| @tsv' kubectl get no -o json | jq -r '.items | sort_by(.status.capacity.memory)[]|[.metadata.name,.status.capacity.memory]| @tsv' # 查看nodelabel kubectl get nodes --show-labels # You can remove incorrect label with