# 说明 ```bash 192.168.33.55 consul-agent(-server) nomad-server 192.168.33.56 consul-agent(-server) nomad-server 192.168.33.57 consul-agent(-server) nomad-server 192.168.33.88 consul-agent(-server=false) ``` ```ruby [55,56,57,88].each do |i| config.vm.define "n#{i}" do |node| node.vm.network "private_network", ip: "192.168.33.#{i}" node.vm.synced_folder "/data/vagrant/shell", "/shell" node.vm.network :forwarded_port, guest: 22, host: "2#{i}22", host_ip: "0.0.0.0" node.vm.provider "virtualbox" do |vb| vb.memory = "2048" vb.cpus = 2 end node.vm.provision "shell", inline: <<-SHELL echo "vagrant:vagrant" | sudo chpasswd mkdir -p /data chown -R vagrant:vagrant /data hostnamectl set-hostname n#{i} SHELL end end ``` # 下载 ```bash # download nomad curl -Ls https://releases.hashicorp.com/nomad/0.9.1/nomad_0.9.1_linux_amd64.zip -o nomad.zip unzip nomad.zip sudo chmod +x nomad sudo mv nomad /usr/local/bin # download consul curl -Ls https://releases.hashicorp.com/consul/1.4.4/consul_1.4.4_linux_amd64.zip -o consul.zip unzip consul.zip sudo chmod +x consul sudo mv consul /usr/local/bin # sudo mkdir -p /etc/consul.d # sudo mkdir -p /etc/nomad.d # sudo mkdir -p /data/consul # sudo mkdir -p /data/nomad ``` # consul集群 ## server ```bash # 单机模式 # /usr/local/bin/consul agent -dev # /usr/local/bin/consul agent -server -bootstrap-expect 1 -data-dir /data/consul -bind=10.10.49.193 -ui -client 0.0.0.0 /usr/local/bin/consul keygen # 替换"encrypt": "goplCZgdmOFMZ2Q43To0jw==" # sudo vi /etc/consul.d/config.json # ==> Multiple private IPv4 addresses found. Please configure one with 'bind' and/or 'advertise'. sudo /usr/local/bin/consul agent -config-dir /etc/consul.d/ ``` ```json { "bootstrap_expect": 3, "server": true, "client_addr": "0.0.0.0", "datacenter": "east-aws", "data_dir": "/data/consul", "dns_config": { "enable_truncate": true, "only_passing": true }, "encrypt": "goplCZgdmOFMZ2Q43To0jw==", "leave_on_terminate": true, "log_level": "INFO", "log_file": "/var/log/consul/", "rejoin_after_leave": true, "ui": true, "enable_debug": false, "retry_join": [ "192.168.33.55", "192.168.33.56", "192.168.33.57" ], "retry_interval": "30s", "start_join": [ "192.168.33.55", "192.168.33.56", "192.168.33.57" ], "disable_update_check": true } ``` ```bash ## 查看端口情况 sudo netstat -tunlp |grep consul tcp 0 0 192.168.33.55:8300 0.0.0.0:* LISTEN 5687/consul tcp 0 0 192.168.33.55:8301 0.0.0.0:* LISTEN 5687/consul tcp 0 0 192.168.33.55:8302 0.0.0.0:* LISTEN 5687/consul tcp6 0 0 :::8500 :::* LISTEN 5687/consul tcp6 0 0 :::8600 :::* LISTEN 5687/consul udp 0 0 192.168.33.55:8301 0.0.0.0:* 5687/consul udp 0 0 192.168.33.55:8302 0.0.0.0:* 5687/consul udp6 0 0 :::8600 :::* 5687/consul ## 查看集群节点 consul operator raft list-peers Node ID Address State Voter RaftProtocol n55 66c5b93f-24e9-fa4e-9f65-6bbd6376a1b6 192.168.33.55:8300 leader true 3 n56 bc46826a-de43-fe99-79cf-91cac82e5fa1 192.168.33.56:8300 follower true 3 n57 20e7c41b-71c3-4025-e89e-0d715412d552 192.168.33.57:8300 follower true 3 ``` ## client ```bash ## 启动 ./consul agent -data-dir="/data/consul" -bind=192.168.33.88 \ -retry-join=192.168.33.55 \ -retry-join=192.168.33.56 \ -retry-join=192.168.33.57 \ -retry-interval=10s \ -datacenter="east-aws" \ -server=false \ -log-level="INFO" \ -client=0.0.0.0 #### #### ### 查看端口 sudo netstat -tunlp |grep consul tcp 0 0 192.168.33.88:8301 0.0.0.0:* LISTEN 28326/./consul tcp6 0 0 :::8500 :::* LISTEN 28326/./consul tcp6 0 0 :::8600 :::* LISTEN 28326/./consul udp 0 0 192.168.33.88:8301 0.0.0.0:* 28326/./consul udp6 0 0 :::8600 :::* 28326/./consul ## 查看成员 consul members Node Address Status Type Build Protocol DC Segment n55 192.168.33.55:8301 alive server 1.4.4 2 east-aws n56 192.168.33.56:8301 alive server 1.4.4 2 east-aws n57 192.168.33.57:8301 alive server 1.4.4 2 east-aws n88 192.168.33.88:8301 alive client 1.4.4 2 east-aws ``` ## systemd ```toml [Unit] Description=Consul Startup process After=network.target [Service] Type=simple ExecStart=/bin/bash -c '/usr/local/bin/consul agent -bind=192.168.33.55 -config-dir /etc/consul.d/' ExecStop=/usr/local/bin/consul leave ExecReload=/bin/kill -HUP $MAINPID TimeoutStartSec=0 [Install] WantedBy=default.target ``` ```bash sudo vi /lib/systemd/system/consul.service; sudo systemctl daemon-reload sudo systemctl enable consul.service sudo systemctl start consul.service sudo systemctl status consul.service ``` ## 端口 * 8600 dns * 8500 http * 8501 https * 8502 grpc * 8300 server * 8301 serf_lan * 8302 serf_wan ## HTTP API ```bash ############################# # kv - Key/Value存储 # agent - Agent控制 # catalog - 管理nodes和services # health - 管理健康监测 # session - Session操作 # acl - ACL创建和管理event - 用户Events # status - Consul系统状态 # /v1/agent/checks : 返回本地agent注册的所有检查(包括配置文件和HTTP接口) # /v1/agent/services : 返回本地agent注册的所有 服务 # /v1/agent/members : 返回agent在集群的gossip pool中看到的成员 # /v1/agent/self : 返回本地agent的配置和成员信息/v1/agent/join/
: 触发本地agent加入node/v1/agent/force-leave/>: 强制删除node # /v1/agent/check/register : 在本地agent增加一个检查项，使用PUT方法传输一个json格式的数据/v1/agent/check/deregister/ : 注销一个本地agent的检查项/v1/agent/# check/pass/ : 设置一个本地检查项的状态为passing/v1/agent/check/warn/ : 设置一个本地检查项的状态为warning/v1/agent/check/fail/ : 设置一个本地检查项的状态为critical # /v1/agent/service/register : 在本地agent增加一个新的服务项，使用PUT方法传输一个json格式的数据/v1/agent/service/deregister/ : 注销一个本地agent的服务项 # /v1/catalog/register : Registers a new node, service, or check/v1/catalog/deregister : Deregisters a node, service, or check/v1/catalog/datacenters : Lists known datacenters # /v1/catalog/nodes : Lists nodes in a given DC # /v1/catalog/services : Lists services in a given DC # /v1/catalog/service/ : Lists the nodes in a given service # /v1/catalog/node/ : Lists the services provided by a node # /v1/healt/node/: 返回node所定义的检查，可用参数?dc= # /v1/health/checks/: 返回和服务相关联的检查，可用参数?dc= # /v1/health/service/: 返回给定datacenter中给定node中service # /v1/health/state/: 返回给定datacenter中指定状态的服务，state可以是"any", "unknown", "passing", "warning", or "critical"，可用参数?dc= # /v1/session/create: Creates a new session # /v1/session/destroy/: Destroys a given session # /v1/session/info/: Queries a given session # /v1/session/node/: Lists sessions belonging to a node # /v1/session/list: Lists all the active sessions # /v1/acl/create: Creates a new token with policy # /v1/acl/update: Update the policy of a token # /v1/acl/destroy/: Destroys a given token # /v1/acl/info/: Queries the policy of a given token # /v1/acl/clone/: Creates a new token by cloning an existing token # /v1/acl/list: Lists all the active tokens # /v1/status/leader : 返回当前集群的Raft leader # /v1/status/peers : 返回当前集群中同事 ############################# # 获取服务 curl -s http://127.0.0.1:8500/v1/catalog/services |jq # 集群成员 curl -s http://127.0.0.1:8500/v1/agent/members |jq # kv # kv 新增 curl --request PUT --data "hehe" http://127.0.0.1:8500/v1/kv/my-key # kv 查询 curl -s http://127.0.0.1:8500/v1/kv/my-key |jq -r '.[].Value' |base64 -d # kv 删除 curl --request DELETE http://127.0.0.1:8500/v1/kv/my-key # 服务注册 curl -X PUT -d '{"id":"tomcat1","name":"tomcat","address":"192.168.33.101","port":9999,"tags":["dev"],"checks":[{"http":"http://192.168.33.101:9999","interval":"1s"}]}' http://127.0.0.1:8500/v1/agent/service/register curl -X PUT -d '{"id":"tomcat2","name":"tomcat","address":"192.168.33.102","port":9999,"tags":["dev"],"checks":[{"http":"http://192.168.33.102:9999","interval":"1s"}]}' http://127.0.0.1:8500/v1/agent/service/register curl -X PUT -d '{"id":"tomcat3","name":"tomcat","address":"192.168.33.109","port":9999,"tags":["dev"],"checks":[{"http":"http://192.168.33.109:9999","interval":"1s"}]}' http://127.0.0.1:8500/v1/agent/service/register # 服务查询 curl -s http://127.0.0.1:8500/v1/catalog/service/tomcat\?pretty # 检查服务 curl -s http://127.0.0.1:8500/v1/health/service/tomcat\?pretty | grep -A 2 Status curl -GLs http://127.0.0.1:8500/v1/health/checks/tomcat --data-urlencode 'pretty' | grep -A 3 Status # 取消注册服务 curl -X PUT http://127.0.0.1:8500/v1/agent/service/deregister/my-service-id # dns发现 dig @127.0.0.1 -p 8600 tomcat.service.consul SRV ################################################# # bootstrap server sudo docker run -d -p 18300:8300 -p 18330:8301 -p 18360:8302 -p 18500:8500 -p 18600:8600 --name docker-consul-0 consul agent -server -bootstrap # 获取 docker-consul-0 的 IP 地址 # docker inspect -f '{{.NetworkSettings.IPAddress}}' docker-consul-0` sudo docker run -d -p 18301:8300 -p 18331:8301 -p 18361:8302 -p 18501:8500 -p 18601:8600 --name docker-consul-1 consul agent -server -join=`sudo docker inspect -f '{{.NetworkSettings.IPAddress}}' docker-consul-0` sudo docker run -d -p 18302:8300 -p 18332:8301 -p 18362:8302 -p 18502:8500 -p 18602:8600 --name docker-consul-2 consul agent -server -join=`sudo docker inspect -f '{{.NetworkSettings.IPAddress}}' docker-consul-0` sudo docker run -d -p 18303:8300 -p 18333:8301 -p 18363:8302 -p 18503:8500 -p 18603:8600 --name docker-consul-3 consul agent -server -join=`sudo docker inspect -f '{{.NetworkSettings.IPAddress}}' docker-consul-0` sudo docker run -d -p 18304:8300 -p 18334:8301 -p 18364:8302 -p 18504:8500 -p 18604:8600 --name docker-consul-4 consul agent -client -join=`sudo docker inspect -f '{{.NetworkSettings.IPAddress}}' docker-consul-0` ``` # nomad集群 ## server ```bash sudo vi /etc/nomad.d/server.conf ``` ```toml log_level = "DEBUG" data_dir = "/data/nomad" name = "client1" bind_addr = "0.0.0.0" advertise { http = "192.168.33.51" rpc = "192.168.33.51" serf = "192.168.33.51" } ports { http = 4646 rpc = 4647 serf = 4648 } client { enabled = true servers = ["192.168.33.51:4647"] options { "driver.raw_exec.enable" = "1" } } server { enabled = true bootstrap_expect = 1 } ``` ```bash /usr/local/bin/nomad agent -config=/etc/nomad.d ``` ## systemd * [nomad.service](https://github.com/hashicorp/nomad/blob/master/dist/systemd/nomad.service) # 参考 * [nomad.service](https://github.com/hashicorp/nomad/blob/master/dist/systemd/nomad.service) * [Schedulers](https://www.nomadproject.io/docs/schedulers.html) * [使用Consul做服务发现的若干姿势](http://blog.didispace.com/consul-service-discovery-exp/) * [How To Setup Consul Cluster](https://devopscube.com/setup-consul-cluster-guide/) * [Configuration](https://www.consul.io/docs/agent/options.html) * [Consul高可用安裝](https://www.twblogs.net/a/5bc10d2f2b717711c9242246) * [HTTP API Structure](https://www.consul.io/api/index.html) * [consul集群搭建与Golang服务发现示例](https://lihaoquan.me/2018/5/31/consul-in-action.html)