监控容器

使用docker inspect命令获取容器的详细信息

[root@docker ~]# docker inspect r-Default-nginx-1-fc08b70a [    {        "Id": "bdbc735f0802a755bfc2e4c669a6a029cf06a00d17591b5b1daf987855a4be48",        "Created": "2020-05-14T11:23:09.025156613Z",        "Path": "/.r/r",        "Args": [            "nginx",            "-g",            "daemon off;"        ],

• Docker的inspect命令有一个format参数,通过该参数指定一个Golang模板来获得一个容器或者镜像中指定的部分信息

[root@docker ~]# docker inspect -f '{{.NetworkSettings.IPAddress}}' friendly_banach 172.17.0.2[root@docker ~]# docker inspect -f '{{.State.Running}}' friendly_banach true

• 也可以使用docker-py获取详细信息

[root@docker ~]# python3Python 3.6.8 (default, Apr  2 2020, 13:34:55) [GCC 4.8.5 20150623 (Red Hat 4.8.5-39)] on linuxType "help", "copyright", "credits" or "license" for more information.>>> from docker import Client>>> c=Client(base_url="unix://var/run/docker.sock")>>> c.inspect_container('friendly_banach')['State']['Running']True>>> c.inspect_container('friendly_banach')['NetworkSettings']['IPAddress']'172.17.0.2'

获取运行中容器的使用统计信息

• 监控容器的资源使用情况
• 使用docker stats命令

[root@docker ~]# docker run -d -p 5000:5000 flask[root@docker ~]# docker stats 0212

• 接收到的是一个流输出

image.png

• 通过守护进程模式收集信息

[root@docker ~]# vim /etc/sysconfig/dockerOPTIONS='-H tcp://127.0.0.1:2375'[root@docker ~]# docker -H tcp://127.0.0.1:2375 run -d -p 5002:5000 localhost:5000/flask:foobardab59d1f84a1233be3d1a144f80e5a5edb9c7c7361f2da4f4868d8a7f77f064d[root@docker ~]# docker -H tcp://127.0.0.1:2375 psCONTAINER ID        IMAGE                         COMMAND                  CREATED             STATUS              PORTS                    NAMESdab59d1f84a1        localhost:5000/flask:foobar   "python /tmp/hello.py"   6 seconds ago       Up 5 seconds        0.0.0.0:5002->5000/tcp   nostalgic_wright# 使用curl访问Docker remote API[root@docker ~]# curl http://127.0.0.1:2375/containers/nostagic_wright/stats{"message":"No such container: nostagic_wright"}[root@docker ~]# curl http://127.0.0.1:2375/containers/nostalgic_wright/stats{"read":"2020-05-14T11:57:54.149141537Z","preread":"0001-01-01T00:00:00Z","pids_stats":{"current":1},"blkio_stats":{"io_service_bytes_recursive":[{"major":8,"minor":0,"op":"Read","value":15392768},{"major":8,"minor":0,"op":"Write","value":0},{"major":8,"minor":0,"op":"Sync","value":0},{"major":8,"minor":0,"op":"Async","value":15392768},{"major":8,"minor":0,"op":"Total","value":15392768}],"io_serviced_recursive":[{"major":8,"minor":0,"op":"Read","value":410},{"major":8,"minor":0,"op":"Write","value":0},{"major":8,"minor":0,"op":"Sync","value":0},{"major":8,"minor":0,"op":"Async","value":410},{"major":8,"minor":0,"op":"Total","value":410}],"io_queue_recursive":[],"io_service_time_recursive":[],"io_wait_time_recursive":[],"io_merged_recursive":[],"io_time_recursive":[],"sectors_recursive":[]},"num_procs":0,"storage_stats":{},"cpu_stats":{"cpu_usage":{"total_usage":510879029,"percpu_usage":[28316780,482562249],"usage_in_kernelmode":260000000,"usage_in_usermode":160000000},"system_cpu_usage":6079130000000,"throttling_data":{"periods":0,"throttled_periods":0,"throttled_time":0}},"precpu_stats":{"cpu_usage":{"total_usage":0,"usage_in_kernelmode":0,"usage_in_usermode":0},"throttling_data":{"periods":0,"throttled_periods":0,"throttled_time":0}},"memory_stats":{"usage":27471872,"max_usage":27541504,"stats":{"active_anon":13373440,"active_file":589824,"cache":14098432,"hierarchical_memory_limit":9223372036854771712,"hierarchical_memsw_limit":9223372036854771712,"inactive_anon":0,"inactive_file":13508608,"mapped_file":3932160,"pgfault":6898,"pgmajfault":20,"pgpgin":7376,"pgpgout":669,"rss":13373440,"rss_huge":0,"swap":0,"total_active_anon":13373440,"total_active_file":589824,"total_cache":14098432,"total_inactive_anon":0,"total_inactive_file":13508608,"total_mapped_file":3932160,"total_pgfault":6898,"total_pgmajfault":20,"total_pgpgin":7376,"total_pgpgout":669,"total_rss":13373440,"total_rss_huge":0,"total_swap":0,"total_unevictable":0,"unevictable":0},"limit":1019826176},"name":"/nostalgic_wright","id":"dab59d1f84a1233be3d1a144f80e5a5edb9c7c7361f2da4f4868d8a7f77f064d","networks":{"eth0":{"rx_bytes":656,"rx_packets":8,"rx_errors":0,"rx_dropped":0,"tx_bytes":656,"tx_packets":8,"tx_errors":0,"tx_dropped":0}}}

在Docker主机上监听Docker事件

• 监控主机上的Docker事件.对取消镜像的标记、删除镜像以及容器的生命周期事件
• 使用docker events命令

[root@docker ~]# docker events --since `date +%s`2020-05-14T20:19:47.804778385+08:00 container destroy d453efdbe9ed7c5163861c807c2b17188a202f0672346b6b0624183a0411ec63 (image=c22, name=stupefied_bhaskara)2020-05-14T20:20:38.504168502+08:00 container destroy 22197e73e2846564e42df146829fd76743098302f914a40c54b51278a0d18635 (image=c22, name=serene_feynman)2020-05-14T20:21:13.541290011+08:00 container destroy f37dd785b8688e5d2168d14622a69a0f6470fbc007a87b323c684e75a6093d23 (image=hello-world, name=distracted_lumiere)2020-05-14T20:21:16.308051302+08:00 image untag sha256:bf756fb1ae65adf866bd8c456593cd24beb6a0a061dedf42b26a993176745f6b (name=sha256:bf756fb1ae65adf866bd8c456593cd24beb6a0a061dedf42b26a993176745f6b)2020-05-14T20:21:16.310636070+08:00 image delete sha256:bf756fb1ae65adf866bd8c456593cd24beb6a0a061dedf42b26a993176745f6b (name=sha256:bf756fb1ae65adf866bd8c456593cd24beb6a0a061dedf42b26a993176745f6b)

• 使用docker-py

[root@docker ~]# cat 111.py#!/usr/bin/env pythonimport jsonimport dockerimport sysimport timecli=docker.Client(base_url='unix://var/run/docker.sock')events=cli.events(since=time.time())for e in events:    print(e.decode())    [root@docker ~]# python3 111.py{"status":"destroy","id":"1dfd11ab934aae34ca62be0ff519f272784ea5b60299e045cf495d6715c01903","from":"bac","Type":"container","Action":"destroy","Actor":{"ID":"1dfd11ab934aae34ca62be0ff519f272784ea5b60299e045cf495d6715c01903","Attributes":{"image":"bac","name":"upbeat_hugle"}},"time":1589459334,"timeNano":1589459334373629869}

使用docker logs命令获取容器的日志

• 在容器所在的主机上访问这个进程的日志
• 使用docker logs命令,通过-f获得一个持续输出的日志流

[root@docker ~]# docker logs -f rancher-agent Found container ID: a551cf12eac99b78808d43b165b7366583bf6a2804e05f1465ba08b84e495a18Checking root: /host/run/runcChecking file: 4202ac2f9a86003d9bfb7e1970972d7157df0a7c4ed40267a446d4c86d4f577cChecking file: a551cf12eac99b78808d43b165b7366583bf6a2804e05f1465ba08b84e495a18Found state.json: a551cf12eac99b78808d43b165b7366583bf6a2804e05f1465ba08b84e495a18time="2020-05-14T12:15:59Z" level=info msg="Execing [/usr/bin/nsenter --mount=/proc/7216/ns/mnt -F -- /var/lib/docker/overlay2/cc95229315d87f383653031597e09f89960f1f93d7a57d7a55d114afc5ad1cf0/merged/usr/bin/share-mnt --stage2 /var/lib/rancher/volumes /var/lib/kubelet -- norun]" INFO: Starting agent for CC0E75A94ABFD3800AF3

• 使用docker top命令来监控容器内运行的进程

[root@docker ~]# docker top nginxUID     PID     PPID    C    STIME   TTY     TIME     CMDroot   9587     9568    1    20:34    ?     00:00:00  nginx: master process nginx -g daemon off;101    9613     9587    0    20:34    ?     00:00:00  nginx: worker process

使用Logspout采集容器日志

• 从不同主机上运行的容器收集日志并进行聚合
• Logspout可以收集一台Docker主机上的所有容器日志并把它们路由到其他主机.它以容器的方式运行,并且是完全无状态的

# 在10.0.0.180服务器上[root@docker ~]# docker pull nginx[root@docker ~]# docker pull gliderlabs/logspout[root@docker ~]# docker run -d --name webserver -p 80:80 nginx[root@docker ~]# docker run -d --name logspout -v /var/run/docker.sock:/tmp/docker.sock docker.io/gliderlabs/logspout syslog://10.0.0.181:5000# 为了收集日志,需要在第二台主机(10.0.0.181)上运行Logstash容器.在UDP5000端口上监听syslog输入,并全部输出到同一台主机的标准输出中# 在10.0.0.181服务器上[root@docker02 ~]# cat DockerfileFROM ehazlett/logstashCOPY logstash.conf /etc/logstash.confENTRYPOINT ["/opt/logstash/bin/logstash"]# 镜像下载完后,需要构建自己的Logstash镜像并使用自定义Logstash配置文件[root@docker02 ~]# cat logstash.conf input {  udp {    port => 5000    type => syslog  }}filter {  if [type] == "syslog" {    grok {      match => { "message" => "%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{DATA:syslog_program}(?:\[%{POSINT:syslog_pid}\])?: %{GREEDYDATA:syslog_message}" }      add_field => [ "received_at", "%{@timestamp}" ]      add_field => [ "received_from", "%{host}" ]    }    syslog_pri { }    date {      match => [ "syslog_timestamp", "MMM  d HH:mm:ss", "MMM dd HH:mm:ss" ]    }  }}output {  stdout { codec => rubydebug }}# 构建镜像[root@docker02 ~]# docker build -t logstash .# 启动Logstash容器,将容器的5000端口绑定到宿主机的5000端口[root@docker02 ~]# docker run -d --name logstash -p 5000:5000/udp logstash -f /etc/logstash.conf# 打开浏览器访问第一个Docker主机上运行的Nginx,Logstash容器就能看到相应的日志输出[root@docker02 ~]# docker logs logstash {:timestamp=>"2020-05-14T13:28:01.798000+0000", :message=>"Using milestone 2 input plugin 'udp'. This plugin should be stable, but if you see strange behavior, please let us know! For more information on plugin milestones, see http://logstash.net/docs/1.4.2-modified/plugin-milestones", :level=>:warn}{:timestamp=>"2020-05-14T13:28:01.834000+0000", :message=>"Using milestone 1 filter plugin 'syslog_pri'. This plugin should work, but would benefit from use by folks like you. Please let us know if you find bugs or have suggestions on how to improve this plugin.  For more information on plugin milestones, see http://logstash.net/docs/1.4.2-modified/plugin-milestones", :level=>:warn}{                 "message" => "<11>1 2020-05-14T13:28:45Z bbb87d10eb4c logspout 11944 - - 2020/05/14 13:28:45 # logspout v3.2.11 by gliderlabs\n",                "@version" => "1",              "@timestamp" => "2020-05-14T13:28:45.497Z",                    "type" => "syslog",                    "host" => "10.0.0.180",                    "tags" => [        [0] "_grokparsefailure"    ],    "syslog_severity_code" => 5,    "syslog_facility_code" => 1,         "syslog_facility" => "user-level",         "syslog_severity" => "notice"

管理Logspout路由来存储容器日志

• 修改远程服务器的URI.直接在Logspout中查看日志来对容器进行调试,修改路由的URI,或者添加更多的路由URI

# 拉取一个带有curl命令的镜像[root@docker ~]# docker pull tutum/curl# 启动一个交互式容器[root@docker ~]# docker run -it --link logspout:logspout tutum/curl /bin/bash# 确认能ping同Logspout容器root@d5ae3aea7330:/# ping logspoutPING logspout (172.17.0.2) 56(84) bytes of data.64 bytes from logspout (172.17.0.2): icmp_seq=1 ttl=64 time=0.098 ms^C--- logspout ping statistics ---1 packets transmitted, 1 received, 0% packet loss, time 0msrtt min/avg/max/mdev = 0.098/0.098/0.098/0.000 ms# curl来访问Logspout容器的日志root@d5ae3aea7330:/# curl http://logspout:80/logs       webserver|10.0.0.1 - - [14/May/2020:13:47:15 +0000] "GET / HTTP/1.1" 304 0 "-" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3902.4 Safari/537.36" "-"

# 路由APIroot@d5ae3aea7330:/# curl http://logspout:80/routes[  {    "id": "7f9822b34ec1",    "adapter": "syslog",    "address": "10.0.0.181:5000"  }]root@d5ae3aea7330:/# curl http://logspout:80/routes/7f9822b34ec1{  "id": "7f9822b34ec1",  "adapter": "syslog",  "address": "10.0.0.181:5000"}# 删除路由root@d5ae3aea7330:/# curl -X DELETE http://logspout:80/routes/7f9822b34ec1root@d5ae3aea7330:/# curl http://logspout:80/routes []# 添加新的路由信息root@d5ae3aea7330:/# curl -X POST -d '{"id": "7f9822b34ec1","adapter": "syslog","address": "10.0.0.181:5000"}' http://logspout:80/routes{  "id": "7f9822b34ec1",  "adapter": "syslog",  "address": "10.0.0.181:5000"}root@d5ae3aea7330:/# curl -X POST -d '{"id": "g59822b34qw2","adapter": "syslog","address": "10.0.0.182:5000"}' http://logspout:80/routes{  "id": "g59822b34qw2",  "adapter": "syslog",  "address": "10.0.0.182:5000"}# 查看添加的路由信息root@d5ae3aea7330:/# curl http://logspout:80/routes[  {    "id": "7f9822b34ec1",    "adapter": "syslog",    "address": "10.0.0.181:5000"  },  {    "id": "g59822b34qw2",    "adapter": "syslog",    "address": "10.0.0.182:5000"  }]

使用Elasticsearch和Kibana对容器日志进行存储和可视化

• Kibana是一个仪表盘系统.允许你查询Elasticsearch中的索引并方便进行可视化.使用ehazlett/logstash镜像以及它的默认配置启动一个Logstash容器

[root@docker02 ~]# docker run --name es -d -p 9200:9200 -p 9300:9300 docker.io/ehazlett/elasticsearch[root@docker02 ~]# docker run --name kibana -d -p 80:80 docker.io/ehazlett/kibana[root@docker02 ~]# docker run -d --name logstash -p 5000:5000/udp --link es:elasticsearch docker.io/ehazlett/logstash -f /etc/logstash.conf.sample

• 访问10.0.0.181:80端口

image.png

• 访问10.0.0.180的80端口,日志就自动被存储到了Kibana

image.png

• 如果停止并删除了Elasticsearch容器,那么为溶出Logspout采集的日志流而创建的索引信息将会消失.可以挂载一个本地卷备份索引对Elasticsearch的数据进行持久化

使用Collected对容器指标进行可视化

• 监控容器的各种指标

• 在所有希望对容器进行监控的主机上运行该容器,将/var/run/docker.sock文件挂载到collectd的容器中,使用Collectd插件通过Docker统计API采集指标数据,并将指标数据发送到在其他主机上运行的Graphite仪表盘中

• 一台主机运行四个容器:一个Nginx容器用来标准输出输出测试日志,一个Logspout容器用于将所有标准输出日志发送到Logstash实例,一个容器用于产生模拟负载,以及一个Collectd容器

• 另一台主机也运行四个容器:一个Logstash容器用于接收来及Logstash的日志,一个Elasticsearch容器用于存储日志,一个Kibana容器用于对日志进行可视化,以及一个Graphite容器.Graphite容器也运行着carbon来存储指标数据

[root@docker ~]# vim docbook/ch09/collectd/Dockerfile FROM debian:jessieRUN apt-get update && apt-get -y install \    collectd \    python \    python-pipRUN apt-get cleanRUN pip install docker-pyRUN groupadd -r docker && useradd -r -g docker dockerADD docker-stats.py /opt/collectd/bin/docker-stats.pyADD docker-report.py /opt/collectd/bin/docker-report.pyADD collectd.conf /etc/collectd/collectd.confRUN chown -R docker /opt/collectd/binCMD ["/usr/sbin/collectd","-f"][root@docker ~]# vim docbook/ch09/collectd/worker.yml nginx:  image: nginx  ports:   - 80:80logspout:  image: gliderlabs/logspout  volumes:   - /var/run/docker.sock:/tmp/docker.sock  command: syslog://192.168.33.11:5000collectd:  build: .  volumes:   - /var/run/docker.sock:/var/run/docker.sockload:  image: borja/unixbench[root@docker ~]# vim docbook/ch09/collectd/monitor.yml es:  image: ehazlett/elasticsearch  ports:   - 9300:9300   - 9200:9200kibana:  image: ehazlett/kibana  ports:   - 8080:80graphite:  image: hopsoft/graphite-statsd  ports:   - 80:80   - 2003:2003   - 8125:8125/udplogstash:  image: ehazlett/logstash  ports:   - 5000:5000   - 5000:5000/udp  volumes:   - /vagrant/logstash.conf:/etc/logstash.conf  links:   - es:elasticsearch  command: -f /etc/logstash.conf    # 在被监控的主机上运行docker-compose -f docbook/ch09/collectd/worker.yml up -d# 在收集日志的主机上运行docker-compose -f docbook/ch09/collectd/monitor.yml up -d

使用cAdvisor监控容器资源使用状况

• cAdvisor是一个资源利用率监控系统.能用来监控容器资源使用情况和性能的软件.cAdvisor在Docker主机上以容器方式运行.通过挂载本地卷,它可以监控同一台主机上运行的所有容器.还提供了一个本地Web界面和API,并且能将数据存储到InfluxDB.将运行中容器的数据存储到远程InfluxDB集群,这样就可以对所有在一个集群中运行的容器的性能指标进行聚合

# 下载cAdvisor镜像和borja/unixbench镜像,borja/unixbench镜像用来模拟产生负载[root@docker ~]# docker pull google/cadvisor[root@docker ~]# docker pull borja/unixbench[root@docker ~]# docker run -v /var/run:/var/run:rw -v /sys:/sys:ro -v /var/lib/docker/:/var/lib/docker:ro -p 8080:8080 -d --name cadvisor docker.io/google/cadvisor[root@docker ~]# docker run -d docker.io/borja/unixbench

使用Weave Scope对容器布局进行可视化

[root@docker ~]# cat docbook/ch09/weavescope/docker-compose.ymldb1:  image: peterbourgon/tns-dbdb2:  image: peterbourgon/tns-db  links:    - db1db3:  image: peterbourgon/tns-db  links:    - db1    - db2app1:  image: peterbourgon/tns-app  links:    - db1    - db2    - db3app2:  image: peterbourgon/tns-app  links:    - db1    - db2    - db3lb1:  image: peterbourgon/tns-lb  links:    - app1    - app2  ports:    - 0.0.0.0:8001:80lb2:  image: peterbourgon/tns-lb  links:    - app1    - app2  ports:    - 0.0.0.0:8002:80    [root@docker ~]# docker-compose docbook/ch09/weavescope/docker-compose.yml

给这篇文章的作者打赏

微信扫一扫打赏

支付宝扫一扫打赏

×

打赏 点赞(0)