1"use strict";(globalThis.webpackChunkmikes_dev_notebook=globalThis.webpackChunkmikes_dev_notebook||[]).push([[13779],{76519(e,n,r){r.r(n),r.d(n,{assets:()=>c,contentTitle:()=>o,default:()=>h,frontMatter:()=>i,metadata:()=>s,toc:()=>l});const s=JSON.parse('{"id":"DevOps/Elasticsearch/2022-02-03--elasticsearch-v8-data-ingestion/index","title":"Elasticsearch v8, Filebeat (Docker) and NGINX","description":"Shenzhen, China","source":"@site/docs/DevOps/Elasticsearch/2022-02-03--elasticsearch-v8-data-ingestion/index.md","sourceDirName":"DevOps/Elasticsearch/2022-02-03--elasticsearch-v8-data-ingestion","slug":"/DevOps/Elasticsearch/2022-02-03--elasticsearch-v8-data-ingestion/2022-02-03","permalink":"/docs/DevOps/Elasticsearch/2022-02-03--elasticsearch-v8-data-ingestion/2022-02-03","draft":false,"unlisted":false,"editUrl":"https://github.com/facebook/docusaurus/tree/main/packages/create-docusaurus/templates/shared/docs/DevOps/Elasticsearch/2022-02-03--elasticsearch-v8-data-ingestion/index.md","tags":[{"inline":true,"label":"LINUX","permalink":"/docs/tags/linux"},{"inline":true,"label":"Elasticsearch","permalink":"/docs/tags/elasticsearch"}],"version":"current","sidebarPosition":7080,"frontMatter":{"sidebar_position":7080,"slug":"2022-02-03","title":"Elasticsearch v8, Filebeat (Docker) and NGINX","authors":"mpolinowski","tags":["LINUX","Elasticsearch"]},"sidebar":"tutorialSidebar","previous":{"title":"Elasticsearch & Kibana v8 Search Cheat Sheet","permalink":"/docs/DevOps/Elasticsearch/2022-02-04--kibana-v8-search-queries/2022-02-04"},"next":{"title":"Performing an Elasticsearch v8 Upgrade","permalink":"/docs/DevOps/Elasticsearch/2022-02-02--elasticsearch-v8-upgrade/2022-02-02"}}');var a=r(474848),t=r(28453);const i={sidebar_position:7080,slug:"2022-02-03",title:"Elasticsearch v8, Filebeat (Docker) and NGINX",authors:"mpolinowski",tags:["LINUX","Elasticsearch"]},o=void 0,c={},l=[{value:"Setting Up Elasticsearch & Kibana",id:"setting-up-elasticsearch--kibana",level:2},{value:"Setting up Filebeats",id:"setting-up-filebeats",level:2},{value:"Run the Filebeat Setup",id:"run-the-filebeat-setup",level:3},{value:"Configuration",id:"configuration",level:3},{value:"Add Kibana Dashboard",id:"add-kibana-dashboard",level:3},{value:"Enable and configure data collection modules",id:"enable-and-configure-data-collection-modules",level:3},{value:"Prepare the Filebeat Container",id:"prepare-the-filebeat-container",level:4},{value:"Check the NGINX Module",id:"check-the-nginx-module",level:4},{value:"Monitor NGINX in Docker",id:"monitor-nginx-in-docker",level:3},{value:"Alternative ?",id:"alternative-",level:4},{value:"Monitor Elasticsearch in Docker",id:"monitor-elasticsearch-in-docker",level:3}];function d(e){const n={a:"a",blockquote:"blockquote",code:"code",em:"em",h2:"h2",h3:"h3",h4:"h4",img:"img",li:"li",ol:"ol",p:"p",pre:"pre",strong:"strong",table:"table",tbody:"tbody",td:"td",th:"th",thead:"thead",tr:"tr",ul:"ul",...(0,t.R)(),...e.components};return(0,a.jsxs)(a.Fragment,{children:[(0,a.jsx)(n.p,{children:(0,a.jsx)(n.img,{alt:"Shenzhen, China",src:r(204891).A+"",width:"1500",height:"403"})}),"\n",(0,a.jsxs)(n.ul,{children:["\n",(0,a.jsx)(n.li,{children:(0,a.jsx)(n.a,{href:"#setting-up-elasticsearch--kibana",children:"Setting Up Elasticsearch & Kibana"})}),"\n",(0,a.jsxs)(n.li,{children:[(0,a.jsx)(n.a,{href:"#setting-up-filebeats",children:"Setting up Filebeats"}),"\n",(0,a.jsxs)(n.ul,{children:["\n",(0,a.jsx)(n.li,{children:(0,a.jsx)(n.a,{href:"#run-the-filebeat-setup",children:"Run the Filebeat Setup"})}),"\n",(0,a.jsx)(n.li,{children:(0,a.jsx)(n.a,{href:"#configuration",children:"Configuration"})}),"\n",(0,a.jsx)(n.li,{children:(0,a.jsx)(n.a,{href:"#add-kibana-dashboard",children:"Add Kibana Dashboard"})}),"\n",(0,a.jsxs)(n.li,{children:[(0,a.jsx)(n.a,{href:"#enable-and-configure-data-collection-modules",children:"Enable and configure data collection modules"}),"\n",(0,a.jsxs)(n.ul,{children:["\n",(0,a.jsx)(n.li,{children:(0,a.jsx)(n.a,{href:"#prepare-the-filebeat-container",children:"Prepare the Filebeat Container"})}),"\n",(0,a.jsx)(n.li,{children:(0,a.jsx)(n.a,{href:"#check-the-nginx-module",children:"Check the NGINX Module"})}),"\n"]}),"\n"]}),"\n",(0,a.jsxs)(n.li,{children:[(0,a.jsx)(n.a,{href:"#monitor-nginx-in-docker",children:"Monitor NGINX in Docker"}),"\n",(0,a.jsxs)(n.ul,{children:["\n",(0,a.jsx)(n.li,{children:(0,a.jsx)(n.a,{href:"#alternative-",children:"Alternative ?"})}),"\n"]}),"\n"]}),"\n",(0,a.jsx)(n.li,{children:(0,a.jsx)(n.a,{href:"#monitor-elasticsearch-in-docker",children:"Monitor Elasticsearch in Docker"})}),"\n"]}),"\n"]}),"\n"]}),"\n",(0,a.jsx)(n.h2,{id:"setting-up-elasticsearch--kibana",children:"Setting Up Elasticsearch & Kibana"}),"\n",(0,a.jsxs)(n.p,{children:["I will use the ",(0,a.jsx)(n.a,{href:"/docs/DevOps/Elasticsearch/2022-02-02--elasticsearch-v8-upgrade/2022-02-02",children:"Elasticsearch compose file from here"}),":"]}),"\n",(0,a.jsx)(n.pre,{children:(0,a.jsx)(n.code,{className:"language-yml",children:'version: \'3.8\'\r\n\r\nservices:\r\n elasticsearch:\r\n container_name: elasticsearch\r\n restart: always\r\n build:\r\n context: elasticsearch/\r\n args:\r\n ELK_VERSION: $ELK_VERSION\r\n volumes:\r\n - type: bind\r\n source: ./elasticsearch/config/elasticsearch.yml\r\n target: /usr/share/elasticsearch/config/elasticsearch.yml\r\n read_only: true\r\n - type: volume\r\n source: elasticsearch\r\n target: /usr/share/elasticsearch/data\r\n ports:\r\n - "9200:9200"\r\n - "9300:9300"\r\n environment:\r\n ES_JAVA_OPTS: \'-Xms2g -Xmx2g\'\r\n ELASTIC_PASSWORD: \'changeme\'\r\n discovery.type: single-node\r\n networks:\r\n elastic:\r\n ipv4_address: 172.16.239.102\r\n aliases:\r\n - elasticsearch\r\n\r\n kibana:\r\n container_name: kibana\r\n restart: unless-stopped\r\n build:\r\n context: kibana/\r\n args:\r\n ELK_VERSION: $ELK_VERSION\r\n volumes:\r\n - type: bind\r\n source: ./kibana/config/kibana.yml\r\n target: /usr/share/kibana/config/kibana.yml\r\n read_only: true\r\n ports:\r\n - "5601:5601"\r\n networks:\r\n elastic:\r\n ipv4_address: 172.16.239.104\r\n aliases:\r\n - kibana\r\n depends_on:\r\n - elasticsearch\r\n\r\nnetworks:\r\n elastic:\r\n driver: bridge\r\n driver_opts:\r\n com.docker.network.enable_ipv6: "false"\r\n ipam:\r\n driver: default\r\n config:\r\n - subnet: 172.16.239.0/24\r\n\r\nvolumes:\r\n elasticsearch:\n'})}),"\n",(0,a.jsxs)(n.p,{children:["Start the service with ",(0,a.jsx)(n.code,{children:"docker-compose up -d"}),". Elasticsearch will start with the following configuration:"]}),"\n",(0,a.jsx)(n.pre,{children:(0,a.jsx)(n.code,{className:"language-yml",children:'---\r\n## Default Elasticsearch configuration from Elasticsearch base image.\r\n## https://github.com/elastic/elasticsearch/blob/master/distribution/docker/src/docker/config/elasticsearch.yml\r\n#\r\ncluster.name: "docker-cluster"\r\n# network.host: _site_\r\nnetwork.host: 0.0.0.0\r\n\r\n## X-Pack settings\r\n## see https://www.elastic.co/guide/en/elasticsearch/reference/current/setup-xpack.html\r\n#\r\n# xpack.license.self_generated.type: trial\r\nxpack.license.self_generated.type: basic\r\nxpack.security.enabled: true\r\nxpack.monitoring.collection.enabled: true\r\nxpack.security.authc:\r\n anonymous:\r\n username: anonymous_user \r\n roles: search_agent\r\n authz_exception: true \r\n\r\n## CORS\r\nhttp.cors.enabled : true\r\nhttp.cors.allow-origin: "*"\r\nhttp.cors.allow-methods: OPTIONS, HEAD, GET, POST, PUT, DELETE\r\nhttp.cors.allow-credentials: true\r\nhttp.cors.allow-headers: X-Requested-With, X-Auth-Token, Content-Type, Content-Length, Authorization, Access-Control-Allow-Headers, Accept\n'})}),"\n",(0,a.jsx)(n.h2,{id:"setting-up-filebeats",children:"Setting up Filebeats"}),"\n",(0,a.jsx)(n.p,{children:"Start by pulling the a fresh version of Filebeat:"}),"\n",(0,a.jsx)(n.pre,{children:(0,a.jsx)(n.code,{className:"language-bash",children:"docker pull elastic/filebeat:8.0.0\n"})}),"\n",(0,a.jsx)(n.h3,{id:"run-the-filebeat-setup",children:"Run the Filebeat Setup"}),"\n",(0,a.jsx)(n.p,{children:"Running Filebeat with the setup command will create the index pattern and load visualizations , dashboards, and machine learning jobs. Run this command:"}),"\n",(0,a.jsxs)(n.blockquote,{children:["\n",(0,a.jsxs)(n.p,{children:[(0,a.jsx)(n.strong,{children:"Note"}),": If you set up Elasticsearch according to ",(0,a.jsx)(n.a,{href:"/docs/DevOps/Elasticsearch/2022-02-02--elasticsearch-v8-upgrade/2022-02-02",children:"this guide"}),", you will have a different ",(0,a.jsx)(n.code,{children:"elastic"})," user password - e.g. ",(0,a.jsx)(n.code,{children:"ELASTIC_PASSWORD: 'a1hyme+ry1-AltBfpqxY'"}),"."]}),"\n"]}),"\n",(0,a.jsx)(n.pre,{children:(0,a.jsx)(n.code,{className:"language-bash",children:'docker run \\\r\n-E setup.kibana.host=kibana:5601 \\\r\n-E output.elasticsearch.hosts=["elasticsearch:9200"] \\\r\n-E output.elasticsearch.username=elastic \\\r\n-E output.elasticsearch.password=changeme \\\r\nelastic/filebeat:8.0.0\n'})}),"\n",(0,a.jsxs)(n.p,{children:[(0,a.jsx)(n.strong,{children:"ERROR"}),": This step did not work for me. I keep getting the error message ",(0,a.jsx)(n.code,{children:"no matches found: output.elasticsearch.hosts=[elasticsearch:9200]"}),". I already added the ",(0,a.jsx)(n.code,{children:"username"})," and ",(0,a.jsx)(n.code,{children:"password"})," that were missing in the ",(0,a.jsx)(n.a,{href:"https://www.elastic.co/guide/en/beats/filebeat/current/running-on-docker.html#_run_the_filebeat_setup",children:"official documentation"}
1),". Is this step still necessary in version 8 since the new ",(0,a.jsx)(n.strong,{children:"Datastream"})," tab is automatically populated with the Filebeat data? I am missing the default Filebeat dashboard though - if there is supposed to be one with that name."]}),"\n",(0,a.jsxs)(n.p,{children:[(0,a.jsx)(n.strong,{children:"UPDATE"}),": You can manually activate the Kibana integration once you started the container - see ",(0,a.jsx)(n.a,{href:"#add-kibana-dashboard",children:"Add Kibana Dashboard"}),". I don't think this step is still necessary:"]}),"\n",(0,a.jsx)(n.p,{children:"I will create a folder:"}),"\n",(0,a.jsx)(n.pre,{children:(0,a.jsx)(n.code,{className:"language-bash",children:"mkdir -p /opt/beats/config/\n"})}),"\n",(0,a.jsx)(n.p,{children:"and continue working from there."}),"\n",(0,a.jsx)(n.h3,{id:"configuration",children:"Configuration"}),"\n",(0,a.jsx)(n.p,{children:"When running Filebeat in a container, you need to provide access to Docker\u2019s unix socket in order for the add_docker_metadata processor to work. You can do this by mounting the socket inside the container. For example:"}),"\n",(0,a.jsxs)(n.blockquote,{children:["\n",(0,a.jsx)(n.p,{children:"docker run -v /var/run/docker.sock:/var/run/docker.sock ..."}),"\n"]}),"\n",(0,a.jsx)(n.p,{children:"To avoid privilege issues, you may also need to add --user=root to the docker run flags. Because the user must be part of the docker group in order to access /var/run/docker.sock, root access is required if Filebeat is running as non-root inside the container."}),"\n",(0,a.jsx)(n.p,{children:"If Docker daemon is restarted the mounted socket will become invalid and metadata will stop working, in these situations there are two options:"}),"\n",(0,a.jsxs)(n.ul,{children:["\n",(0,a.jsx)(n.li,{children:"Restart Filebeat every time Docker is restarted"}),"\n",(0,a.jsx)(n.li,{children:"Mount the entire /var/run directory (instead of just the socket)"}),"\n"]}),"\n",(0,a.jsx)(n.pre,{children:(0,a.jsx)(n.code,{className:"language-yml",children:'processors:\r\n - add_docker_metadata:\r\n host: "unix:///var/run/docker.sock"\r\n #match_fields: ["system.process.cgroup.id"]\r\n #match_pids: ["process.pid", "process.parent.pid"]\r\n #match_source: true\r\n #match_source_index: 4\r\n #match_short_id: true\r\n #cleanup_timeout: 60\r\n #labels.dedot: false\r\n # To connect to Docker over TLS you must specify a client and CA certificate.\r\n #ssl:\r\n # certificate_authority: "/etc/pki/root/ca.pem"\r\n # certificate: "/etc/pki/client/cert.pem"\r\n # key: \n'})}),"\n",(0,a.jsx)(n.p,{children:"It has the following settings:"}),"\n",(0,a.jsxs)(n.table,{children:[(0,a.jsx)(n.thead,{children:(0,a.jsxs)(n.tr,{children:[(0,a.jsx)(n.th,{}),(0,a.jsx)(n.th,{})]})}),(0,a.jsxs)(n.tbody,{children:[(0,a.jsxs)(n.tr,{children:[(0,a.jsx)(n.td,{children:"host"}),(0,a.jsxs)(n.td,{children:[(0,a.jsx)(n.em,{children:"(Optional)"})," Docker socket (UNIX or TCP socket). It uses unix:///var/run/docker.sock by default."]})]}),(0,a.jsxs)(n.tr,{children:[(0,a.jsx)(n.td,{children:"ssl"}),(0,a.jsxs)(n.td,{children:[(0,a.jsx)(n.em,{children:"(Optional)"})," SSL configuration to use when connecting to the Docker socket."]})]}),(0,a.jsxs)(n.tr,{children:[(0,a.jsx)(n.td,{children:"match_fields"}),(0,a.jsxs)(n.td,{children:[(0,a.jsx)(n.em,{children:"(Optional)"})," A list of fields to match a container ID, at least one of them should hold a container ID to get the event enriched."]})]}),(0,a.jsxs)(n.tr,{children:[(0,a.jsx)(n.td,{children:"match_pids"}),(0,a.jsxs)(n.td,{children:[(0,a.jsx)(n.em,{children:"(Optional)"}),' A list of fields that contain process IDs. If the process is running in Docker then the event will be enriched. The default value is ["process.pid", "process.parent.pid"].']})]}),(0,a.jsxs)(n.tr,{children:[(0,a.jsx)(n.td,{children:"match_source"}),(0,a.jsxs)(n.td,{children:[(0,a.jsx)(n.em,{children:"(Optional)"})," Match container ID from a log path present in the log.file.path field. Enabled by default."]})]}),(0,a.jsxs)(n.tr,{children:[(0,a.jsx)(n.td,{children:"match_short_id"}),(0,a.jsxs)(n.td,{children:[(0,a.jsx)(n.em,{children:"(Optional)"})," Match container short ID from a log path present in the log.file.path field. Disabled by default. This allows to match directories names that have the first 12 characters of the container ID. For example, /var/log/containers/b7e3460e2b21/*.log."]})]}),(0,a.jsxs)(n.tr,{children:[(0,a.jsx)(n.td,{children:"match_source_index"}),(0,a.jsxs)(n.td,{children:[(0,a.jsx)(n.em,{children:"(Optional)"}
1)," Index in the source path split by / to look for container ID. It defaults to 4 to match /var/lib/docker/containers/<container_id>/*.log"]})]}),(0,a.jsxs)(n.tr,{children:[(0,a.jsx)(n.td,{children:"cleanup_timeout"}),(0,a.jsxs)(n.td,{children:[(0,a.jsx)(n.em,{children:"(Optional)"})," Time of inactivity to consider we can clean and forget metadata for a container, 60s by default."]})]}),(0,a.jsxs)(n.tr,{children:[(0,a.jsx)(n.td,{children:"labels.dedot"}),(0,a.jsxs)(n.td,{children:[(0,a.jsx)(n.em,{children:"(Optional)"})," Default to be false. If set to true, replace dots in labels with _."]})]})]})]}),"\n",(0,a.jsx)(n.p,{children:"But I think I am just going to use CLI flags to mount the docker socket as volumes. This simplifies the configuration to:"}),"\n",(0,a.jsx)(n.pre,{children:(0,a.jsx)(n.code,{className:"language-bash",children:"nano /opt/beats/config/filebeat.yml\n"})}),"\n",(0,a.jsx)(n.pre,{children:(0,a.jsx)(n.code,{className:"language-yml",children:"filebeat.config:\r\n modules:\r\n path: ${path.config}/modules.d/*.yml # enable all modules (nginx, kafka, redis, etc)\r\n reload.enabled: false\r\n\r\nfilebeat.autodiscover: # auto-discover tagged docker container\r\n providers:\r\n - type: docker\r\n hints.enabled: true\r\n\r\nsetup:\r\n kibana.host: \"http://localhost:5601\"\r\n dashboards.enable: true\r\n\r\n# processors:\r\n# - add_cloud_metadata: ~ # for AWS, GCO, Azure etc.\r\n# - add_docker_metadata: ~ # add docker metadata (container id, name, image and labels)\r\n\r\noutput.elasticsearch:\r\n hosts: 'http://localhost:9200'\r\n username: 'elastic'\r\n password: 'changeme'\n"})}),"\n",(0,a.jsxs)(n.blockquote,{children:["\n",(0,a.jsxs)(n.p,{children:[(0,a.jsx)(n.strong,{children:"Note"}),": If you set up Elasticsearch according to ",(0,a.jsx)(n.a,{href:"/docs/DevOps/Elasticsearch/2022-02-02--elasticsearch-v8-upgrade/2022-02-02",children:"this guide"}),", you will have a different ",(0,a.jsx)(n.code,{children:"elastic"})," user password - e.g. ",(0,a.jsx)(n.code,{children:"ELASTIC_PASSWORD: 'a1hyme+ry1-AltBfpqxY'"}),"."]}),"\n"]}),"\n",(0,a.jsxs)(n.p,{children:["The beat configuration file must belong to the ",(0,a.jsx)(n.code,{children:"root"})," user and all write permissions for other users must be revoked:"]}),"\n",(0,a.jsx)(n.pre,{children:(0,a.jsx)(n.code,{className:"language-bash",children:"chown root:root /opt/beats/config/filebeat.yml\r\nchmod go-w /opt/beats/config/filebeat.yml\n"})}),"\n",(0,a.jsxs)(n.p,{children:["Now we need to bind the location of docker container directory ",(0,a.jsx)(n.code,{children:"/var/lib/docker/containers"})," and our docker socket ",(0,a.jsx)(n.code,{children:"/var/run/docker.sock"})," to the container:"]}),"\n",(0,a.jsx)(n.pre,{children:(0,a.jsx)(n.code,{className:"language-bash",children:"docker run -d \\\r\n --name filebeat \\\r\n --user root \\\r\n --net=host \\\r\n -v /opt/beats/config/filebeat.yml:/usr/share/filebeat/filebeat.yml \\\r\n -v /var/lib/docker/containers:/var/lib/docker/containers:ro \\\r\n -v /var/run/docker.sock:/var/run/docker.sock:ro \\\r\n elastic/filebeat:8.0.0\n"})}),"\n",(0,a.jsx)(n.p,{children:"Check if everything started up:"}),"\n",(0,a.jsx)(n.pre,{children:(0,a.jsx)(n.code,{className:"language-bash",children:'{"log.level":"info","@timestamp":"2022-02-21T04:46:17.294Z","log.origin":{"file.name":"instance/beat.go","file.line":332},"message":"Setup Beat: filebeat; Version: 8.0.0","service.name":"filebeat","ecs.version":"1.6.0"}\r\n{"log.level":"info","@timestamp":"2022-02-21T04:46:17.295Z","log.logger":"esclientleg","log.origin":{"file.name":"eslegclient/connection.go","file.line":105},"message":"elasticsearch url: http://localhost:9200","service.name":"filebeat","ecs.version":"1.6.0"}\r\n{"log.level":"info","@timestamp":"2022-02-21T04:46:17.296Z","log.logger":"publisher","log.origin":{"file.name":"pipeline/module.go","file.line":113},"message":"Beat name: nomad-minion","service.name":"filebeat","ecs.version":"1.6.0"}\r\n{"log.level":"info","@timestamp":"2022-02-21T04:46:17.298Z","log.origin":{"file.name":"fileset/modules.go","file.line":103},"message":"Enabled modules/filesets: ()","service.name":"filebeat","ecs.version":"1.6.0"}\r\n{"log.level":"info","@timestamp":"2022-02-21T04:46:17.298Z","log.logger":"monitoring","log.origin":{"file.name":"log/log.go","file.line":142},"message":"Starting metrics logging every 30s","service.name":"filebeat","ecs.version":"1.6.0"}\n'})}),"\n",(0,a.jsx)(n.h3,{id:"add-kibana-dashboard",children:"Add Kibana Dashboard"}),"\n",(0,a.jsxs)(n.p,{children:[(0,a.jsx)(n.strong,{children:"UPDATE"}),": You can already setup the dashboards using the Filebeat config with ",(0,a.jsx)(n.code,{children:"setup.dashboards.enable: true"})," I added this line to the configuration file above!"]}
1),"\n",(0,a.jsx)(n.p,{children:"To load the recommended index template for writing to Elasticsearch and deploy the sample dashboards for visualizing the data in Kibana, use the command that works with your system."}),"\n",(0,a.jsx)(n.pre,{children:(0,a.jsx)(n.code,{className:"language-bash",children:"docker exec -ti filebeat /bin/bash\r\n\r\n\r\n./filebeat setup --dashboards\r\nLoading dashboards (Kibana must be running and reachable)\r\nLoaded dashboards\n"})}),"\n",(0,a.jsxs)(n.p,{children:["The Filebeat ",(0,a.jsx)(n.strong,{children:"Data View"})," is now listed in Kibana:"]}),"\n",(0,a.jsx)(n.p,{children:(0,a.jsx)(n.img,{alt:"Elasticsearch v8 Filebeats",src:r(939364).A+"",width:"1176",height:"595"})}),"\n",(0,a.jsxs)(n.p,{children:["I can see results come in in ",(0,a.jsx)(n.strong,{children:"Discover"}),":"]}),"\n",(0,a.jsx)(n.p,{children:(0,a.jsx)(n.img,{alt:"Elasticsearch v8 Filebeats",src:r(852927).A+"",width:"1432",height:"536"})}),"\n",(0,a.jsxs)(n.p,{children:["There are also plenty of ",(0,a.jsx)(n.code,{children:"Filebeat*"})," ",(0,a.jsx)(n.strong,{children:"Dashboards"})," loaded. But so far no interesting data to fill them with."]}),"\n",(0,a.jsx)(n.h3,{id:"enable-and-configure-data-collection-modules",children:"Enable and configure data collection modules"}),"\n",(0,a.jsx)(n.h4,{id:"prepare-the-filebeat-container",children:"Prepare the Filebeat Container"}),"\n",(0,a.jsxs)(n.p,{children:["Since we are running Filebeat in Docker, of course this log path does not exist. We have to modify the command that we used to start the Docker container to mount our NGINX logs into the container. The NGINX logs might be found in the ",(0,a.jsx)(n.code,{children:"/var/log/nginx"})," directory - depending on your NGINX configuration. But I just copied a few logs onto my test system ",(0,a.jsx)(n.code,{children:"/opt/beats/logs"}),":"]}),"\n",(0,a.jsx)(n.pre,{children:(0,a.jsx)(n.code,{className:"language-bash",children:"-v /opt/beats/logs:/var/log/nginx:ro\n"})}),"\n",(0,a.jsxs)(n.p,{children:["And secondly, we need to mount our module configuration file. The template configuration is located inside the Filebeat container under ",(0,a.jsx)(n.code,{children:"/usr/share/filebeat/modules.d/nginx.yml.disabled"}),":"]}),"\n",(0,a.jsx)(n.pre,{children:(0,a.jsx)(n.code,{className:"language-yml",children:'# Module: nginx\r\n# Docs: https://www.elastic.co/guide/en/beats/filebeat/master/filebeat-module-nginx.html\r\n\r\n- module: nginx\r\n # Access logs\r\n access:\r\n enabled: true\r\n var.paths: ["/var/log/nginx/access.log*"]\r\n\r\n # Error logs\r\n error:\r\n enabled: true\r\n var.paths: [ "/var/log/nginx/error.log*" ]\r\n\r\n # Ingress-nginx controller logs. This is disabled by default. It could be used in Kubernetes environments to parse ingress-nginx logs\r\n ingress_controller:\r\n enabled: false\r\n\r\n # Set custom paths for the log files. If left empty,\r\n # Filebeat will choose the paths depending on your OS.\r\n #var.paths:\n'})}),"\n",(0,a.jsxs)(n.p,{children:["save this file under ",(0,a.jsx)(n.code,{children:"nginx.yml"})," next to your ",(0,a.jsx)(n.code,{children:"filebeat.yml"})," and mount it into the ",(0,a.jsx)(n.code,{children:"modules.d"})," configuration folder - the complete docker command now looks like this:"]}),"\n",(0,a.jsx)(n.pre,{children:(0,a.jsx)(n.code,{className:"language-bash",children:"docker run -d \\\r\n --name filebeat \\\r\n --user root \\\r\n --net=host \\\r\n --restart unless-stopped \\\r\n -v /opt/beats/config/filebeat.yml:/usr/share/filebeat/filebeat.yml \\\r\n -v /opt/beats/config/nginx.yml:/usr/share/filebeat/modules.d/nginx.yml \\\r\n -v /var/lib/docker/containers:/var/lib/docker/containers:ro \\\r\n -v /var/run/docker.sock:/var/run/docker.sock:ro \\\r\n -v /opt/beats/logs:/var/log/nginx:ro \\\r\n elastic/filebeat:8.0.0\n"})}),"\n",(0,a.jsx)(n.p,{children:"Restart the container and verify that the logs and module configuration was actually mounted:"}),"\n",(0,a.jsx)(n.pre,{children:(0,a.jsx)(n.code,{className:"language-bash",children:"docker exec -ti filebeat /bin/bash\r\n\r\n/usr/share/filebeat# ls -la /var/log/nginx/\r\n-rwxrw
1xrwx 1 root root 9634840 Feb 21 03:47 access.log\r\n-rwxrwxrwx 1 root root 12225 Feb 21 03:47 error.log\r\n\r\nls -la /usr/share/filebeat/modules.d | grep nginx\r\n-rw-r--r-- 1 root root 613 Feb 22 05:54 nginx.yml\r\n-rw-r--r-- 1 root root 788 Feb 3 18:06 nginx.yml.disabled\n"})}),"\n",(0,a.jsx)(n.h4,{id:"check-the-nginx-module",children:"Check the NGINX Module"}),"\n",(0,a.jsxs)(n.ol,{children:["\n",(0,a.jsx)(n.li,{children:"Verify that the NGINX modules was actually enable. To see a list of available modules, run:"}),"\n"]}),"\n",(0,a.jsx)(n.pre,{children:(0,a.jsx)(n.code,{className:"language-bash",children:"docker exec -ti filebeat /bin/bash\r\n\r\n\r\n/usr/share/filebeat# ./filebeat modules list\r\nEnabled:\r\nnginx\r\n\r\nDisabled:\r\nactivemq apache auditd aws awsfargate azure barracuda bluecoat cef checkpoint cisco coredns crowdstrike cyberarkpas cylance elasticsearch envoyproxy f5 fortinet gcp google_workspace haproxy ibmmq icinga iis imperva infoblox iptables juniper kafka kibana logstash microsoft misp mongodb mssql mysql mysqlenterprise nats netflow netscout nginx o365 okta oracle osquery panw pensando postgresql proofpoint rabbitmq radware redis santa snort snyk sonicwall sophos squid suricata system threatintel tomcat traefik zeek zookeeper zoom zscaler\n"})}),"\n",(0,a.jsx)(n.p,{children:"To manually activate or deactivate modules run:"}),"\n",(0,a.jsx)(n.pre,{children:(0,a.jsx)(n.code,{className:"language-bash",children:"./filebeat modules enable nginx\n"})}),"\n",(0,a.jsx)(n.p,{children:"To test your configuration file, change to the directory where the Filebeat binary is installed, and run Filebeat in the foreground with the following options specified:"}),"\n",(0,a.jsx)(n.pre,{children:(0,a.jsx)(n.code,{className:"language-bash",children:"./filebeat test config -e\r\nConfig OK\n"})}),"\n",(0,a.jsxs)(n.p,{children:["The documentation I found says that you now should run the setup command to load the available dashboards. I am not sure if this is still necessary since I already did this in the previous step. But running the command returns a ",(0,a.jsx)(n.code,{children:"Loaded Ingest pipelines"})," - sounds good ~"]}),"\n",(0,a.jsx)(n.pre,{children:(0,a.jsx)(n.code,{className:"language-bash",children:"./filebeat setup -e\r\nLoaded Ingest pipelines\n"})}),"\n",(0,a.jsxs)(n.p,{children:["I can see all the NGINX related data points in ",(0,a.jsx)(n.strong,{children:"Data Views"}),":"]}),"\n",(0,a.jsx)(n.p,{children:(0,a.jsx)(n.img,{alt:"Elasticsearch v8 Filebeats",src:r(74838).A+"",width:"1376",height:"859"})}),"\n",(0,a.jsxs)(n.p,{children:["Switching to the ",(0,a.jsx)(n.strong,{children:"Discover"})," I first cannot see anything. But I have to select a time window that matches the date in my logs - by default you only see the last 15 minutes. Drilling in I find my log data from yesterday:"]}),"\n",(0,a.jsx)(n.p,{children:(0,a.jsx)(n.img,{alt:"Elasticsearch v8 Filebeats",src:r(965449).A+"",width:"1383",height:"371"})}),"\n",(0,a.jsxs)(n.p,{children:["Under ",(0,a.jsx)(n.strong,{children:"Dashboards"})," I now open the NGINX Access & Error Log template:"]}),"\n",(0,a.jsx)(n.p,{children:(0,a.jsx)(n.img,{alt:"Elasticsearch v8 Filebeats",src:r(994272).A+"",width:"1277",height:"524"})}),"\n",(0,a.jsx)(n.p,{children:"And can start analyzing my data:"}),"\n",(0,a.jsx)(n.p,{children:(0,a.jsx)(n.img,{alt:"Elasticsearch v8 Filebeats",src:r(349307).A+"",width:"1375",height:"875"})}),"\n",(0,a.jsx)(n.p,{children:(0,a.jsx)(n.img,{alt:"Elasticsearch v8 Filebeats",src:r(600306).A+"",width:"2550",height:"886"})}),"\n",(0,a.jsx)(n.h3,{id:"monitor-nginx-in-docker",children:"Monitor NGINX in Docker"}),"\n",(0,a.jsx)(n.p,{children:"Configure filebeat to read the StdOut und StdErr stream of your NGINX container:"}),"\n",(0,a.jsx)(n.pre,{children:(0,a.jsx)(n.code,{className:"language-yml",children:'filebeat.autodiscover:\r\n providers:\r\n - type: docker\r\n hints.enabled: true\r\n templates:\r\n - condition.contains:\r\n docker.container.image: nginx\r\n config:\r\n - module: nginx\r\n access.input:\r\n type: docker\r\n containers:\r\n ids: "${data.docker.container.id}"\r\n stream: "stdout"\r\n error.input:\r\n type: docker\r\n containers:\r\n ids: "${data.docker.container.id}"\r\n stream: "stderr"\n'})}),"\n",(0,a.jsx)(n.p,{children:"Run the Filebeat container inside the same Docker network as your NGINX container:"}),"\n",(0,a.jsx)(n.pre,{children:(0,a.jsx)(n.code,{className:"language-bash",children:"docker run -d \\\r\n --name filebeat \\\r\n --user root \\\r\n --network mynetwork \\\r\n --restart unless-stopped \\\r\n -v /opt/beats/config/filebeat.yml:/usr/share/filebeat/filebeat.yml \\\r\n -v /var/lib/docker/containers:/var/lib/docker/containers:ro \\\r\n -v /var/run/docker.sock:/var/run/docker.sock:ro \\\r\n elastic/filebeat:8.0.0\n"})}),"\n",(0,a.jsxs)(n.p,{children:["Since the container is now wrapped inside the virtual Docker network, I am now seeing the ",(0,a.jsx)(n.strong,{children:"Filebeat Container ID"})," as ",(0,a.jsx)(n.code,{children:"host.name"}),". In my case ",(0,a.jsx)(n.code,{children:"b803b794fcb2"}),":"]}),"\n",(0,a.jsx)(n.pre,{children:(0,a.jsx)(n.code,{className:"language-bash",children:"docker ps\r\nCONTAINER ID IMAGE\r\nb803b794fcb2 elastic/filebeat:8.0.0\n"})}),"\n",(0,a.jsx)(n.p,{children:"Which I can use to filter all log entries from this beat:"}),"\n",(0,a.jsx)(n.pre,{children:(0,a.jsx)(n.code,{className:"language-bash",children:'host.name : "b803b794fcb2"\n'})}),"\n",(0,a.jsx)(n.p,{children:(0,a.jsx)(n.img,{alt:"Elasticsearch v8 Filebeats",src:r(835957).A+"",width:"1216",height:"556"})}),"\n",(0,a.jsx)(n.h4,{id:"alternative-",children:"Alternative ?"}),"\n",(0,a.jsxs)(n.p,{children:[(0,a.jsx)(n.strong,{children:"Untested"}),": Give the filepath to your container logs:"]}),"\n",(0,a.jsx)(n.pre,{children:(0,a.jsx)(n.code,{className:"language-yml",children:'# # =========================== Filebeat autodiscover ============================\r\n\r\nfilebeat.autodiscover:\r\n providers:\r\n - type: docker\r\n templates:\r\n - condition:\r\n contains:\r\n docker.container.image:<your_label_condition>\r\n config:\r\n - type: container\r\n paths:\r\n - "/var/lib/docker/containers/${data.docker.container.id}/*.log"\r\n exclude_lines: ["^\\\\s+[\\\\-`(\'.|_]"] # drop asciiart lines\r\n\r\nfilebeat.shutdown_timeout: 5s #optional\r\n\r\n# ------------------------------- Console Output -------------------------------\r\noutput.console:\r\n enabled: true\r\n codec.json:\r\n pretty: true\r\n escape_html: false \r\n\r\nlogging.metrics.enabled: false\n'})}),"\n",(0,a.jsx)(n.h3,{id:"monitor-elasticsearch-in-docker",children:"Monitor Elasticsearch in Docker"}),"\n",(0,a.jsx)(n.p,{children:"I can now add my Elasticsearch container to the Filebeat configuration file:"}),"\n",(0,a.jsx)(n.pre,{children:(0,a.jsx)(n.code,{className:"language-yml",children:'filebeat.autodiscover\r\n providers:\r\n - type: docker\r\n hints.enabled: true\r\n templates:\r\n - condition.contains:\r\n docker.container.image: nginx\r\n config:\r\n - module: nginx\r\n access.input:\r\n type: docker\r\n containers:\r\n ids: "${data.docker.container.id}"\r\n stream: "stdout"\r\n error.input:\r\n type: docker\r\n containers:\r\n ids: "${data.docker.container.id}"\r\n stream: "stderr"\r\n - condition.contains:\r\n docker.container.image: docker.elastic.co/elasticsearch/elasticsearch\r\n config:\r\n - module: elasticsearch\r\n access.input:\r\n type: docker\r\n containers:\r\n ids: "${data.docker.container.id}"\r\n stream: "stdout"\r\n error.input:\r\n type: docker\r\n containers:\r\n ids: "${data.docker.container.id}"\r\n stream: "stderr"\n'})}),"\n",(0,a.jsxs)(n.p,{children:["Since the Elasticsearch container name is set to ",(0,a.jsx)(n.code,{children:"elasticsearch"})," I can now filter the Filebeat index with:"]}),"\n",(0,a.jsx)(n.pre,{children:(0,a.jsx)(n.code,{className:"language-bash",children:'host.name : "b803b794fcb2" and container.name : "elasticsearch"\n'})}),"\n",(0,a.jsx)(n.p,{children:"And see all outgoing search responses:"}),"\n",(0,a.jsx)(n.p,{children:(0,a.jsx)(n.img,{alt:"Elasticsearch v8 Filebeats",src:r(269420).A+"",width:"1212",height:"498"})})]})}function h(e={}){const{wrapper:n}={...(0,t.R)(),...e.components};return n?(0,a.jsx)(n,{...e,children:(0,a.jsx)(d,{...e})}):d(e)}},939364(e,n,r){const s=r.p+"assets/images/Elasticsearch8_Filebeats_01-2d57922b0b28d119ec3fc314114b0ba9.png";r.d(n,["A",0,s])},852927(e,n,r){const s=r.p+"assets/images/Elasticsearch8_Filebeats_02-e825ec315c47136e111506df001123e7.png";r.d(n,["A",0,s])},74838(e,n,r){const s=r.p+"assets/images/Elasticsearch8_Filebeats_03-18b92378e8187a27eeca6ac9c7482024.png";r.d(n,["A",0,s])},965449(e,n,r){const s=r.p+"assets/images/Elasticsearch8_Filebeats_04-20779b50fa72aa454a06b6e7ff2e15a9.png";r.d(n,["A",0,s])},994272(e,n,r){const s=r.p+"assets/images/Elasticsearch8_Filebeats_05-a939bf55798a05e698241c24f6056bd0.png";r.d(n,["A",0,s])},349307(e,n,r){const s=r.p+"assets/images/Elasticsearch8_Filebeats_06-996a5fb68f34307f07e91bb879dc7065.png";r.d(n,["A",0,s])},600306(e,n,r){const s=r.p+"assets/images/Elasticsearch8_Filebeats_07-01d3b6eb315bfbf7c839c10b25c83bd9.png";r.d(n,["A",0,s])},835957(e,n,r){const s=r.p+"assets/images/Elasticsearch8_Filebeats_08-ea92dcd7720dddcce30013cd8978a346.png";r.d(n,["A",0,s])},269420(e,n,r){const s=r.p+"assets/images/Elasticsearch8_Filebeats_09-dd481960c17b095f73627015eac50a53.png";r.d(n,["A",0,s])},204891(e,n,r){const s=r.p+"assets/images/photo-kt443t6d_64hdh43hfh6dgjdfhg4_d-94d62ec3ff079da54fce88aad4901707.jpg";r.d(n,["A",0,s])},28453(e,n,r){r.d(n,{R:()=>i,x:()=>o});var s=r(296540);const a={},t=s.createContext(a);function i(e){const n=s.useContext(t);return s.useMemo(function(){return"function"==typeof e?e(n):{...n,...e}},[n,e])}function o(e){let n;return n=e.disableParentContext?"function"==typeof e.components?e.components(a):e.components||a:i(e.components),s.createElement(t.Provider,{value:n},e.children)}}}]);
Line numbers count LF bytes from the start of the resource, as the search results do. Vendor segments are library code the classifier recognised; they are stored but not indexed. Bytes are shown as Latin1 characters, one per byte.