1"use strict";(self.webpackChunkmy_website=self.webpackChunkmy_website||[]).push([[9558],{3905:(e,t,n)=>{n.d(t,{Zo:()=>p,kt:()=>c});var a=n(67294);function l(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function o(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);t&&(a=a.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,a)}return n}function i(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?o(Object(n),!0).forEach((function(t){l(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):o(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function r(e,t){if(null==e)return{};var n,a,l=function(e,t){if(null==e)return{};var n,a,l={},o=Object.keys(e);for(a=0;a<o.length;a++)n=o[a],t.indexOf(n)>=0||(l[n]=e[n]);return l}(e,t);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(a=0;a<o.length;a++)n=o[a],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(l[n]=e[n])}return l}var d=a.createContext({}),u=function(e){var t=a.useContext(d),n=t;return e&&(n="function"==typeof e?e(t):i(i({},t),e)),n},p=function(e){var t=u(e.components);return a.createElement(d.Provider,{value:t},e.children)},s="mdxType",g={inlineCode:"code",wrapper:function(e){var t=e.children;return a.createElement(a.Fragment,{},t)}},m=a.forwardRef((function(e,t){var n=e.components,l=e.mdxType,o=e.originalType,d=e.parentName,p=r(e,["components","mdxType","originalType","parentName"]),s=u(n),m=l,c=s["".concat(d,".").concat(m)]||s[m]||g[m]||o;return n?a.createElement(c,i(i({ref:t},p),{},{components:n})):a.createElement(c,i({ref:t},p))}));function c(e,t){var n=arguments,l=t&&t.mdxType;if("string"==typeof e||l){var o=n.length,i=new Array(o);i[0]=m;var r={};for(var d in t)hasOwnProperty.call(t,d)&&(r[d]=t[d]);r.originalType=e,r[s]="string"==typeof e?e:l,i[1]=r;for(var u=2;u<o;u++)i[u]=n[u];return a.createElement.apply(null,i)}return a.createElement.apply(null,n)}m.displayName="MDXCreateElement"},49862:(e,t,n)=>{n.r(t),n.d(t,{assets:()=>d,contentTitle:()=>i,default:()=>g,frontMatter:()=>o,metadata:()=>r,toc:()=>u});var a=n(87462),l=(n(67294),n(3905));const o={id:"Fluentd-data",title:"Fluentd",overview:"Fluentd is a data collector, which unifies the data collection and consumption. This integration allows you to use Fluentd to send logs to your Logz.io account.",product:["logs"],os:["windows","linux"],filters:["Other","Most Popular"],logo:"https://logzbucket.s3.eu-west-1.amazonaws.com/logz-docs/shipper-logos/fluentd.svg",logs_dashboards:[],logs_alerts:[],logs2metrics:[],metrics_dashboards:[],metrics_alerts:[],drop_filter:[]},i=void 0,r={unversionedId:"shipping/Other/Fluentd-data",id:"shipping/Other/Fluentd-data",title:"Fluentd",description:"Fluentd is a data collector, which unifies the data collection and consumption. This integration allows you to use Fluentd to send logs to your Logz.io account.",source:"@site/docs/shipping/Other/fluentd.md",sourceDirName:"shipping/Other",slug:"/shipping/Other/Fluentd-data",permalink:"/docs/shipping/Other/Fluentd-data",draft:!1,editUrl:"https://github.com/logzio/documentation/tree/master/docs/shipping/Other/fluentd.md",tags:[],version:"current",frontMatter:{id:"Fluentd-data",title:"Fluentd",overview:"Fluentd is a data collector, which unifies the data collection and consumption. This integration allows you to use Fluentd to send logs to your Logz.io account.",product:["logs"],os:["windows","linux"],filters:["Other","Most Popular"],logo:"https://logzbucket.s3.eu-west-1.amazonaws.com/logz-docs/shipper-logos/fluentd.svg",logs_dashboards:[],logs_alerts:[],logs2metrics:[],metrics_dashboards:[],metrics_alerts:[],drop_filter:[]},sidebar:"tutorialSidebar",previous:{title:"Fluent Bit",permalink:"/docs/shipping/Other/Fluent-Bit-data"},next:{title:"FPM",permalink:"/docs/shipping/Other/FPM-data"}},d={},u=[{value:"Configure Fluentd with Ruby Gems",id:"configure-fluentd-with-ruby-gems",level:2},{value:"Install Fluentd and the Logz.io plugin",id:"install-fluentd-and-the-logzio-plugin",level:3},{value:"Set up Fluentd",id:"set-up-fluentd",level:3},{value:"Add an input plugin",id:"add-an-input-plugin",level:3}
1,{value:"Configure Fluentd with Logz.io output",id:"configure-fluentd-with-logzio-output",level:3},{value:"Parameters",id:"parameters",level:3},{value:"Run Fluentd",id:"run-fluentd",level:3},{value:"Check Logz.io for your logs",id:"check-logzio-for-your-logs",level:3},{value:"Configure Fluentd with td-agent for Windows",id:"configure-fluentd-with-td-agent-for-windows",level:2},{value:"Install Fluentd td-agent",id:"install-fluentd-td-agent",level:3},{value:"Install the Logz.io plugin",id:"install-the-logzio-plugin",level:3},{value:"Set up td-agent.conf",id:"set-up-td-agentconf",level:3},{value:"Parameters",id:"parameters-1",level:3},{value:"Run Fluentd td-agent",id:"run-fluentd-td-agent",level:3},{value:"Check Logz.io for your logs",id:"check-logzio-for-your-logs-1",level:3},{value:"Docker logs with Fluentd",id:"docker-logs-with-fluentd",level:2},{value:"Architecture overview",id:"architecture-overview",level:3},{value:"Pull the Docker image for containerized Fluentd",id:"pull-the-docker-image-for-containerized-fluentd",level:3},{value:"Start the container",id:"start-the-container",level:3},{value:"Check Logz.io for your logs",id:"check-logzio-for-your-logs-2",level:3},{value:"Advanced settings",id:"advanced-settings",level:3},{value:"Ship Kubernetes logs with Fluentd",id:"ship-kubernetes-logs-with-fluentd",level:2},{value:"Multiline logs",id:"multiline-logs",level:2},{value:"Add multiline parser to your input plugin",id:"add-multiline-parser-to-your-input-plugin",level:3},{value:"Filter processing",id:"filter-processing",level:3},{value:"Docker",id:"docker",level:3},{value:"Kubernetes",id:"kubernetes",level:3}],p={toc:u},s="wrapper";function g(e){let{components:t,...n}=e;return(0,l.kt)(s,(0,a.Z)({},p,n,{components:t,mdxType:"MDXLayout"}),(0,l.kt)("p",null,"Fluentd is a data collector, which unifies the data collection and consumption. This integration allows you to use Fluentd to send logs to your Logz.io account. "),(0,l.kt)("admonition",{type:"note"},(0,l.kt)("p",{parentName:"admonition"},"Fluentd will fetch all existing logs, as it is not able to ignore older logs.")),(0,l.kt)("h2",{id:"configure-fluentd-with-ruby-gems"},"Configure Fluentd with Ruby Gems"),(0,l.kt)("p",null,(0,l.kt)("strong",{parentName:"p"},"Before you begin, you'll need"),":\nRuby and ruby-dev 2.1 or higher"),(0,l.kt)("h3",{id:"install-fluentd-and-the-logzio-plugin"},"Install Fluentd and the Logz.io plugin"),(0,l.kt)("pre",null,(0,l.kt)("code",{parentName:"pre",className:"language-shell"},"gem install fluentd fluent-plugin-logzio\n")),(0,l.kt)("h3",{id:"set-up-fluentd"},"Set up Fluentd"),(0,l.kt)("pre",null,(0,l.kt)("code",{parentName:"pre",className:"language-shell"},"fluentd --setup ./fluent\n")),(0,l.kt)("h3",{id:"add-an-input-plugin"},"Add an input plugin"),(0,l.kt)("p",null,"Add a required input plugin to the configuration file. You can find the list of available input plugins ",(0,l.kt)("a",{parentName:"p",href:"https://docs.fluentd.org/input"},"here"),"."),(0,l.kt)("p",null,"An example input plugin looks as follows:"),(0,l.kt)("pre",null,(0,l.kt)("code",{parentName:"pre",className:"language-xml"},"<source>\n @type tail\n path /var/log/httpd-access.log\n pos_file /var/log/td-agent/httpd-access.log.pos\n tag apache.access\n <parse>\n @type apache2\n </parse>\n</source>\n")),(0,l.kt)("h3",{id:"configure-fluentd-with-logzio-output"},"Configure Fluentd with Logz.io output"),(0,l.kt)("p",null,"Add this code block to your Fluent configuration file (",(0,l.kt)("inlineCode",{parentName:"p"},"fluent.conf")," by default)."),(0,l.kt)("p",null,"See the configuration parameters below the code block.\ud83d\udc47"),(0,l.kt)("pre",null,(0,l.kt)("code",{parentName:"pre",className:"language-conf"},"<match **>\n @type logzio_buffered\n endpoint_url https://<<LISTENER-HOST>>:8071?token=<<LOG-SHIPPING-TOKEN>>&type=my_type\n output_include_time true\n output_include_tags true\n http_idle_timeout 10\n <buffer>\n @type memory\n flush_thread_count 4\n flush_interval 3s\n chunk_limit_size 16m\n queue_limit_length 4096\n </buffer>\n</match>\n")),(0,l.kt)("h3",{id:"parameters"},"Parameters"),(0,l.kt)("table",null,(0,l.kt)("thead",{parentName:"table"},(0,l.kt)("tr",{parentName:"thead"},(0,l.kt)("th",{parentName:"tr",align:null},"Parameter"),(0,l.kt)("th",{parentName:"tr",align:null},"Description"))),(0,l.kt)("tbody",{parentName:"table"},(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"endpoint_url"),(0,l.kt)("td",{parentName:"tr",align:null},"A url composed of your Logz.io region's listener URL, account token, and log type. Replace ",(0,l.kt)("inlineCode",{parentName:"td"},"<<LISTENER-HOST>>")," with the host ",(0,l.kt)("a",{parentName:"td",href:"https://docs.logz.io/docs/user-guide/admin/hosting-regions/account-region/#available-regions"},"for your region"),". The required port depends whether HTTP or HTTPS is used: HTTP = 8070, HTTPS = 8071.")),(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"output_include_time"),(0,l.kt)("td",{parentName:"tr",align:null},"To add a timestamp to your logs when they're processed, ",(0,l.kt)("inlineCode",{parentName:"td"},"true")," (recommended). Otherwise, ",(0,l.kt)("inlineCode",{parentName:"td"},"false"),".")),(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"output_include_tags"),(0,l.kt)("td",{parentName:"tr",align:null},"To add the ",(0,l.kt)("inlineCode",{parentName:"td"},"fluentd")," tag to logs, ",(0,l.kt)("inlineCode",{parentName:"td"},"true"),". Otherwise, ",(0,l.kt)("inlineCode",{parentName:"td"},"false"),". If ",(0,l.kt)("inlineCode",{parentName:"td"},"true"),", use in combination with ",(0,l.kt)("inlineCode",{parentName:"td"},"output_tags_fieldname"),".")),(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"output_tags_fieldname"),(0,l.kt)("td",{parentName:"tr",align:null},"If ",(0,l.kt)("inlineCode",{parentName:"td"},"output_include_tags")," is ",(0,l.kt)("inlineCode",{parentName:"td"},"true"),", sets output tag's field name. The default is ",(0,l.kt)("inlineCode",{parentName:"td"},"fluentd_tag"))),(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"http_idle_timeout"),(0,l.kt)("td",{parentName:"tr",align:null},"Time, in seconds, that the HTTP connection will stay open without traffic before timing out.")),(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"retry_count"),(0,l.kt)("td",{parentName:"tr",align:null},"Counter of the times to resend failed bulks. The default is ",(0,l.kt)("inlineCode",{parentName:"td"},"4"),".")),(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"retry_sleep"),(0,l.kt)("td",{parentName:"tr",align:null},"Interval in seconds to sleep initially between retries, exponential step-off. The default is ",(0,l.kt)("inlineCode",{parentName:"td"},"2s"),".")),(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"bulk_limit"),(0,l.kt)("td",{parentName:"tr",align:null},"Limit to the size of the Logz.io upload bulk. Defaults to 1000000 bytes, leaving about 24kB for overhead.")),(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"bulk_limit_warning_limit"),(0,l.kt)("td",{parentName:"tr",align:null}
1,"Limit to the size of the Logz.io warning message when a record exceeds bulk_limit to prevent a recursion when Fluent warnings are sent to the Logz.io output. The default is ",(0,l.kt)("inlineCode",{parentName:"td"},"nil")," (no truncation).")),(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"proxy_uri"),(0,l.kt)("td",{parentName:"tr",align:null},"Your proxy uri. The default is ",(0,l.kt)("inlineCode",{parentName:"td"},"nil"),'. For example: "my.ip:12345".')),(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"proxy_cert"),(0,l.kt)("td",{parentName:"tr",align:null},"Your proxy cert. The default is ",(0,l.kt)("inlineCode",{parentName:"td"},"nil"),".")),(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"gzip"),(0,l.kt)("td",{parentName:"tr",align:null},"Defines if the plugin needs to ship the logs in gzip compression. The default is ",(0,l.kt)("inlineCode",{parentName:"td"},"false"),".")))),(0,l.kt)("h3",{id:"run-fluentd"},"Run Fluentd"),(0,l.kt)("pre",null,(0,l.kt)("code",{parentName:"pre",className:"language-shell"},"fluentd -c ./fluent/fluent.conf -vv\n")),(0,l.kt)("h3",{id:"check-logzio-for-your-logs"},"Check Logz.io for your logs"),(0,l.kt)("p",null,"Give your logs some time to get from your system to ours, and then open ",(0,l.kt)("a",{parentName:"p",href:"https://app.logz.io/#/dashboard/osd"},"Open Search Dashboards"),"."),(0,l.kt)("p",null,"If you still don't see your logs, see ",(0,l.kt)("a",{parentName:"p",href:"https://docs.logz.io/docs/user-guide/log-management/troubleshooting/log-shipping-troubleshooting/"},"log shipping troubleshooting"),"."),(0,l.kt)("p",null,"Fluentd can receive and concatenate multiline logs. To do this, you need to add a parser and concatenation plugin to your Fluentd configuration."),(0,l.kt)("h2",{id:"configure-fluentd-with-td-agent-for-windows"},"Configure Fluentd with td-agent for Windows"),(0,l.kt)("p",null,(0,l.kt)("strong",{parentName:"p"},"Before you begin, you'll need"),":\nRuby and ruby-dev 2.1 or higher"),(0,l.kt)("h3",{id:"install-fluentd-td-agent"},"Install Fluentd td-agent"),(0,l.kt)("p",null,"Navigate to the ",(0,l.kt)("a",{parentName:"p",href:"https://docs.fluentd.org/installation/install-by-msi"},"downloads")," page of td-agent and download the latest version of the installer. After that, run the installer and follow the wizard instructions."),(0,l.kt)("h3",{id:"install-the-logzio-plugin"},"Install the Logz.io plugin"),(0,l.kt)("pre",null,(0,l.kt)("code",{parentName:"pre",className:"language-shell"},"gem install fluentd fluent-plugin-logzio\n")),(0,l.kt)("h3",{id:"set-up-td-agentconf"},"Set up td-agent.conf"),(0,l.kt)("p",null,"Open C:/opt/td-agent/etc/td-agent/td-agent.conf and replace its content with the following configuration:"),(0,l.kt)("pre",null,(0,l.kt)("code",{parentName:"pre",className:"language-xml"},"<source>\n @type windows_eventlog2\n @id windows_eventlog2\n channels application,system,security\n read_existing_events false\n tag winevt.raw\n rate_limit 200\n <storage>\n @type local\n persistent true\n path C:\\opt\\td-agent\\winlog.json\n </storage>\n</source>\n\n<match **>\n @type logzio_buffered\n endpoint_url https://<<LISTENER-HOST>>:8071?token=<<LOG-SHIPPING-TOKEN>>&type=<<LOG-TYPE>>\n output_include_time true\n output_include_tags true\n http_idle_timeout 10\n <buffer>\n @type memory\n flush_thread_count 4\n flush_interval 3s\n chunk_limit_size 16m\n queue_limit_length 4096\n </buffer>\n</match>\n")),(0,l.kt)("h3",{id:"parameters-1"},"Parameters"),(0,l.kt)("table",null,(0,l.kt)("thead",{parentName:"table"},(0,l.kt)("tr",{parentName:"thead"},(0,l.kt)("th",{parentName:"tr",align:null},"Parameter"),(0,l.kt)("th",{parentName:"tr",align:null},"Description"))),(0,l.kt)("tbody",{parentName:"table"},(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"endpoint_url"),(0,l.kt)("td",{parentName:"tr",align:null},"A url composed of your Logz.io region's listener URL, account token, and log type. Replace ",(0,l.kt)("inlineCode",{parentName:"td"},"<<LISTENER-HOST>>")," with the host ",(0,l.kt)("a",{parentName:"td",href:"https://docs.logz.io/docs/user-guide/admin/hosting-regions/account-region/#available-regions"},"for your region"),". The required port depends whether HTTP or HTTPS is used: HTTP = 8070, HTTPS = 8071. Replace ",(0,l.kt)("inlineCode",{parentName:"td"},"<<LOG-SHIPPING-TOKEN>>")," with the token of the account you want to ship to.")),(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"type"),(0,l.kt)("td",{parentName:"tr",align:null},"Log type. If required, replace ",(0,l.kt)("inlineCode",{parentName:"td"},"<<LOG_TYPE>>")," with the desired name for the log type, the default value is ",(0,l.kt)("inlineCode",{parentName:"td"},"fluentbit"))),(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"output_include_time"),(0,l.kt)("td",{parentName:"tr",align:null},"To add a timestamp to your logs when they're processed, ",(0,l.kt)("inlineCode",{parentName:"td"},"true")," (recommended). Otherwise, ",(0,l.kt)("inlineCode",{parentName:"td"},"false"),".")),(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"output_include_tags"),(0,l.kt)("td",{parentName:"tr",align:null},"To add the ",(0,l.kt)("inlineCode",{parentName:"td"},"fluentd")," tag to logs, ",(0,l.kt)("inlineCode",{parentName:"td"},"true"),". Otherwise, ",(0,l.kt)("inlineCode",{parentName:"td"},"false"),". If ",(0,l.kt)("inlineCode",{parentName:"td"},"true"),", use in combination with ",(0,l.kt)("inlineCode",{parentName:"td"},"output_tags_fieldname"),".")),(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"output_tags_fieldname"),(0,l.kt)("td",{parentName:"tr",align:null},"If ",(0,l.kt)("inlineCode",{parentName:"td"},"output_include_tags")," is ",(0,l.kt)("inlineCode",{parentName:"td"},"true"),", sets output tag's field name. The default is ",(0,l.kt)("inlineCode",{parentName:"td"},"fluentd_tag"))),(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"http_idle_timeout"),(0,l.kt)("td",{parentName:"tr",align:null},"Time, in seconds, that the HTTP connection will stay open without traffic before timing out.")))),(0,l.kt)("h3",{id:"run-fluentd-td-agent"},"Run Fluentd td-agent"),(0,l.kt)("p",null,"Open ",(0,l.kt)("inlineCode",{parentName:"p"},"Td-agent Command Prompt")," from the Windows Start menu and run the following command:"),(0,l.kt)("pre",null,(0,l.kt)("c
1ode",{parentName:"pre",className:"language-shell"},"C:\\opt\\td-agent> td-agent\n")),(0,l.kt)("h3",{id:"check-logzio-for-your-logs-1"},"Check Logz.io for your logs"),(0,l.kt)("p",null,"Give your logs some time to get from your system to ours, and then open ",(0,l.kt)("a",{parentName:"p",href:"https://app.logz.io/#/dashboard/osd"},"Open Search Dashboards"),"."),(0,l.kt)("p",null,"If you still don't see your logs, see ",(0,l.kt)("a",{parentName:"p",href:"https://docs.logz.io/docs/user-guide/log-management/troubleshooting/log-shipping-troubleshooting/"},"log shipping troubleshooting"),"."),(0,l.kt)("p",null,"Fluentd can receive and concatenate multiline logs. To do this, you need to add a parser and concatenation plugin to your Fluentd configuration."),(0,l.kt)("h2",{id:"docker-logs-with-fluentd"},"Docker logs with Fluentd"),(0,l.kt)("h3",{id:"architecture-overview"},"Architecture overview"),(0,l.kt)("p",null,"This integration includes:"),(0,l.kt)("ul",null,(0,l.kt)("li",{parentName:"ul"},"Pulling a Docker image of containerized Fluentd"),(0,l.kt)("li",{parentName:"ul"},"Configuring and running containerized Fluentd")),(0,l.kt)("p",null,(0,l.kt)("img",{parentName:"p",src:"https://dytvr9ot2sszz.cloudfront.net/logz-docs/log-shipping/docker-fluentd-hla-v2.png",alt:"Integration architecture Fluentd on Docker"})),(0,l.kt)("p",null,"Upon deployment, each container on your host system, including the Fluentd container, writes logs to a dedicated log file. Fluentd fetches the log data from this file and ships the data over HTTP or HTTPS to your Logz.io account, either via an optional proxy sever or directly."),(0,l.kt)("admonition",{type:"note"},(0,l.kt)("p",{parentName:"admonition"},(0,l.kt)("a",{parentName:"p",href:"https://github.com/logzio/fluentd-docker-logs/"},"Project's GitHub repo"))),(0,l.kt)("p",null,(0,l.kt)("strong",{parentName:"p"},"Before you begin, you'll need"),":\nDocker installed on your host system"),(0,l.kt)("h3",{id:"pull-the-docker-image-for-containerized-fluentd"},"Pull the Docker image for containerized Fluentd"),(0,l.kt)("pre",null,(0,l.kt)("code",{parentName:"pre",className:"language-shell"},"docker pull logzio/fluentd-docker-logs\n")),(0,l.kt)("h3",{id:"start-the-container"},"Start the container"),(0,l.kt)("p",null,"Run the following command:"),(0,l.kt)("pre",null,(0,l.kt)("code",{parentName:"pre",className:"language-conf"},'docker run -it --rm \\\n--name fluentd-docker-logs \\\n-v $(pwd)/log:/fluentd/log \\\n-v /var/lib/docker/containers:/var/lib/docker/containers \\\n-v /var/run/docker.sock:/var/run/docker.sock:ro \\\n-p 5001:5001 \\\n-e LOGZIO_LOG_LISTENER="https://<<LISTENER-HOST>>:8071" \\\n-e LOGZIO_LOG_SHIPPING_TOKEN=<<LOG-SHIPPING-TOKEN>> \\\n-e LOGZIO_TYPE=docker-fluentd \\\nlogzio/fluentd-docker-logs\n')),(0,l.kt)("p",null,"Replace ",(0,l.kt)("inlineCode",{parentName:"p"},"<<LISTENER-HOST>>")," with the host ",(0,l.kt)("a",{parentName:"p",href:"https://docs.logz.io/docs/user-guide/admin/hosting-regions/account-region/#available-regions"},"for your region"),". The required port depends whether HTTP or HTTPS is used: HTTP = 8070, HTTPS = 8071.\nReplace ",(0,l.kt)("inlineCode",{parentName:"p"},"<<LOG-SHIPPING-TOKEN>>")," with the token of the account you want to ship to."),(0,l.kt)("p",null,"If you need to send the logs via a proxy server:"),(0,l.kt)("ul",null,(0,l.kt)("li",{parentName:"ul"},"Add ",(0,l.kt)("inlineCode",{parentName:"li"},"-e LOGZIO_PROXY_URI=<<YOUR-PROXY-URI>>")," to the above command and replace ",(0,l.kt)("inlineCode",{parentName:"li"},"<<YOUR-PROXY-URI>>")," with your proxy URI."),(0,l.kt)("li",{parentName:"ul"},"Add ",(0,l.kt)("inlineCode",{parentName:"li"},"-e LOGZIO_PROXY_CERT=<<YOUR-PROXY-CERTIFICATE>>")," to the above commad and replace ",(0,l.kt)("inlineCode",{parentName:"li"},"<<YOUR-PROXY-CERTIFICATE>>")," with your proxy certificate value.")),(0,l.kt)("h3",{id:"check-logzio-for-your-logs-2"},"Check Logz.io for your logs"),(0,l.kt)("p",null,"Give your logs some time to get from your system to ours, and then open ",(0,l.kt)("a",{parentName:"p",href:"https://app.logz.io/#/dashboard/osd"},"Open Search Dashboards"),". You can filter for data of type ",(0,l.kt)("inlineCode",{parentName:"p"},"docker-fluentd")," to see the incoming container logs."),(0,l.kt)("p",null,"If you still don\u2019t see your data, see ",(0,l.kt)("a",{parentName:"p",href:"https://docs.logz.io/docs/user-guide/log-management/troubleshooting/log-shipping-troubleshooting/"},"log shipping troubleshooting"),"."),(0,l.kt)("h3",{id:"advanced-settings"},"Advanced settings"),(0,l.kt)("p",null,"If you need to customize the default settings of the configuration parameters, add any of the following lines to the command:"),(0,l.kt)("table",null,(0,l.kt)("thead",{parentName:"table"},(0,l.kt)("tr",{parentName:"thead"},(0,l.kt)("th",{parentName:"tr",align:null},"Parameter"),(0,l.kt)("th",{parentName:"tr",align:null},"Description"),(0,l.kt)("th",{parentName:"tr",align:null},"Default"))),(0,l.kt)("tbody",{parentName:"table"},(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("inlineCode",{parentName:"td"},"-e LOGZIO_INCLUDE_REGEX=<<LOGZIO_INCLUDE_REGEX>> \\")),(0,l.kt)("td",{parentName:"tr",align:null},"If a container name does not match the Regex, logs from this container will not be shipped."),(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("inlineCode",{parentName:"td"},".+"))),(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("inlineCode",{parentName:"td"},"-e LOGZIO_SLOW_FLUSH_LOG_THRESHOLD=<<LOGZIO_SLOW_FLUSH_LOG_THRESHOLD>> \\")),(0,l.kt)("td",{parentName:"tr",align:null},"Specifies the threshold for chunk flush performance check."),(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("inlineCode",{parentName:"td"},"20.0"))),(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("inlineCode",{parentName:"td"},"-e LOGZIO_BUFFER_TYPE=<<LOGZIO_BUFFER_TYPE>> \\")),(0,l.kt)("td",{parentName:"tr",align:null},"Specifies which plugin to use as the backend."),(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("inlineCode",{parentName:"td"},"file"))),(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("inlineCode",{parentName:"td"},"-e LOGZIO_BUFFER_PATH=<<LOGZIO_BUFFER_PATH>> \\")),(0,l.kt)("td",{parentName:"tr",align:null},"Specifies the path to the backend plugin."),(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("inlineCode",{parentName:"td"},"/var/log/Fluentd-buffers/stackdriver.buffer"))),(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("inlineCode",{parentName:"td"},"-e LOGZIO_OVERFLOW_ACTION=<<LOGZIO_OVERFLOW_ACTION>> \\")),(0,l.kt)("td",{parentName:"tr",align:null}
1,"Specifies the parameter that controls the behavior when the queue becomes full. Refer to ",(0,l.kt)("a",{parentName:"td",href:"https://docs.fluentd.org/output#overflow_action"},"documentation on Fluentd")," for more on this."),(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("inlineCode",{parentName:"td"},"block"))),(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("inlineCode",{parentName:"td"},"-e LOGZIO_CHUNK_LIMIT_SIZE=<<LOGZIO_CHUNK_LIMIT_SIZE>> \\")),(0,l.kt)("td",{parentName:"tr",align:null},"Specifies the maximum size of a chunk allowed."),(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("inlineCode",{parentName:"td"},"2M"))),(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("inlineCode",{parentName:"td"},"-e LOGZIO_QUEUE_LIMIT_LENGTH=<<LOGZIO_QUEUE_LIMIT_LENGTH>> \\")),(0,l.kt)("td",{parentName:"tr",align:null},"Specifies the maximum length of the output queue."),(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("inlineCode",{parentName:"td"},"6"))),(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("inlineCode",{parentName:"td"},"-e LOGZIO_FLUSH_INTERVAL=<<LOGZIO_FLUSH_INTERVAL>> \\")),(0,l.kt)("td",{parentName:"tr",align:null},"Specifies the interval, in seconds, to wait before invoking the next buffer flush."),(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("inlineCode",{parentName:"td"},"5s"))),(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("inlineCode",{parentName:"td"},"-e LOGZIO_RETRY_MAX_INTERVAL=<<LOGZIO_RETRY_MAX_INTERVAL>> \\")),(0,l.kt)("td",{parentName:"tr",align:null},"Specifies the maximum interval, in seconds, to wait between retries."),(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("inlineCode",{parentName:"td"},"30s"))),(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("inlineCode",{parentName:"td"},"-e LOGZIO_FLUSH_THREAD_COUNT=<<LOGZIO_FLUSH_THREAD_COUNT>> \\")),(0,l.kt)("td",{parentName:"tr",align:null},"Specifies the number of threads to flush the buffer."),(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("inlineCode",{parentName:"td"},"2"))),(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("inlineCode",{parentName:"td"},"-e LOGZIO_LOG_LEVEL=<<LOGZIO_LOG_LEVEL>> \\")),(0,l.kt)("td",{parentName:"tr",align:null},"Specifies the log level for this container."),(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("inlineCode",{parentName:"td"},"info"))),(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("inlineCode",{parentName:"td"},"ADDITIONAL_FIELDS")),(0,l.kt)("td",{parentName:"tr",align:null},"Include additional fields with every message sent, formatted as ",(0,l.kt)("inlineCode",{parentName:"td"},"fieldName1=fieldValue1,fieldName2=fieldValue2")),(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("inlineCode",{parentName:"td"},"nil"))))),(0,l.kt)("h2",{id:"ship-kubernetes-logs-with-fluentd"},"Ship Kubernetes logs with Fluentd"),(0,l.kt)("p",null," See the full documentation for shipping your Kubernetes logs with Fluentd ",(0,l.kt)("a",{parentName:"p",href:"https://docs.logz.io/docs/shipping/containers/kubernetes/"},"here"),"."),(0,l.kt)("h2",{id:"multiline-logs"},"Multiline logs"),(0,l.kt)("p",null,"Fluentd splits multiline logs by default. If your original logs span multiple lines, you may find that they arrive in your Logz.io account split into several partial logs."),(0,l.kt)("h3",{id:"add-multiline-parser-to-your-input-plugin"},"Add multiline parser to your input plugin"),(0,l.kt)("admonition",{type:"note"},(0,l.kt)("p",{parentName:"admonition"}," Multiline parsing only works with ",(0,l.kt)("inlineCode",{parentName:"p"},"in_tail")," plugins. Refer to the ",(0,l.kt)("a",{parentName:"p",href:"https://docs.fluentd.org/parser/multiline"},"Fluentd documentation")," for more on this.")),(0,l.kt)("p",null,"The multine parser is supported in the ",(0,l.kt)("inlineCode",{parentName:"p"},"in_tail")," input plugin that allows Fluentd to read events from the tail of text files."),(0,l.kt)("p",null,"Add the following code block to your ",(0,l.kt)("inlineCode",{parentName:"p"}
1,"in_tail")," plugin:"),(0,l.kt)("pre",null,(0,l.kt)("code",{parentName:"pre",className:"language-xml"},"<parse>\n @type multiline\n format_firstline /^<<YOUR-REGEX-PATTERN>>/\n</parse>\n")),(0,l.kt)("ul",null,(0,l.kt)("li",{parentName:"ul"},"Replace ",(0,l.kt)("inlineCode",{parentName:"li"},"<<YOUR-REGEX-PATTERN>>")," with the definition of your Regex pattern. You can use ",(0,l.kt)("a",{parentName:"li",href:"https://regex101.com/"},"regex101")," to define it.")),(0,l.kt)("p",null,"The indentation of the parse plugin must be one level under the tail function as in the example below:"),(0,l.kt)("pre",null,(0,l.kt)("code",{parentName:"pre",className:"language-xml"},"<source>\n @type tail\n path /var/log/httpd-access.log\n pos_file /var/log/td-agent/httpd-access.log.pos\n tag apache.access\n <parse>\n @type multiline\n format_firstline /\\d{4}-\\d{1,2}-\\d{1,2}/\n format1 /^(?<time>\\d{4}-\\d{1,2}-\\d{1,2} \\d{1,2}:\\d{1,2}:\\d{1,2}) \\[(?<thread>.*)\\] (?<level>[^\\s]+)(?<message>.*)/\n </parse>\n</source>\n")),(0,l.kt)("h3",{id:"filter-processing"},"Filter processing"),(0,l.kt)("p",null,"The following is an example of a multiline log sent from a deployment on a Kubernetes cluster:"),(0,l.kt)("pre",null,(0,l.kt)("code",{parentName:"pre",className:"language-shell"},'2021-02-08 09:37:51,031 - errorLogger - ERROR - Traceback (most recent call last):\nFile "./code.py", line 25, in my_func\n1/0\nZeroDivisionError: division by zero\n')),(0,l.kt)("p",null,"Fluentd's default configuration will split the above log into 4 logs, 1 for each line of the original log. In other words, each line break (",(0,l.kt)("inlineCode",{parentName:"p"},"\\n"),") causes a split."),(0,l.kt)("p",null,"To avoid this, you can use the ",(0,l.kt)("inlineCode",{parentName:"p"},"fluent-plugin-concat")," filter plugin and customize the configuration to meet your needs."),(0,l.kt)("p",null,"For the above example, we could use the following regex expressions to demarcate the start and end of our example log:"),(0,l.kt)("pre",null,(0,l.kt)("code",{parentName:"pre",className:"language-shell"},"<filter **> # Match all logs, you can replace ** with your tag\n @type concat\n key message # The key for part of multiline log\n multiline_start_regexp /^[0-9]{4}-[0-9]{2}-[0-9]{2}/ # This regex expression identifies line starts.\n</filter>\n")),(0,l.kt)("h3",{id:"docker"},"Docker"),(0,l.kt)("p",null,"The Logz.io Docker image comes with a pre-built Fluentd filter plug-in that can be used to concatenate multiline logs. The plug-in is named ",(0,l.kt)("inlineCode",{parentName:"p"},"fluent-plugin-concat"),", and it concatenate multiline log separated in multiple events. You can review the full list of configuration options in the ",(0,l.kt)("a",{parentName:"p",href:"https://github.com/fluent-plugins-nursery/fluent-plugin-concat"},"GitHub project"),"."),(0,l.kt)("h3",{id:"kubernetes"},"Kubernetes"),(0,l.kt)("p",null,"The additional configuration that applies to the matched logs should be added to the ",(0,l.kt)("a",{parentName:"p",href:"https://github.com/logzio/logzio-helm/blob/master/charts/fluentd/values.yaml"},"values.yml file"),"."))}g.isMDXComponent=!0}}]);
Line numbers count LF bytes from the start of the resource, as the search results do. Vendor segments are library code the classifier recognised; they are stored but not indexed. Bytes are shown as Latin1 characters, one per byte.