https://docs.logz.io/assets/js/36ddd3cc.55abed12.js

1"use strict";(self.webpackChunkmy_website=self.webpackChunkmy_website||[]).push([[1664],{3905:(e,t,r)=>{r.d(t,{Zo:()=>d,kt:()=>m});var o=r(67294);function a(e,t,r){return t in e?Object.defineProperty(e,t,{value:r,enumerable:!0,configurable:!0,writable:!0}):e[t]=r,e}function n(e,t){var r=Object.keys(e);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);t&&(o=o.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),r.push.apply(r,o)}return r}function l(e){for(var t=1;t<arguments.length;t++){var r=null!=arguments[t]?arguments[t]:{};t%2?n(Object(r),!0).forEach((function(t){a(e,t,r[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(r)):n(Object(r)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(r,t))}))}return e}function i(e,t){if(null==e)return{};var r,o,a=function(e,t){if(null==e)return{};var r,o,a={},n=Object.keys(e);for(o=0;o<n.length;o++)r=n[o],t.indexOf(r)>=0||(a[r]=e[r]);return a}(e,t);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);for(o=0;o<n.length;o++)r=n[o],t.indexOf(r)>=0||Object.prototype.propertyIsEnumerable.call(e,r)&&(a[r]=e[r])}return a}var p=o.createContext({}),s=function(e){var t=o.useContext(p),r=t;return e&&(r="function"==typeof e?e(t):l(l({},t),e)),r},d=function(e){var t=s(e.components);return o.createElement(p.Provider,{value:t},e.children)},c="mdxType",u={inlineCode:"code",wrapper:function(e){var t=e.children;return o.createElement(o.Fragment,{},t)}},f=o.forwardRef((function(e,t){var r=e.components,a=e.mdxType,n=e.originalType,p=e.parentName,d=i(e,["components","mdxType","originalType","parentName"]),c=s(r),f=a,m=c["".concat(p,".").concat(f)]||c[f]||u[f]||n;return r?o.createElement(m,l(l({ref:t},d),{},{components:r})):o.createElement(m,l({ref:t},d))}));function m(e,t){var r=arguments,a=t&&t.mdxType;if("string"==typeof e||a){var n=r.length,l=new Array(n);l[0]=f;var i={};for(var p in t)hasOwnProperty.call(t,p)&&(i[p]=t[p]);i.originalType=e,i[c]="string"==typeof e?e:a,l[1]=i;for(var s=2;s<n;s++)l[s]=r[s];return o.createElement.apply(null,l)}return o.createElement.apply(null,r)}f.displayName="MDXCreateElement"},40232:(e,t,r)=>{r.r(t),r.d(t,{assets:()=>p,contentTitle:()=>l,default:()=>u,frontMatter:()=>n,metadata:()=>i,toc:()=>s});var o=r(87462),a=(r(67294),r(3905));const n={sidebar_position:1,title:"Drop Filters Overview",image:"https://dytvr9ot2sszz.cloudfront.net/logz-docs/social-assets/docs-social.jpg",description:"Filter out logs from your account with drop filters",keywords:["Data hub","data optimization","optimization","drop filters","rollups","metrics","recommendations"]},l=void 0,i={unversionedId:"open360/data-hub/drop-filters/drop-fiters-main",id:"open360/data-hub/drop-filters/drop-fiters-main",title:"Drop Filters Overview",description:"Filter out logs from your account with drop filters",source:"@site/docs/open360/data-hub/drop-filters/drop-fiters-main.md",sourceDirName:"open360/data-hub/drop-filters",slug:"/open360/data-hub/drop-filters/drop-fiters-main",permalink:"/docs/open360/data-hub/drop-filters/drop-fiters-main",draft:!1,editUrl:"https://github.com/logzio/documentation/tree/master/docs/open360/data-hub/drop-filters/drop-fiters-main.md",tags:[],version:"current",sidebarPosition:1,frontMatter:{sidebar_position:1,title:"Drop Filters Overview",image:"https://dytvr9ot2sszz.cloudfront.net/logz-docs/social-assets/docs-social.jpg",description:"Filter out logs from your account with drop filters",keywords:["Data hub","data optimization","optimization","drop filters","rollups","metrics","recommendations"]},sidebar:"tutorialSidebar",previous:{title:"Drop Filters",permalink:"/docs/category/drop-filters-1"},next:{title:"Drop Log Filters",permalink:"/docs/open360/data-hub/drop-filters/drop-fiters-logs"}},p={},s=[{value:"How much data can I filter?",id:"how-much-data-can-i-filter",level:2},{value:"Deactivating drop filters",id:"deactivating-drop-filters",level:2},{value:"Some important notes on drop filtering",id:"some-important-notes",level:2},{value:"Set up a drop filter",id:"set-up-a-drop-filter",level:2},{value:"1. Choose a log type",id:"1-choose-a-log-type",level:3},{value:"2. Add fields to filter",id:"2-add-fields-to-filter",level:3},{value:"An example",id:"an-example",level:4},{value:"3. Confirm and save",id:"3-confirm-and-save",level:3}],d={toc:s},c="wrapper";

1function u(e){let{components:t,...r}=e;return(0,a.kt)(c,(0,o.Z)({},d,r,{components:t,mdxType:"MDXLayout"}),(0,a.kt)("p",null,"Drop filters offer a great way to filter out logs from an account to help manage your account volume and lower costs."),(0,a.kt)("p",null,"Drop filters evaluate logs for field:value exact matches. Incoming logs that match your account's active drop filters will not be indexed and will not appear in your OpenSearch Dashboards account. Dropped logs are not searchable, cannot trigger alerts, and will not appear in dashboards, reports, or anything else. However, dropped logs will be archived if you are archiving logs."),(0,a.kt)("p",null,"The following diagram explains how Drop filters are applied when sending data to Logz.io:"),(0,a.kt)("p",null,(0,a.kt)("img",{parentName:"p",src:"https://dytvr9ot2sszz.cloudfront.net/logz-docs/drop-filters/drop-filter-flow.png",alt:"Drop filters overview"})),(0,a.kt)("p",null,"You can turn drop filters on and off, as often as you like, making them ideal for logs that are only needed sometimes."),(0,a.kt)("p",null,"In general, drop filters are recommended for logs that are needed infrequently, while logs that are never needed should not be shipped at all."),(0,a.kt)("p",null,"To set up your Drop filters, select ",(0,a.kt)("a",{parentName:"p",href:"https://app.logz.io/#/dashboard/tools/drop-filters"},(0,a.kt)("strong",{parentName:"a"},"Data Hub > Drop filters > Logs"))," from the navigation menu."),(0,a.kt)("p",null,"You can apply Drop filters to your logs, metrics, and traces. "),(0,a.kt)("h2",{id:"how-much-data-can-i-filter"},"How much data can I filter?"),(0,a.kt)("ul",null,(0,a.kt)("li",{parentName:"ul"},(0,a.kt)("p",{parentName:"li"},"You can use Drop filters to drop as much as twice your plan's daily volume.\nIn other words, drop filters can drop up to 200% of your daily volume."),(0,a.kt)("p",{parentName:"li"},(0,a.kt)("em",{parentName:"p"},"For instance"),":\nIf you have 50 GB daily volume,\nyou can index 50 GB and filter 100 GB per day.")),(0,a.kt)("li",{parentName:"ul"},(0,a.kt)("p",{parentName:"li"},"You can add up to 10 drop filters."))),(0,a.kt)("h2",{id:"deactivating-drop-filters"},"Deactivating drop filters"),(0,a.kt)("p",null,"Your account needs to have enough space\nto accommodate logs\nwhen you deactivate a drop filter.\nIf you expect to go over your daily limit,\nplease contact ",(0,a.kt)("a",{class:"intercom-launch",href:"mailto:[email protected]"},"the Support team or your account manager"),"."),(0,a.kt)("h2",{id:"some-important-notes"},"Some important notes on drop filtering"),(0,a.kt)("ul",null,(0,a.kt)("li",{parentName:"ul"},(0,a.kt)("p",{parentName:"li"},(0,a.kt)("strong",{parentName:"p"},"Dropped logs can't be searched in OpenSearch Dashboards and they can't trigger alerts.")," ","\\","\nAll incoming logs are compared to your drop filters.\nLogs that meet your filter criteria are dropped,\nmeaning they won't be parsed and indexed.")),(0,a.kt)("li",{parentName:"ul"},(0,a.kt)("p",{parentName:"li"},(0,a.kt)("strong",{parentName:"p"},"Dropped logs are still archived.")," ","\\","\nIf you have [archiving enabled,\nyour logs will be archived before they're dropped.\nThis means that you can restore from your archives,\neven if the logs didn't originally make it to OpenSearch Dashboards.")),(0,a.kt)("li",{parentName:"ul"},(0,a.kt)("p",{parentName:"li"},(0,a.kt)("strong",{parentName:"p"},"Restored logs pass through drop filters.")," ","\\","\nIf you're restoring logs from an archive,\nturn off drop filters if you want them to be indexed in your OpenSearch Dashboards account.\nWhen restoring,\nalways make sure that the logs you need are not filtered out using drop filters.")),(0,a.kt)("li",{parentName:"ul"},(0,a.kt)("p",{parentName:"li"},(0,a.kt)("strong",{parentName:"p"},'Working with "dotted fields".')," ","\\","\nApplying drop filters when your logs include dotted fields is a more complicated scenario: Dotted fields don't work in a drop filter. To use a dotted field to trigger a drop filter, please contact the Logz.io Support team. "),(0,a.kt)("p",{parentName:"li"},(0,a.kt)("strong",{parentName:"p"},"Example"),"\nIn the image below, ",(0,a.kt)("inlineCode",{parentName:"p"},'"data.level"')," is dotted, and not nested. The field ",(0,a.kt)("inlineCode",{parentName:"p"},'"data.level": "DEBUG"')," can't be used to trigger a drop filter for the log."))),(0,a.kt)("p",null,(0,a.kt)("img",{parentName:"p",src:"https://dytvr9ot2sszz.cloudfront.net/logz-docs/drop-filters/dotted_fields_feb2022.png",alt:"Dotted fields and drop filters"})),(0,a.kt)("h2",{id:"set-up-a-drop-filter"},"Set up a drop filter"),(0,a.kt)("p",null,"To get started,\nclick ",(0,a.kt)("strong",{parentName:"p"},(0,a.kt)("i",{class:"li li-plus"})," Add drop filter"),"\nto open the ",(0,a.kt)("em",{parentName:"p"},"New drop filter")," form."),(0,a.kt)("p",null,(0,a.kt)("img",{parentName:"p",src:"https://dytvr9ot2sszz.cloudfront.net/logz-docs/drop-filters/new-drop-filter.png",alt:"New drop filter form"})),(0,a.kt)("h3",{id:"1-choose-a-log-type"},"1. Choose a log type"),(0,a.kt)("p",null,"If you choose a ",(0,a.kt)("strong",{parentName:"p"},"Log type"),",\nonly logs of that type are dropped."),(0,a.kt)("p",null,"To include all log types, leave ",(0,a.kt)("strong",{parentName:"p"},"Log type")," blank."),(0,a.kt)("h3",{id:"2-add-fields-to-filter"},"2. Add fields to filter"),(0,a.kt)("p",null,"Add up to 3 ",(0,a.kt)("strong",{parentName:"p"},"Field"),":",(0,a.kt)("strong",{parentName:"p"},"Value")," pairs to filter.\nEach pair must be an exact match.\nDrop filters are case sensitive."),(0,a.kt)("h4",{id:"an-example"},"An example"),(0,a.kt)("p",null,"Logs from a Docker container might contain this field-value pair:"),(0,a.kt)("pre",null,(0,a.kt)("code",{parentName:"pre",className:"language-json"},'{ "docker.container.name": "system-logs" }\n')),(0,a.kt)("p",null,"Those logs are only filtered\nif we set ",(0,a.kt)("strong",{parentName:"p"},"Field")," to ",(0,a.kt)("inlineCode",{parentName:"p"},"docker.container.name"),"\nand ",(0,a.kt)("strong",{parentName:"p"},"Value")," to ",(0,a.kt)("inlineCode",{parentName:"p"},"system-logs"),"."),(0,a.kt)("p",null,"If we set ",(0,a.kt)("strong",{parentName:"p"},"Value")," to anything else\u2014such as ",(0,a.kt)("inlineCode",{parentName:"p"},"system"),"\u2014those logs\nare not filtered."),(0,a.kt)("h3",{id:"3-confirm-and-save"},"3. Confirm and save"),(0,a.kt)("p",null,"Before saving, it's important to know that all the logs that meet\nyour filter criteria will be dropped."),(0,a.kt)("p",null,"If you have [Archiv

1ing enabled,\nyour logs will be archived before they're dropped."),(0,a.kt)("p",null,"Select the confirmation check box,\nand then click ",(0,a.kt)("strong",{parentName:"p"},"Apply the filter"),"."))}u.isMDXComponent=!0}}]);