https://docs.logz.io/assets/js/a8eaaa04.aad8468e.js

1"use strict";(self.webpackChunkmy_website=self.webpackChunkmy_website||[]).push([[13759],{3905:(e,t,a)=>{a.d(t,{Zo:()=>p,kt:()=>c});var n=a(67294);function r(e,t,a){return t in e?Object.defineProperty(e,t,{value:a,enumerable:!0,configurable:!0,writable:!0}):e[t]=a,e}function i(e,t){var a=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);t&&(n=n.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),a.push.apply(a,n)}return a}function o(e){for(var t=1;t<arguments.length;t++){var a=null!=arguments[t]?arguments[t]:{};t%2?i(Object(a),!0).forEach((function(t){r(e,t,a[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(a)):i(Object(a)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(a,t))}))}return e}function l(e,t){if(null==e)return{};var a,n,r=function(e,t){if(null==e)return{};var a,n,r={},i=Object.keys(e);for(n=0;n<i.length;n++)a=i[n],t.indexOf(a)>=0||(r[a]=e[a]);return r}(e,t);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);for(n=0;n<i.length;n++)a=i[n],t.indexOf(a)>=0||Object.prototype.propertyIsEnumerable.call(e,a)&&(r[a]=e[a])}return r}var s=n.createContext({}),d=function(e){var t=n.useContext(s),a=t;return e&&(a="function"==typeof e?e(t):o(o({},t),e)),a},p=function(e){var t=d(e.components);return n.createElement(s.Provider,{value:t},e.children)},u="mdxType",g={inlineCode:"code",wrapper:function(e){var t=e.children;return n.createElement(n.Fragment,{},t)}},m=n.forwardRef((function(e,t){var a=e.components,r=e.mdxType,i=e.originalType,s=e.parentName,p=l(e,["components","mdxType","originalType","parentName"]),u=d(a),m=r,c=u["".concat(s,".").concat(m)]||u[m]||g[m]||i;return a?n.createElement(c,o(o({ref:t},p),{},{components:a})):n.createElement(c,o({ref:t},p))}));function c(e,t){var a=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var i=a.length,o=new Array(i);o[0]=m;var l={};for(var s in t)hasOwnProperty.call(t,s)&&(l[s]=t[s]);l.originalType=e,l[u]="string"==typeof e?e:r,o[1]=l;for(var d=2;d<i;d++)o[d]=a[d];return n.createElement.apply(null,o)}return n.createElement.apply(null,a)}m.displayName="MDXCreateElement"},54281:(e,t,a)=>{a.r(t),a.d(t,{assets:()=>s,contentTitle:()=>o,default:()=>g,frontMatter:()=>i,metadata:()=>l,toc:()=>d});var n=a(87462),r=(a(67294),a(3905));const i={sidebar_position:2,title:"Understanding Invalid Log Errors",description:"Learn about invalid logs and the error tags that help you resolve issues",image:"https://dytvr9ot2sszz.cloudfront.net/logz-docs/social-assets/docs-social.jpg",ke

1ywords:["logz.io","troubleshooting","invalid logs","log analysis","observability"]},o=void 0,l={unversionedId:"user-guide/explore/troubleshooting/invalid-log-errors",id:"user-guide/explore/troubleshooting/invalid-log-errors",title:"Understanding Invalid Log Errors",description:"Learn about invalid logs and the error tags that help you resolve issues",source:"@site/docs/user-guide/explore/troubleshooting/invalid-log-errors.md",sourceDirName:"user-guide/explore/troubleshooting",slug:"/user-guide/explore/troubleshooting/invalid-log-errors",permalink:"/docs/user-guide/explore/troubleshooting/invalid-log-errors",draft:!1,editUrl:"https://github.com/logzio/documentation/tree/master/docs/user-guide/explore/troubleshooting/invalid-log-errors.md",tags:[],version:"current",sidebarPosition:2,frontMatter:{sidebar_position:2,title:"Understanding Invalid Log Errors",description:"Learn about invalid logs and the error tags that help you resolve issues",image:"https://dytvr9ot2sszz.cloudfront.net/logz-docs/social-assets/docs-social.jpg",ke

1ywords:["logz.io","troubleshooting","invalid logs","log analysis","observability"]},sidebar:"tutorialSidebar",previous:{title:"Log Shipping Troubleshooting",permalink:"/docs/explore/troubleshooting/shipping-log-troubleshooting/"},next:{title:"Log Management",permalink:"/docs/category/log-management"}},s={},d=[{value:"Field mapping types",id:"field-mapping-types",level:2},{value:"Mapping errors",id:"mapping-errors",level:2},{value:"Mapping errors through sub accounts",id:"mapping-errors-through-sub-accounts",level:2},{value:"Invalid logs",id:"invalid-logs",level:2},{value:"What causes an invalid log?",id:"what-causes-an-invalid-log",level:3},{value:"Invalid log tags",id:"invalid-log-tags",level:3}],p={toc:d},u="wrapper";function g(e){let{components:t,...a}=e;return(0,r.kt)(u,(0,n.Z)({},p,a,{components:t,mdxType:"MDXLayout"}),(0,r.kt)("p",null,"Invalid log errors can occur in several scenarios. In this document, we'll discuss the different types of log errors and common methods for solving the issue."),(0,r.kt)("admonition",{type:"note"},(0,r.kt)("p",{parentName:"admonition"},"Logz.io no longer changes the log\u2019s ",(0,r.kt)("inlineCode",{parentName:"p"},"type")," field when errors are detected. Instead, tags are added to help you identify and troubleshoot issues.")),(0,r.kt)("h2",{id:"field-mapping-types"},"Field mapping types"),(0,r.kt)("p",null,"To make your search engine queries and analytics more effective, Explore maps each field by a data type so it knows how to display it according to its capabilities. There are two types of mapping fields:"),(0,r.kt)("ul",null,(0,r.kt)("li",{parentName:"ul"},(0,r.kt)("strong",{parentName:"li"},"Dynamic")," - This is the default mapping type, determined by the value of the log fields mapped at the beginning of each day."),(0,r.kt)("li",{parentName:"ul"},(0,r.kt)("strong",{parentName:"li"},"Explicit")," - This is a forced mapping type, and when chosen, Explore will always map this field as the same data type.")),(0,r.kt)("p",null,"For example, if the value of the log field is ",(0,r.kt)("inlineCode",{parentName:"p"},'"yourField":123'),", Explore will map it as a number (Long)."),(0,r.kt)("p",null,(0,r.kt)("inlineCode",{parentName:"p"},"\u201cyourField\u201d:\u201dabc\u201d")," will be mapped as a ",(0,r.kt)("strong",{parentName:"p"},"Keyword (String)"),"."),(0,r.kt)("p",null,(0,r.kt)("inlineCode",{parentName:"p"},"\u201cyourField\u201d:{\u201csomeField\u201d:\u201dsomeValue\u201d}")," will be mapped as an ",(0,r.kt)("strong",{parentName:"p"},"Object"),"."),(0,r.kt)("p",null,(0,r.kt)("inlineCode",{parentName:"p"},"yourField.someField")," will be mapped as a ",(0,r.kt)("strong",{parentName:"p"},"Keyword (String)"),"."),(0,r.kt)("p",null,"If a field is mapped as a string, Explore won\u2019t allow you to run any mathematical queries on the field. If it\u2019s an analyzed field, such as ",(0,r.kt)("inlineCode",{parentName:"p"},"message"),", ",(0,r.kt)("inlineCode",{parentName:"p"},"tags"),", or ",(0,r.kt)("inlineCode",{parentName:"p"},"geoip_location"),", Explore won\u2019t let you use it in an alert, a visualization, or a ",(0,r.kt)("inlineCode",{parentName:"p"},"group by")," rule."),(0,r.kt)("p",null,"Field data type determines how each field is indexed and shown in Explore. Account admins can change the data types according to a predefined set of options:"),(0,r.kt)("p",null,(0,r.kt)("img",{parentName:"p",src:"https://dytvr9ot2sszz.cloudfront.net/logz-docs/kibana-mapping/mapping-fields-main-.png",alt:"Choose field data type"})),(0,r.kt)("p",null,"Changing a field\u2019s data type may affect any dashboards, visualizations, searches, alerts, optimizers, and integrations using that field."),(0,r.kt)("h2",{id:"mapping-errors"},"Mapping errors"),(0,r.kt)("p",null,"Your logs are mapped daily, and each field is assigned a Dynamic or E

1xplicit data type."),(0,r.kt)("p",null,"Dynamic mappings are automatically determined as logs are received, meaning the fields' data type is known. When a field is marked as Explicit, its data type is unclear."),(0,r.kt)("p",null,"Mapping errors occur when different data types are sent to the same field. For example, if field ",(0,r.kt)("inlineCode",{parentName:"p"},"weather")," receives the numeric value ",(0,r.kt)("inlineCode",{parentName:"p"},"35")," and then gets the value ",(0,r.kt)("inlineCode",{parentName:"p"},"hot"),", it'll result in a mapping error since the same field can't contain two different types of inputs."),(0,r.kt)("p",null,"The original log type is preserved, and a tag is added to the log to indicate the issue."),(0,r.kt)("p",null,(0,r.kt)("img",{parentName:"p",src:"https://dytvr9ot2sszz.cloudfront.net/logz-docs/kibana/logzio-index-fail.png",alt:"Fail log example"})),(0,r.kt)("p",null,"When a log type encounters repeated mapping exceptions, the problematic field may be eliminated or flattened and renamed by the support team."),(0,r.kt)("p",null,"If a field is removed, its data is stored in a new entry called ",(0,r.kt)("inlineCode",{parentName:"p"},"logzio_removed_fields"),", which holds all removed entries for that log type and their corresponding values for reference."),(0,r.kt)("p",null,"Here are some of the ",(0,r.kt)("strong",{parentName:"p"},"common mapping errors")," you might encounter and why they happen:"),(0,r.kt)("table",null,(0,r.kt)("thead",{parentName:"table"},(0,r.kt)("tr",{parentName:"thead"},(0,r.kt)("th",{parentName:"tr",align:null},(0,r.kt)("strong",{parentName:"th"},"MPE")),(0,r.kt)("th",{parentName:"tr",align:null},(0,r.kt)("strong",{parentName:"th"},"Description")))),(0,r.kt)("tbody",{parentName:"table"},(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},"object mapping for ","[FIELD_NAME]"," tried to parse field ","[FIELD_NAME]"," as object, but found a concrete value"),(0,r.kt)("td",{parentName:"tr",align:null},"Field is mapped as a JSON object but is being sent as a string (or is being stringified by other means)")),(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},"Can\u2019t get text on a START_OBJECT"),(0,r.kt)("td",{parentName:"tr",align:null},"Field is mapped as a string, but is sent as a JSON object")),(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},"failed to parse field ","[FIELD_NAME]"," of type ","[DATA_TYPE]"),(0,r.kt)("td",{parentName:"tr",align:null},"Field is being mapped as one data type but being sent as another")),(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},"Index -1 out of bounds for length 0"),(0,r.kt)("td",{parentName:"tr",align:null},"A field exists in the log with a dot \u201c.\u201d in its name. For these cases, the system treats the field as an object when mapping it. For example: ",(0,r.kt)("inlineCode",{parentName:"td"},"log.level"),", ",(0,r.kt)("inlineCode",{parentName:"td"},"app.kubernetes"),", etc.")),(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},"Numeric value (NUMBER) out of range of long (-9223372036854775808 - 9223372036854775807)"),(0,r.kt)("td",{parentName:"tr",align:null},"Field mapped as a number, but its value is outside the range of the \u201cLong\u201d data type")),(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},"Object field starting or ending with a ","[.]"," makes object resolution ambiguous"),(0,r.kt)("td",{parentName:"tr",align:null},"Some fields in the logs contain invalid characters in the name. For example: ",(0,r.kt)("inlineCode",{parentName:"td"},".")," , ",(0,r.kt)("inlineCode",{parentName:"td"},",")," , ",(0,r.kt)("inlineCode",{parentName:"td"},"_")," , ",(0,r.kt)("inlineCode",{parentName:"td"},"#"))))),(0,r.kt)("h2",{id:"mapping-errors-through-sub-accounts"},"Mapping errors through sub accounts"),(0,r.kt)("p",null,"When mapping errors occur in your account, you can only assign one data type per specific field."),(0,r.kt)("p",null,"However, sometimes you might want to assign multiple data types to the same field, which isn\u2019t supported with Explore configuration. For these cases, you can create ",(0,r.kt)("strong",{parentName:"p"},"sub accounts"),"."),(0,r.kt)("p",null,"You can use sub accounts to send the same field that\u2019s already sent to any of your accounts but map it as a different data type."),(0,r.kt)("p",null,"For example, suppose you have a ",(0,r.kt)("inlineCode",{parentName:"p"},"metadata")," field assigned as an Object in your production environment. In that case, you can assign it as a String in y

1our testing environment by creating a sub account to which you\u2019ll send the same logs."),(0,r.kt)("p",null,"Use sub accounts to adjust your mapping based on your monitoring needs."),(0,r.kt)("p",null,"Learn more about ",(0,r.kt)("strong",{parentName:"p"},(0,r.kt)("a",{parentName:"strong",href:"https://docs.logz.io/docs/user-guide/admin/logzio-accounts/manage-the-main-account-and-sub-accounts/#add-and-manage-a-log-management-sub-account"},"creating and managing sub accounts"))," and about ",(0,r.kt)("strong",{parentName:"p"},(0,r.kt)("a",{parentName:"strong",href:"https://docs.logz.io/docs/user-guide/data-hub/field-mapping/"},"field mapping"))," in your account."),(0,r.kt)("h2",{id:"invalid-logs"},"Invalid logs"),(0,r.kt)("h3",{id:"what-causes-an-invalid-log"},"What causes an invalid log?"),(0,r.kt)("p",null,"When a log contains specific issues, it is flattened and ingested with a tag indicating the problem. The original ",(0,r.kt)("inlineCode",{parentName:"p"},"type")," field is preserved."),(0,r.kt)("p",null,(0,r.kt)("img",{parentName:"p",src:"https://dytvr9ot2sszz.cloudfront.net/logz-docs/kibana/invalid_log_eg-dec2021.png",alt:"Invalid log example"})),(0,r.kt)("h3",{id:"invalid-log-tags"},"Invalid log tags"),(0,r.kt)("p",null,"The tags in the table below explain the character or field issues that may cause a log to be labeled with the ",(0,r.kt)("strong",{parentName:"p"},(0,r.kt)("inlineCode",{parentName:"strong"},"logzio-invalid-log"))," field."),(0,r.kt)("table",null,(0,r.kt)("thead",{parentName:"table"},(0,r.kt)("tr",{parentName:"thead"},(0,r.kt)("th",{parentName:"tr",align:null},"Tag"),(0,r.kt)("th",{parentName:"tr",align:null},"Description"))),(0,r.kt)("tbody",{parentName:"table"},(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},"MAX_LOG_LINE_LENGTH"),(0,r.kt)("td",{parentName:"tr",align:null},"Exceeded the maximum of 500K characters per log")),(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},"MAX_FIELD_KEY_SIZE ",(0,r.kt)("em",{parentName:"td"},"-or-"),"  INVALID_FIELD_VALUE_LENGTH"),(0,r.kt)("td",{parentName:"tr",align:null},"Exceeded the maximum of 32700 characters per field")),(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},"MAX_JSON_DEPTH"),(0,r.kt)("td",{parentName:"tr",align:null},"Exceeded the maximum of 10 field nesting levels per log message")),(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},"MAX_FIELDS_NUMBER ",(0,r.kt)("em",{parentName:"td"},"-or-"),"<  INVALID_FIELDS_NUMBER"),(0,r.kt)("td",{parentName:"tr",align:null},"Exceeded the maximum of 1000 fields per log message")),(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},"FIELDS_MISSING"),(0,r.kt)("td",{parentName:"tr",align:null},"This error is related to required fields that are missing from your logs: For example, ",(0,r.kt)("inlineCode",{parentName:"td"},"@timestamp"),".  Check if the parsing rules remove or rename the relevant fields.")),(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},"ARRAY_INDEX_OUT_OF_BOUNDS_EXCEPTION"),(0,r.kt)("td",{parentName:"tr",align:null},"One of the field names in the log has a dot (",(0,r.kt)("strong",{parentName:"td"},(0,r.kt)("inlineCode",{parentName:"strong"},".")),") as a name: To resolve the issue, flatten the field that the ",(0,r.kt)("strong",{parentName:"td"},(0,r.kt)("inlineCode",{parentName:"strong"},"."))," is nested under.  If the field is inside an array, you'll need to flatten the array field.   For  example, you'd need to flatten the field ",(0,r.kt)("inlineCode",{parentName:"td"},"xxx.yyy"))))))}g.isMDXComponent=!0}}]);