1"use strict";(self.webpackChunkmarquez_website_docs=self.webpackChunkmarquez_website_docs||[]).push([[9370],{9968:(e,t,a)=>{a.r(t),a.d(t,{assets:()=>d,contentTitle:()=>o,default:()=>h,frontMatter:()=>s,metadata:()=>r,toc:()=>l});var n=a(87462),i=(a(67294),a(3905));const s={slug:"soft-delete",description:"How to use the new soft-delete feature to clean up your lineage graphs",title:"Using the New Soft-delete Feature",authors:["Obuchowski","Robinson"],tags:["API","web","features"]},o=void 0,r={permalink:"/blog/soft-delete",editUrl:"https://github.com/MarquezProject/marquez/tree/docs/v2/docs-v2/blog/2022-10-25-soft-delete/index.mdx",source:"@site/blog/2022-10-25-soft-delete/index.mdx",title:"Using the New Soft-delete Feature",description:"How to use the new soft-delete feature to clean up your lineage graphs",date:"2022-10-25T00:00:00.000Z",formattedDate:"October 25, 2022",tags:[{label:"API",permalink:"/blog/tags/api"},{label:"web",permalink:"/blog/tags/web"},{label:"features",permalink:"/blog/tags/features"}],readingTime:5.195,hasTruncateMarker:!0,authors:[{name:"Maciej Obuchowski",title:"Marquez Committer",url:"https://github.com/mobuchowski",imageURL:"https://github.com/mobuchowski.png",key:"Obuchowski"},{name:"Michael Robinson",title:"Marquez Community Manager",url:"https://github.com/merobi-hub",imageURL:"https://github.com/merobi-hub.png",key:"Robinson"}],frontMatter:{slug:"soft-delete",description:"How to use the new soft-delete feature to clean up your lineage graphs",title:"Using the New Soft-delete Feature",authors:["Obuchowski","Robinson"],tags:["API","web","features"]},prevItem:{title:"Trying Out the New Column Lineage Feature",permalink:"/blog/column-lineage-demo"},nextItem:{title:"Exploring the Marquez Lineage API",permalink:"/blog/using-marquez-api"}},d={authorsImageUrls:[void 0,void 0]},l=[{value:"Background",id:"background",level:2},{value:"Motivations and Goals",id:"motivations-and-goals",level:2},{value:"How It Works",id:"how-it-works",level:2},{value:"Querying and Viewing Datasets",id:"querying-and-viewing-datasets",level:2},{value:"Deleting Datasets",id:"deleting-datasets",level:2},{value:"What\u2019s Next",id:"whats-next",level:2},{value:"Getting Started",id:"getting-started",level:2}],p={toc:l},u="wrapper";function h(e){let{components:t,...s}=e;return(0,i.kt)(u,(0,n.Z)({},p,s,{components:t,mdxType:"MDXLayout"}),(0,i.kt)("p",null,"Read on to learn how to use the new soft-delete feature to clean up your lineage graphs."),(0,i.kt)("h2",{id:"background"},"Background"),(0,i.kt)("p",null,"An exciting new feature is coming to the Marquez UI: the ability to \u201csoft delete\u201d datasets and jobs. Once work on the feature is finished, users will be able to hide inactive datasets and jobs from the UI, allowing for the easy removal of unused or stale metadata. At this point, most of the necessary backend development is complete. "),(0,i.kt)("h2",{id:"motivations-and-goals"},"Motivations and Goals"),(0,i.kt)("p",null,"The idea for this feature came out of a need to keep deleted or renamed datasets and jobs from cluttering the lineage view in Marquez. In Issue ",(0,i.kt)("a",{parentName:"p",href:"https://github.com/MarquezProject/marquez/issues/1736"},"#1736"),", project co-creator ",(0,i.kt)("a",{parentName:"p",href:"https://github.com/julienledem"},"Julien Le Dem")," suggested a change to the internal Marquez model to create a \u201cnew\u201d version of such a dataset or job \u2013 a deleted version \u2013 that would not show up in the current lineage view produced by ",(0,i.kt)("inlineCode",{parentName:"p"},"GET /api/v1/lineage"),". "),(0,i.kt)("h2",{id:"how-it-works"},"How It Works"),(0,i.kt)("p",null,"As noted in pull request ",(0,i.kt)("a",{parentName:"p",href:"https://github.com/MarquezProject/marquez/pull/2032"},"#2032"),", the feature works by adding a new flag, ",(0,i.kt)("inlineCode",{parentName:"p"},"is_hidden"),", to the datasets and jobs a user wishes to hide. Then, it changes the jobs_view and adds a datasets_view that hides any rows where the is_hidden flag is set to true. Currently, the feature operates via API endpoint."),(0,i.kt)("p",null,"Conveniently, the soft-delete condition is reversed whenever the job or dataset is updated because the new version automatically reverts the flag. This allows Marquez users to preserve dataset and job histories regardless of deleti
1on status."),(0,i.kt)("p",null,"The code to create a dataset migration view where the flag is used as a condition (",(0,i.kt)("inlineCode",{parentName:"p"},"api/src/main/resources/marquez/db/migration/R__3_Datasets_view.sql"),"):"),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre"},"CREATE OR REPLACE VIEW datasets_view\nAS\nSELECT uuid,\n type,\n created_at,\n updated_at,\n namespace_uuid,\n source_uuid,\n name,\n physical_name,\n description,\n current_version_uuid,\n last_modified_at,\n namespace_name,\n source_name,\n is_deleted\nFROM datasets\nWHERE is_hidden IS FALSE;\n")),(0,i.kt)("p",null,"The new API endpoint in DatasetResource.java (",(0,i.kt)("inlineCode",{parentName:"p"},"api/src/main/java/marquez/api/DatasetResource.java"),"):"),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre"},'@Timed\n@ResponseMetered\n@ExceptionMetered\n@DELETE\n@Path("{dataset}")\n@Produces(APPLICATION_JSON)\npublic Response delete(\n @PathParam("namespace") NamespaceName namespaceName,\n @PathParam("dataset") DatasetName datasetName) {\nthrowIfNotExists(namespaceName);\n\nDataset dataset =\n datasetService\n .findDatasetByName(namespaceName.getValue(), datasetName.getValue())\n .orElseThrow(() -> new DatasetNotFoundException(datasetName));\n\ndatasetService\n .delete(namespaceName.getValue(), datasetName.getValue())\n .orElseThrow(() -> new DatasetNotFoundException(datasetName));\nreturn Response.ok(dataset).build();\n}\n')),(0,i.kt)("h2",{id:"querying-and-viewing-datasets"},"Querying and Viewing Datasets"),(0,i.kt)("p",null,"Let\u2019s try out the soft-delete feature on the ",(0,i.kt)("inlineCode",{parentName:"p"},"public.top_delivery_times")," dataset in the ",(0,i.kt)("inlineCode",{parentName:"p"},"food_delivery")," namespace that comes with Marquez out of the box. For more information about installing Marquez and using ",(0,i.kt)("inlineCode",{parentName:"p"},"--seed")," for sample metadata when running Marquez, see the project ",(0,i.kt)("a",{parentName:"p",href:"https://marquezproject.ai/quickstart"},"quickstart"),"."),(0,i.kt)("p",null,"First, verify that the dataset exists using the UI and the API."),(0,i.kt)("p",null,"To check via the API, run this command on the command line:"),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre"},"curl http://localhost:5000/api/v1/namespaces/food_delivery/datasets | jq\n")),(0,i.kt)("p",null,"This command returns all the datasets in the ",(0,i.kt)("inlineCode",{parentName:"p"},"food_delivery")," namespace. ",(0,i.kt)("inlineCode",{parentName:"p"},"top_delivery_times")," should be at or near the end of the output."),(0,i.kt)("p",null,"Alternatively, you could also query the database for this specific database with this command: "),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre"},"curl http://localhost:5000/api/v1/namespaces/food_delivery/datasets/public.top_delivery_times | jq)\n")),(0,i.kt)("p",null,"The output should look something like this:"),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre"},'{\n "id": {\n "namespace": "food_delivery",\n "name": "public.top_delivery_times"\n },\n "type": "DB_TABLE",\n "name": "public.top_delivery_times",\n "physicalName": "public.top_delivery_times",\n "createdAt": "2020-02-22T22:42:42Z",\n "updatedAt": "2020-02-22T22:42:42Z",\n "namespace": "food_delivery",\n "sourceName": "default",\n "fields": [\n {\n "name": "order_id",\n "type": "INTEGER",\n "tags": [],\n "description": "The ID of the order."\n },\n {\n "name": "order_placed_on",\n "type": "TIMESTAMP",\n "tags": [],\n "description": "An ISO-8601 timestamp representing the date/time the order was placed."\n },\n {\n "name": "order_dispatched_on",\n "type": "TIMESTAMP",\n "tags": [],\n "description": "An ISO-8601 timestamp representing the date/time the order was dispatched."\n },\n {\n "name": "order_delivered_on",\n "type": "TIMESTAMP",\n "tags": [],\n "description": "An ISO-8601 t
1imestamp representing the date/time the order was delivered."\n },\n {\n "name": "order_delivered_time",\n "type": "TIMESTAMP",\n "tags": [],\n "description": "An ISO-8601 timestamp representing the total time of delivery."\n },\n {\n "name": "customer_email",\n "type": "VARCHAR",\n "tags": [],\n "description": "The email address of the customer."\n },\n {\n "name": "restaurant_id",\n "type": "INTEGER",\n "tags": [],\n "description": "The ID of the restaurant related to the order."\n },\n {\n "name": "driver_id",\n "type": "INTEGER",\n "tags": [],\n "description": "The ID of the driver related to the order."\n }\n ],\n "tags": [],\n "lastModifiedAt": null,\n "lastLifecycleState": "",\n "description": null,\n "currentVersion": "7daad6c6-bbdb-4cf5-bee6-9c42c443732e",\n "columnLineage": null,\n "facets": {\n "schema": {\n "fields": [\n {\n "name": "order_id",\n "type": "INTEGER",\n "description": "The ID of the order."\n },\n {\n "name": "order_placed_on",\n "type": "TIMESTAMP",\n "description": "An ISO-8601 timestamp representing the date/time the order was placed."\n },\n {\n "name": "order_dispatched_on",\n "type": "TIMESTAMP",\n "description": "An ISO-8601 timestamp representing the date/time the order was dispatched."\n },\n {\n "name": "order_delivered_on",\n "type": "TIMESTAMP",\n "description": "An ISO-8601 timestamp representing the date/time the order was delivered."\n },\n {\n "name": "order_delivered_time",\n "type": "TIMESTAMP",\n "description": "An ISO-8601 timestamp representing the total time of delivery."\n },\n {\n "name": "customer_email",\n "type": "VARCHAR",\n "description": "The email address of the customer."\n },\n {\n "name": "restaurant_id",\n "type": "INTEGER",\n "description": "The ID of the restaurant related to the order."\n },\n {\n "name": "driver_id",\n "type": "INTEGER",\n "description": "The ID of the driver related to the order."\n }\n ],\n "_producer": "https://github.com/MarquezProject/marquez/blob/main/docker/metadata.json",\n "_schemaURL": "https://openlineage.io/spec/facets/1-0-0/SchemaDatasetFacet.json"\n }\n },\n "deleted": false\n }\n ]\n')),(0,i.kt)("p",null,"In the UI, you can see the dataset in the ",(0,i.kt)("inlineCode",{parentName:"p"},"delivery_times_7_days")," job:"),(0,i.kt)("p",null,(0,i.kt)("img",{alt:"Marquez UI lineage map",src:a(35202).Z,width:"823",height:"578"})),(0,i.kt)("h2",{id:"deleting-datasets"},"Deleting Datasets"),(0,i.kt)("p",null,"To delete this dataset, query the datasets endpoint with a DELETE API request:"),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre"},"curl -X DELETE http://localhost:5000/api/v1/namespaces/food_delivery/datasets/public.top_delivery_times | jq\n")),(0,i.kt)("p",null,"It will output the dataset. Subsequent requests for the dataset will return 404s:"),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre"},'{\n "code": 404,\n "message": "Dataset \'top_delivery_times\' not found."\n}\n')),(0,i.kt)("p",null,"The UI will be updated accordingly. Note that the dataset is nowhere to be found in the ",(0,i.kt)("inlineCode",{parentName:"p"},"DATASETS")," view or lineage map:"),(0,i.kt)("p",null,(0,i.kt)("img",{alt:"Marquez Datasets view",src:a(81703).Z,width:"1561",height:"646"})),(0,i.kt)("p",null,(0,i.kt)("img",{alt:"Marquez Map view",src:a(53490).Z,width:"1660",height:"627"})),(0,i.kt)("h2",{id:"whats-next"},"What\u2019s Next"),(0,i.kt)("p",null,"A new endpoint to make namespaces deletable is in progress (",(0,i.kt)("a",{parentName:"p",href:"https://github.com/MarquezProject/marquez/issues/2095"},"Issue #2095")," & ",(0,i.kt)("a",{parentName:"p",href:"https://github.com/MarquezProject/marquez/pull/2172"},"PR #2172"),"). The main goal of this addition is to automate the deletion of a large number of datasets and jobs where doing so individually would be a nuisance."),(0,i.kt)("p",null,"The frontend work is in the design stage. Some questions remain: How should the UI be modified to permit the selection of jobs and datasets for deleti
1on? Should there be a new clickable element in every dataset and job in the graph? Should datasets and jobs be deletable within search results?"),(0,i.kt)("p",null,"Whoever takes on this task will have a shaping role in the discussion and ultimate form of this key addition."),(0,i.kt)("h2",{id:"getting-started"},"Getting Started"),(0,i.kt)("p",null,"Contributions to this ongoing effort at implementing a soft delete option Marquez are welcome. If you are interested in contributing, opening an ",(0,i.kt)("a",{parentName:"p",href:"https://github.com/MarquezProject/marquez/issues"},"issue")," is a good way to get started. "),(0,i.kt)("p",null,"Check out our ",(0,i.kt)("a",{parentName:"p",href:"https://github.com/OpenLineage/OpenLineage/blob/main/CONTRIBUTING.md"},"new contributor guide")," to learn more about how to contribute to the project."))}h.isMDXComponent=!0},35202:(e,t,a)=>{a.d(t,{Z:()=>n});const n=a.p+"assets/images/dataset_verify-582f71065674111ce3e770e67efa5f15.png"},81703:(e,t,a)=>{a.d(t,{Z:()=>n});const n=a.p+"assets/images/datasets_delete_view-922b256def8de532fadd000e8350a678.png"},53490:(e,t,a)=>{a.d(t,{Z:()=>n});const n=a.p+"assets/images/map_delete_view-1133ad3f11d0767806270094f9f99d9d.png"}}]);
Line numbers count LF bytes from the start of the resource, as the search results do. Vendor segments are library code the classifier recognised; they are stored but not indexed. Bytes are shown as Latin1 characters, one per byte.