https://docs.formkiq.com/assets/js/48c45942.00822abe.js

1"use strict";(self.webpackChunkdocs_formkiq_com=self.webpackChunkdocs_formkiq_com||[]).push([[1310],{39596:(e,t,n)=>{n.r(t),n.d(t,{assets:()=>d,contentTitle:()=>l,default:()=>h,frontMatter:()=>o,metadata:()=>i,toc:()=>a});const i=JSON.parse('{"id":"tutorials/FormKiQ CLI/fk-cli-import-csv-data-migration","title":"Import CSV Data migration using the FormKiQ CLI","description":"Step-by-step guide to migrating documents, attributes, and content into FormKiQ using the fk CLI CSV import commands.","source":"@site/docs/tutorials/FormKiQ CLI/csv-import-migration.md","sourceDirName":"tutorials/FormKiQ CLI","slug":"/tutorials/fk-cli-import-csv-data-migration","permalink":"/docs/tutorials/fk-cli-import-csv-data-migration","draft":false,"unlisted":false,"editUrl":"https://github.com/formkiq/docs.formkiq.com/tree/main/docs/tutorials/FormKiQ CLI/csv-import-migration.md","tags":[{"inline":true,"label":"tutorial","permalink":"/docs/tags/tutorial"},{"inline":true,"label":"formkiq","permalink":"/docs/tags/formkiq"},{"inline":true,"label":"cli","permalink":"/docs/tags/cli"},{"inline":true,"label":"migration","permalink":"/docs/tags/migration"}],"version":"current","sidebarPosition":1,"frontMatter":{"id":"fk-cli-import-csv-data-migration","title":"Import CSV Data migration using the FormKiQ CLI","sidebar_position":1,"sidebar_label":"Import-CSV Data Migration","slug":"/tutorials/fk-cli-import-csv-data-migration","description":"Step-by-step guide to migrating documents, attributes, and content into FormKiQ using the fk CLI CSV import commands.","tags":["tutorial","formkiq","cli","migration"]},"sidebar":"tutorialSidebar","previous":{"title":"FormKiQ CLI","permalink":"/docs/category/formkiq-cli"},"next":{"title":"DynamoDb Data migration","permalink":"/docs/tutorials/dynamodb-data-migration"}}');var r=n(74848),s=n(28453);const o={id:"fk-cli-import-csv-data-migration",title:"Import CSV Data migration using the FormKiQ CLI",sidebar_position:1,sidebar_label:"Import-CSV Data Migration",slug:"/tutorials/fk-cli-import-csv-data-migration",description:"Step-by-step guide to migrating documents, attributes, and content into FormKiQ using the fk CLI CSV import commands.",tags:["tutorial","formkiq","cli","migration"]},l="Import CSV Data Migration Using the FormKiQ CLI",d={},a=[{value:"What You Will Build",id:"what-you-will-build",level:2},{value:"Before You Begin",id:"before-you-begin",level:2},{value:"Workflow Overview",id:"workflow-overview",level:2},{value:"Step 1: Verify the CLI Setup",id:"step-1-verify-the-cli-setup",level:2},{value:"Install and configure the FormKiQ CLI",id:"install-and-configure-the-formkiq-cli",level:3},{value:"Migration order (important)",id:"migration-order-important",level:3},{value:"Step 2: Import Attributes",id:"step-2-import-attributes",level:2},{value:"Step 3: Import Documents",id:"step-3-import-documents",level:2},{value:"Step 4: Import Document Attributes",id:"step-4-import-document-attributes",level:2},{value:"Step 5: Import Document Contents",id:"step-5-import-document-contents",level:2},{value:"Verify the Result",id:"verify-the-result",level:2},{value:"Performance and environment considerations",id:"performance-and-environment-considerations",level:2},{value:"Local machine",id:"local-machine",level:3},{value:"AWS CloudShell",id:"aws-cloudshell",level:3},{value:"Amazon EC2 (recommended)",id:"amazon-ec2-recommended",level:3},{value:"Clean Up",id:"clean-up",level:2},{value:"Troubleshooting",id:"troubleshooting",level:2},{value:"Next Steps",id:"next-steps",level:2}];function c(e){const t={a:"a",admonition:"admonition",br:"br",code:"code",h1:"h1",h2:"h2",h3:"h3",header:"header",li:"li",ol:"ol",p:"p",pre:"pre",strong:"strong",table:"table",tbody:"tbody",td:"td",th:"th",thead:"thead",tr:"tr",ul:"ul",...(0,s.R)(),...e.components};return(0,r.jsxs)(r.Fragment,{children:[(0,r.jsx)(t.header,{children:(0,r.jsx)(t.h1,{id:"import-csv-data-migration-using-the-formkiq-cli",children:"Import CSV Data Migration Using the FormKiQ CLI"})}),"\n",(0,r.jsx)(t.h2,{id:"what-you-will-build",children:"What You Will Build"}),"\n",(0,r.jsxs)(t.p,{children:["This tutorial explains how to perform a ",(0,r.jsx)(t.strong,{children:"bulk data migration into FormKiQ"})," using the ",(0,r.jsxs)(t.strong,{children:["FormKiQ CLI (",(0,r.jsx)(t.code,{children:"fk"}),")"]})," and its built-in ",(0,r.jsx)(t.code,{children:"--import-csv"})," functionality. The CLI importer is designed for efficiently loading large volumes of documents and metadata during initial onboarding or system migrations."]}),"\n",(0,r.jsxs)(t.p,{children:["You will learn how to import ",(0,r.jsx)(t.strong,{children:"attributes"}),", ",(0,r.jsx)(t.strong,{children:"documents"}),", ",(0,r.jsx)(t.strong,{children:"document content"}),", and ",(0,r.jsx)(t.strong,{children:"document attributes"})," in a safe, repeatable order that supports re-runs without creating duplicates."]}),"\n",(0,r.jsx)(t.h2,{id:"before-you-begin",children:"Before You Begin"}),"\n",(0,r.jsx)(t.p,{children:"Before starting the migration, ensure you have:"}),"\n",(0,r.jsxs)(t.ul,{children:["\n",(0,r.jsxs)(t.li,{children:["The ",(0,r.jsxs)(t.strong,{children:["FormKiQ CLI (",(0,r.jsx)(t.code,{children:"fk"}),")"]})," installed and configured"]}),"\n",(0,r.jsx)(t.li,{children:"Access credentials to your FormKiQ environment"}),"\n",(0,r.jsx)(t.li,{children:"CSV files prepared for each import stage"}),"\n",(0,r.jsx)(t.li,{children:"UUID v4 values generated for documents"}),"\n",(0,r.jsx)(t.li,{children:"Sufficient CPU and network bandwidth on the machine running the CLI"}),"\n"]}),"\n",(0,r.jsx)(t.h2,{id:"workflow-overview",children:"Workflow Overview"}),"\n",(0,r.jsxs)(t.ol,{children:["\n",(0,r.jsx)(t.li,{children:"Verify the CLI configuration."}),"\n",(0,r.jsx)(t.li,{children:"Import attribute definitions."}),"\n",(0,r.jsx)(t.li,{children:"Import document records."}),"\n",(0,r.jsx)(t.li,{children:"Import document attribute values."}),"\n",(0,r.jsx)(t.li,{children:"Import document content."}),"\n",(0,r.jsx)(t.li,{children:"Verify each import stage."}),"\n",(0,r.jsx)(t.li,{children:"Review performance and troubleshooting guidance."}),"\n"]}

1),"\n",(0,r.jsx)(t.h2,{id:"step-1-verify-the-cli-setup",children:"Step 1: Verify the CLI Setup"}),"\n",(0,r.jsx)(t.h3,{id:"install-and-configure-the-formkiq-cli",children:"Install and configure the FormKiQ CLI"}),"\n",(0,r.jsx)(t.p,{children:"Verify the CLI is available:"}),"\n",(0,r.jsx)(t.pre,{children:(0,r.jsx)(t.code,{className:"language-bash",children:"fk --version\n"})}),"\n",(0,r.jsx)(t.p,{children:"Ensure the CLI is authenticated and pointing to the correct environment:"}),"\n",(0,r.jsx)(t.pre,{children:(0,r.jsx)(t.code,{className:"language-bash",children:"fk status\n"})}),"\n",(0,r.jsx)(t.h3,{id:"migration-order-important",children:"Migration order (important)"}),"\n",(0,r.jsx)(t.p,{children:"Always run imports in the following order:"}),"\n",(0,r.jsxs)(t.ul,{children:["\n",(0,r.jsx)(t.li,{children:"Attributes"}),"\n",(0,r.jsx)(t.li,{children:"Documents"}),"\n",(0,r.jsx)(t.li,{children:"Document attributes"}),"\n",(0,r.jsx)(t.li,{children:"Document contents"}),"\n"]}),"\n",(0,r.jsx)(t.p,{children:"This order ensures all referenced objects exist before they are used."}),"\n",(0,r.jsx)(t.admonition,{type:"note",children:(0,r.jsxs)(t.p,{children:["When importing a large number of documents, review ",(0,r.jsx)(t.a,{href:"/docs/platform/overview#scaling-formkiq-components",children:"Scaling FormKiQ Components"})," to reduce overall migration time."]})}),"\n",(0,r.jsx)(t.h2,{id:"step-2-import-attributes",children:"Step 2: Import Attributes"}),"\n",(0,r.jsx)(t.p,{children:"Attributes define the metadata fields that can later be assigned to documents."}),"\n",(0,r.jsx)(t.p,{children:(0,r.jsx)(t.strong,{children:"CSV format"})}),"\n",(0,r.jsx)(t.pre,{children:(0,r.jsx)(t.code,{children:"AttributeKey,DataType,Type\nstatus,STRING,STANDARD\npriority,NUMBER,STANDARD\nreviewed,BOOLEAN,STANDARD\n"})}),"\n",(0,r.jsxs)(t.ul,{children:["\n",(0,r.jsx)(t.li,{children:"AttributeKey: Unique attribute identifier"}),"\n",(0,r.jsx)(t.li,{children:"DataType: STRING, NUMBER, BOOLEAN, or KEY_ONLY"}),"\n",(0,r.jsx)(t.li,{children:"Type: STANDARD, GOVERNANCE, or OPA"}),"\n"]}),"\n",(0,r.jsx)(t.p,{children:(0,r.jsx)(t.strong,{children:"Command"})}),"\n",(0,r.jsx)(t.pre,{children:(0,r.jsx)(t.code,{className:"language-bash",children:"fk --import-csv --attributes attributes.csv\n"})}),"\n",(0,r.jsx)(t.p,{children:"This step can be safely re-run to add or update attribute definitions."}),"\n",(0,r.jsx)(t.h2,{id:"step-3-import-documents",children:"Step 3: Import Documents"}),"\n",(0,r.jsx)(t.p,{children:"This step registers document records in FormKiQ without uploading content yet."}),"\n",(0,r.jsx)(t.p,{children:(0,r.jsx)(t.strong,{children:"CSV format"})}),"\n",(0,r.jsx)(t.pre,{children:(0,r.jsx)(t.code,{children:"DocumentId,Path,ContentType,DeepLink\n550e8400-e29b-41d4-a716-446655440000,/invoices/2025/05/001.pdf,application/pdf,\n123e4567-e89b-12d3-a456-426614174000,/reports/2025/Q1.xlsx,application/vnd.openxmlformats-officedocument.spreadsheetml.sheet,\n"})}),"\n",(0,r.jsxs)(t.ul,{children:["\n",(0,r.jsx)(t.li,{children:"DocumentId: UUID v4 (user-supplied)"}),"\n",(0,r.jsx)(t.li,{children:"Path: Virtual FormKiQ path"}),"\n",(0,r.jsx)(t.li,{children:"ContentType: MIME type"}),"\n",(0,r.jsx)(t.li,{children:"DeepLink: Optional external URL"}),"\n"]}),"\n",(0,r.jsx)(t.admonition,{type:"note",children:(0,r.jsx)(t.p,{children:"Using a consistent DocumentId allows you to re-run imports without creating duplicates."})}),"\n",(0,r.jsx)(t.p,{children:(0,r.jsx)(t.strong,{children:"Command"})}),"\n",(0,r.jsx)(t.pre,{children:(0,r.jsx)(t.code,{className:"language-bash",children:"fk --import-csv --documents documents.csv\n"})}),"\n",(0,r.jsx)(t.h2,{id:"step-4-import-document-attributes",children:"Step 4: Import Document Attributes"}),"\n",(0,r.jsx)(t.p,{children:"This step assigns metadata values to existing documents."}),"\n",(0,r.jsx)(t.p,{children:(0,r.jsx)(t.strong,{children:"CSV format"})}),"\n",(0,r.jsx)(t.pre,{children:(0,r.jsx)(t.code,{children:"DocumentId,AttributeKey,StringValue,NumberValue,BooleanValue\n550e8400-e29b-41d4-a716-446655440000,status,approved,,\n550e8400-e29b-41d4-a716-446655440000,priority,,5,\n123e4567-e89b-12d3-a456-426614174000,isPublished,,,true\n"})}),"\n",(0,r.jsx)(t.p,{children:"Rules:"}),"\n",(0,r.jsxs)(t.ul,{children:["\n",(0,r.jsx)(t.li,{children:"DocumentId must already exist"}),"\n",(0,r.jsx)(t.li,{children:"AttributeKey must be defined"}),"\n",(0,r.jsx)(t.li,{children:"Only one value column should be populated per row"}),"\n"]}),"\n",(0,r.jsx)(t.p,{children:(0,r.jsx)(t.strong,{children:"Command"})}),"\n",(0,r.jsx)(t.pre,{children:(0,r.jsx)(t.code,{className:"language-bash",children:"fk --import-csv --document-attributes document-attributes.csv\n"})}),"\n",(0,r.jsx)(t.p,{children:"This step performs the bulk data transfer and is the most resource-intensive part of the migration."}),"\n",(0,r.jsx)(t.h2,{id:"step-5-import-document-contents",children:"Step 5: Import Document Contents"}),"\n",(0,r.jsx)(t.p,{children:"This step uploads or links the actual binary content for each document."}),"\n",(0,r.jsx)(t.p,{children:(0,r.jsx)(t.strong,{children:"CSV format"})}),"\n",(0,r.jsx)(t.pre,{children:(0,r.jsx)(t.code,{children:"DocumentId,Location\n550e8400-e29b-41d4-a716-446655440000,/path/to/file.pdf\n123e4567-e89b-12d3-a456-426614174000,s3://my-bucket/documents/report.xlsx\n"})}),"\n",(0,r.jsx)(t.p,{children:"Location can be:"}),"\n",(0,r.jsxs)(t.ul,{children:["\n",(0,r.jsx)(t.li,{children:"A local filesystem path"}),"\n",(0,r.jsx)(t.li,{children:"An S3 URI (s3://bucket/key)"}),"\n"]}),"\n",(0,r.jsx)(t.p,{children:(0,r.jsx)(t.strong,{children:"Command"})}),"\n",(0,r.jsx)(t.pre,{children:(0,r.jsx)(t.code,{className:"language-bash",children:"fk --import-csv --document-contents document-contents.csv\n"})}),"\n",(0,r.jsx)(t.p,{children:"This step performs the bulk data transfer and is the most resource-intensive part of the migration."}),"\n",(0,r.jsx)(t.h2,{id:"verify-the-result",children:"Verify the Result"}),"\n",(0,r.jsxs)(t.p,{children:["After running an import, you can use the ",(0,r.jsx)(t.code,{children:"--verify"}

1)," option to ",(0,r.jsx)(t.strong,{children:"validate the imported data against FormKiQ"}),".",(0,r.jsx)(t.br,{}),"\n","This verification step confirms that the data written during the import is ",(0,r.jsx)(t.strong,{children:"consistent with FormKiQ\u2019s internal rules and state"}),"."]}),"\n",(0,r.jsx)(t.p,{children:"The verification process checks:"}),"\n",(0,r.jsxs)(t.ul,{children:["\n",(0,r.jsxs)(t.li,{children:["That imported ",(0,r.jsx)(t.strong,{children:"attributes"})," are correctly registered"]}),"\n",(0,r.jsxs)(t.li,{children:["That ",(0,r.jsx)(t.strong,{children:"documents"})," exist and are accessible"]}),"\n",(0,r.jsxs)(t.li,{children:["That ",(0,r.jsx)(t.strong,{children:"document content"})," is properly associated"]}),"\n",(0,r.jsxs)(t.li,{children:["That ",(0,r.jsx)(t.strong,{children:"document attributes"})," are valid and resolvable"]}),"\n",(0,r.jsx)(t.li,{children:"That relationships and data types conform to FormKiQ expectations"}),"\n"]}),"\n",(0,r.jsx)(t.admonition,{type:"note",children:(0,r.jsxs)(t.p,{children:["The ",(0,r.jsx)(t.code,{children:"--verify"})," option is run ",(0,r.jsx)(t.strong,{children:"after an import has completed"}),".",(0,r.jsx)(t.br,{}),"\n","It does ",(0,r.jsx)(t.strong,{children:"not"})," validate CSV structure or local file contents, and it does ",(0,r.jsx)(t.strong,{children:"not"})," modify any data."]})}),"\n",(0,r.jsx)(t.p,{children:(0,r.jsx)(t.strong,{children:"Example: verify imported attributes"})}),"\n",(0,r.jsx)(t.pre,{children:(0,r.jsx)(t.code,{className:"language-bash",children:"fk --import-csv --attributes attributes.csv --verify\n"})}),"\n",(0,r.jsx)(t.p,{children:(0,r.jsx)(t.strong,{children:"Example: verify imported documents"})}),"\n",(0,r.jsx)(t.pre,{children:(0,r.jsx)(t.code,{className:"language-bash",children:"fk --import-csv --documents documents.csv --verify\n"})}),"\n",(0,r.jsx)(t.p,{children:(0,r.jsx)(t.strong,{children:"Example: verify imported document attributes"})}),"\n",(0,r.jsx)(t.pre,{children:(0,r.jsx)(t.code,{className:"language-bash",children:"fk --import-csv --document-attributes document-attributes.csv --verify\n"})}),"\n",(0,r.jsx)(t.p,{children:(0,r.jsx)(t.strong,{children:"Example: verify imported document contents"})}),"\n",(0,r.jsx)(t.pre,{children:(0,r.jsx)(t.code,{className:"language-bash",children:"fk --import-csv --document-contents document-contents.csv --verify\n"})}),"\n",(0,r.jsx)(t.admonition,{type:"tip",children:(0,r.jsx)(t.p,{children:"Run verification immediately after each import stage to catch issues early before proceeding to the next step in the migration."})}),"\n",(0,r.jsx)(t.h2,{id:"performance-and-environment-considerations",children:"Performance and environment considerations"}),"\n",(0,r.jsx)(t.p,{children:"Migration speed is heavily influenced by where the CLI is executed:"}),"\n",(0,r.jsx)(t.h3,{id:"local-machine",children:"Local machine"}),"\n",(0,r.jsx)(t.p,{children:"Limited by your internet bandwidth to AWS"}),"\n",(0,r.jsx)(t.h3,{id:"aws-cloudshell",children:"AWS CloudShell"}),"\n",(0,r.jsx)(t.p,{children:"Convenient, but has CPU and throughput limits"}),"\n",(0,r.jsx)(t.h3,{id:"amazon-ec2-recommended",children:"Amazon EC2 (recommended)"}),"\n",(0,r.jsxs)(t.ul,{children:["\n",(0,r.jsx)(t.li,{children:"Use C5 instances for compute-intensive workloads"}),"\n",(0,r.jsx)(t.li,{children:"High network throughput improves import speed"}),"\n",(0,r.jsx)(t.li,{children:"Spot instances can significantly reduce cost for large migrations"}),"\n"]}),"\n",(0,r.jsx)(t.admonition,{type:"tip",children:(0,r.jsx)(t.p,{children:"Running the CLI on an EC2 instance in the same region as FormKiQ provides the best performance."})}),"\n",(0,r.jsx)(t.h2,{id:"clean-up",children:"Clean Up"}),"\n",(0,r.jsx)(t.p,{children:"Archive the import CSV files and CLI output logs with your migration records. Remove local files containing sensitive data after validation is complete."}),"\n",(0,r.jsx)(t.h2,{id:"troubleshooting",children:"Troubleshooting"}),"\n",(0,r.jsxs)(t.table,{children:[(0,r.jsx)(t.thead,{children:(0,r.jsxs)(t.tr,{children:[(0,r.jsx)(t.th,{children:"Problem"}),(0,r.jsx)(t.th,{children:"Likely cause"}),(0,r.jsx)(t.th,{children:"What to check"})]})}),(0,r.jsxs)(t.tbody,{children:[(0,r.jsxs)(t.tr,{children:[(0,r.jsx)(t.td,{children:"Document not found"}),(0,r.jsx)(t.td,{children:"Document content or attributes were imported before the document records."}),(0,r.jsx)(t.td,{children:"Run the import stages in the documented order."})]}),(0,r.jsxs)(t.tr,{children:[(0,r.jsx)(t.td,{children:"Attribute does not exist"}),(0,r.jsx)(t.td,{children:"Document attributes reference keys that were not created."}),(0,r.jsx)(t.td,{children:"Verify the attributes import before assigning document attributes."})]}),(0,r.jsxs)(t.tr,{children:[(0,r.jsx)(t.td,{children:"Slow import performance"}),(0,r.jsx)(t.td,{children:"The CLI host, network, API, or downstream services are constrained."}),(0,r.jsx)(t.td,{children:"Run the CLI from EC2 in the same Region and review scaling guidance."})]}),(0,r.jsxs)(t.tr,{children:[(0,r.jsx)(t.td,{children:"Duplicate documents"}),(0,r.jsx)(t.td,{children:"Document IDs were regenerated between import attempts."}),(0,r.jsxs)(t.td,{children:["Use stable UUIDs in the ",(0,r.jsx)(t.code,{children:"DocumentId"})," column."]})]})]})]}),"\n",(0,r.jsx)(t.h2,{id:"next-steps",children:"Next Steps"}),"\n",(0,r.jsx)(t.p,{children:"After completing the migration, validate imported data using the FormKiQ UI or API."}),"\n",(0,r.jsxs)(t.ul,{children:["\n",(0,r.jsx)(t.li,{children:(0,r.jsx)(t.a,{href:"/docs/formkiq-modules/modules/form

1kiq-cli",children:"FormKiQ CLI"})}),"\n",(0,r.jsx)(t.li,{children:(0,r.jsx)(t.a,{href:"/docs/platform/migration-and-data-import",children:"Migration and Data Import"})}),"\n"]})]})}function h(e={}){const{wrapper:t}={...(0,s.R)(),...e.components};return t?(0,r.jsx)(t,{...e,children:(0,r.jsx)(c,{...e})}):c(e)}}}]);