https://java2ai.com/assets/js/9071c71f.35b2562c.js

1"use strict";(self.webpackChunkspring_ai_alibaba_website=self.webpackChunkspring_ai_alibaba_website||[]).push([[6415],{410:(e,i,t)=>{t.d(i,{A:()=>n});const n=t.p+"assets/images/multimodal.test-23deeeaefa6b2b3db6c0674f6405f9fe.png"},28453:(e,i,t)=>{t.d(i,{R:()=>l,x:()=>r});var n=t(96540);const a={},s=n.createContext(a);function l(e){const i=n.useContext(s);return n.useMemo(function(){return"function"==typeof e?e(i):{...i,...e}},[i,e])}function r(e){let i;return i=e.disableParentContext?"function"==typeof e.components?e.components(a):e.components||a:l(e.components),n.createElement(s.Provider,{value:i},e.children)}},58148:(e,i,t)=>{t.r(i),t.d(i,{assets:()=>o,contentTitle:()=>r,default:()=>h,frontMatter:()=>l,metadata:()=>n,toc:()=>d});const n=JSON.parse('{"id":"multimodals/multimodality","title":"Multimodality API","description":"\\"\u6240\u6709\u81ea\u7136\u8fde\u63a5\u7684\u4e8b\u7269\u90fd\u5e94\u8be5\u7ec4\u5408\u5728\u4e00\u8d77\u6559\u6388\\" - John Amos Comenius, \\"Orbis Sensualium Pictus\\", 1658","source":"@site/integration/multimodals/multimodality.md","sourceDirName":"multimodals","slug":"/multimodals/multimodality","permalink":"/integration/multimodals/multimodality","draft":false,"unlisted":false,"editUrl":"https://github.com/spring-ai-alibaba/website/tree/main/integration/multimodals/multimodality.md","tags":[],"version":"current","lastUpdatedBy":"Ken Liu","lastUpdatedAt":1768271906000,"frontMatter":{},"sidebar":"integrationSidebar","previous":{"title":"ZhiPu AI Chat","permalink":"/integration/chatmodels/more/zhipuai-chat"},"next":{"title":"Text-To-Speech (TTS) API","permalink":"/integration/multimodals/audio/speech"}}');var a=t(74848),s=t(28453);const l={},r="Multimodality API",o={},d=[{value:"Spring AI Multimodality",id:"spring-ai-multimodality",level:2}];function c(e){const i={a:"a",blockquote:"blockquote",code:"code",h1:"h1",h2:"h2",header:"header",img:"img",li:"li",p:"p",pre:"pre",strong:"strong",ul:"ul",...(0,s.R)(),...e.components};return(0,a.jsxs)(a.Fragment,{children:[(0,a.jsx)(i.header,{children:(0,a.jsx)(i.h1,{id:"multimodality-api",children:"Multimodality API"})}),"\n",(0,a.jsxs)(i.blockquote,{children:["\n",(0,a.jsx)(i.p,{children:'"\u6240\u6709\u81ea\u7136\u8fde\u63a5\u7684\u4e8b\u7269\u90fd\u5e94\u8be5\u7ec4\u5408\u5728\u4e00\u8d77\u6559\u6388" - John Amos Comenius, "Orbis Sensualium Pictus", 1658'}),"\n"]}),"\n",(0,a.jsx)(i.p,{children:"\u4eba\u7c7b\u540c\u65f6\u8de8\u591a\u79cd\u6570\u636e\u8f93\u5165\u6a21\u5f0f\u5904\u7406\u77e5\u8bc6\u3002\n\u6211\u4eec\u5b66\u4e60\u7684\u65b9\u5f0f\uff0c\u6211\u4eec\u7684\u7ecf\u5386\u90fd\u662f\u591a\u6a21\u6001\u7684\u3002\n\u6211\u4eec\u4e0d\u4ec5\u6709\u89c6\u89c9\uff0c\u8fd8\u6709\u97f3\u9891\u548c\u6587\u672c\u3002"}),"\n",(0,a.jsx)(i.p,{children:"\u4e0e\u8fd9\u4e9b\u539f\u5219\u76f8\u53cd\uff0c\u673a\u5668\u5b66\u4e60\u901a\u5e38\u4e13\u6ce8\u4e8e\u5904\u7406\u5355\u4e00\u6a21\u6001\u7684\u4e13\u7528\u6a21\u578b\u3002\n\u4f8b\u5982\uff0c\u6211\u4eec\u5f00\u53d1\u4e86\u7528\u4e8e\u6587\u672c\u8f6c\u8bed\u97f3\u6216\u8bed\u97f3\u8f6c\u6587\u672c\u7b49\u4efb\u52a1\u7684\u97f3\u9891\u6a21\u578b\uff0c\u4ee5\u53ca\u7528\u4e8e\u5bf9\u8c61\u68c0\u6d4b\u548c\u5206\u7c7b\u7b49\u4efb\u52a1\u7684\u8ba1\u7b97\u673a\u89c6\u89c9\u6a21\u578b\u3002"}),"\n",(0,a.jsx)(i.p,{children:"\u7136\u800c\uff0c\u65b0\u4e00\u6ce2\u591a\u6a21\u6001\u5927\u8bed\u8a00\u6a21\u578b\u5f00\u59cb\u51fa\u73b0\u3002\n\u793a\u4f8b\u5305\u62ec OpenAI \u7684 GPT-4o\u3001Google \u7684 Vertex AI Gemini 1.5\u3001Anthropic \u7684 Claude3\uff0c\u4ee5\u53ca\u5f00\u6e90\u4ea7\u54c1 Llama3.2\u3001LLaVA \u548c BakLLaVA \u80fd\u591f\u63a5\u53d7\u591a\u79cd\u8f93\u5165\uff0c\u5305\u62ec\u6587\u672c\u56fe\u50cf\u3001\u97f3\u9891\u548c\u89c6\u9891\uff0c\u5e76\u901a\u8fc7\u6574\u5408\u8fd9\u4e9b\u8f93\u5165\u751f\u6210\u6587\u672c\u54cd\u5e94\u3002"}),"\n",(0,a.jsxs)(i.blockquote,{children:["\n",(0,a.jsxs)(i.p,{children:[(0,a.jsx)(i.strong,{children:"NOTE:"})," \u591a\u6a21\u6001\u5927\u8bed\u8a00\u6a21\u578b\uff08LLM\uff09\u529f\u80fd\u4f7f model \u80fd\u591f\u5904\u7406\u548c\u751f\u6210\u6587\u672c\uff0c\u5e76\u7ed3\u5408\u5176\u4ed6\u6a21\u6001\uff0c\u5982\u56fe\u50cf\u3001\u97f3\u9891\u6216\u89c6\u9891\u3002"]}),"\n"]}),"\n",(0,a.jsx)(i.h2,{id:"spring-ai-multimodality",children:"Spring AI Multimodality"}),"\n",(0,a.jsx)(i.p,{children:"\u591a\u6a21\u6001\u662f\u6307 model \u540c\u65f6\u7406\u89e3\u548c\u5904\u7406\u6765\u81ea\u5404\u79cd\u6765\u6e90\u7684\u4fe1\u606f\u7684\u80fd\u529b\uff0c\u5305\u62ec\u6587\u672c\u3001\u56fe\u50cf\u3001\u97f3\u9891\u548c\u5176\u4ed6\u6570\u636e\u683c\u5f0f\u3002"}),"\n",(0,a.jsx)(i.p,{children:"Spring AI Message API \u63d0\u4f9b\u4e86\u652f\u6301\u591a\u6a21\u6001 LLM \u6240\u9700\u7684\u6240\u6709\u62bd\u8c61\u3002"}),"\n",(0,a.jsx)(i.p,{children:(0,a.jsx)(i.img,{alt:"Spring AI Message API",src:t(78709).A+"",width:"4429",height:"3611"})}),"\n",(0,a.jsxs)(i.p,{children:["UserMessage \u7684 ",(0,a.jsx)(i.co

1de,{children:"content"})," \u5b57\u6bb5\u4e3b\u8981\u7528\u4e8e\u6587\u672c\u8f93\u5165\uff0c\u800c\u53ef\u9009\u7684 ",(0,a.jsx)(i.code,{children:"media"})," \u5b57\u6bb5\u5141\u8bb8\u6dfb\u52a0\u4e00\u4e2a\u6216\u591a\u4e2a\u4e0d\u540c\u6a21\u6001\u7684\u9644\u52a0\u5185\u5bb9\uff0c\u5982\u56fe\u50cf\u3001\u97f3\u9891\u548c\u89c6\u9891\u3002\n",(0,a.jsx)(i.code,{children:"MimeType"})," \u6307\u5b9a\u6a21\u6001\u7c7b\u578b\u3002\n\u6839\u636e\u4f7f\u7528\u7684 LLM\uff0c",(0,a.jsx)(i.code,{children:"Media"})," \u6570\u636e\u5b57\u6bb5\u53ef\u4ee5\u662f\u4f5c\u4e3a ",(0,a.jsx)(i.code,{children:"Resource"})," \u5bf9\u8c61\u7684\u539f\u59cb\u5a92\u4f53\u5185\u5bb9\uff0c\u4e5f\u53ef\u4ee5\u662f\u5185\u5bb9\u7684 ",(0,a.jsx)(i.code,{children:"URI"}),"\u3002"]}),"\n",(0,a.jsxs)(i.blockquote,{children:["\n",(0,a.jsxs)(i.p,{children:[(0,a.jsx)(i.strong,{children:"NOTE:"})," media \u5b57\u6bb5\u76ee\u524d\u4ec5\u9002\u7528\u4e8e\u7528\u6237\u8f93\u5165\u6d88\u606f\uff08\u4f8b\u5982\uff0c",(0,a.jsx)(i.code,{children:"UserMessage"}),"\uff09\u3002\u5b83\u5bf9\u7cfb\u7edf\u6d88\u606f\u6ca1\u6709\u610f\u4e49\u3002\u5305\u542b LLM \u54cd\u5e94\u7684 ",(0,a.jsx)(i.code,{children:"AssistantMessage"})," \u4ec5\u63d0\u4f9b\u6587\u672c\u5185\u5bb9\u3002\u8981\u751f\u6210\u975e\u6587\u672c\u5a92\u4f53\u8f93\u51fa\uff0c\u60a8\u5e94\u8be5\u4f7f\u7528\u4e13\u7528\u7684\u5355\u6a21\u6001 model \u4e4b\u4e00\u3002"]}),"\n"]}),"\n",(0,a.jsxs)(i.p,{children:["\u4f8b\u5982\uff0c\u6211\u4eec\u53ef\u4ee5\u5c06\u4ee5\u4e0b\u56fe\u7247\uff08",(0,a.jsx)(i.code,{children:"multimodal.test.png"}),"\uff09\u4f5c\u4e3a\u8f93\u5165\uff0c\u5e76\u8981\u6c42 LLM \u89e3\u91ca\u5b83\u770b\u5230\u7684\u5185\u5bb9\u3002"]}),"\n",(0,a.jsx)(i.p,{children:(0,a.jsx)(i.img,{alt:"Multimodal Test Image",src:t(410).A+"",width:"532",height:"350"})}),"\n",(0,a.jsx)(i.p,{children:"\u5bf9\u4e8e\u5927\u591a\u6570\u591a\u6a21\u6001 LLM\uff0cSpring AI \u4ee3\u7801\u770b\u8d77\u6765\u50cf\u8fd9\u6837\uff1a"}),"\n",(0,a.jsx)(i.pre,{children:(0,a.jsx)(i.code,{className:"language-java",children:'var imageResource = new ClassPathResource("/multimodal.test.png");\n\nvar userMessage = UserMessage.builder()\n    .text("Explain what do you see in this picture?") // content\n    .media(new Media(MimeTypeUtils.IMAGE_PNG, this.imageResource)) // media\n    .build();\n\nChatResponse response = chatModel.call(new Prompt(this.userMessage));\n'})}),"\n",(0,a.jsxs)(i.p,{children:["\u6216\u4f7f\u7528\u6d41\u5f0f ",(0,a.jsx)(i.a,{href:"../chatclient",children:"ChatClient"})," API\uff1a"]}),"\n",(0,a.jsx)(i.pre,{children:(0,a.jsx)(i.code,{className:"language-java",children:'String response = ChatClient.create(chatModel).prompt()\n\t\t.user(u -> u.text("Explain what do you see on this picture?")\n\t\t\t\t    .media(MimeTypeUtils.IMAGE_PNG, new ClassPathResource("/multimodal.test.png")))\n\t\t.call()\n\t\t.content();\n'})}),"\n",(0,a.jsx)(i.p,{children:"\u5e76\u4ea7\u751f\u5982\u4e0b\u54cd\u5e94\uff1a"}),"\n",(0,a.jsxs)(i.blockquote,{children:["\n",(0,a.jsx)(i.p,{children:"This is an image of a fruit bowl with a simple design. The bowl is made of metal with curved wire edges that create an open structure, allowing the fruit to be visible from all angles. Inside the bowl, there are two yellow bananas resting on top of what appears to be a red apple. The bananas are slightly overripe, as indicated by the brown spots on their peels. The bowl has a metal ring at the top, likely to serve as a handle for carrying. The bowl is placed on a flat surface with a neutral-colored background that provides a clear view of the fruit inside."}),"\n"]}),"\n",(0,a.jsx)(i.p,{children:"Spring AI \u4e3a\u4ee5\u4e0b chat model \u63d0\u4f9b\u591a\u6a21\u6001\u652f\u6301\uff1a"}),"\n",(0,a.jsxs)(i.ul,{children:["\n",(0,a.jsx)(i.li,{children:(0,a.jsx)(i.a,{href:"https://docs.spring.io/spring-ai/reference/api/chat/anthropic-chat.html#_multimodal",children:"Anthropic Claude 3"})}),"\n",(0,a.jsx)(i.li,{children:(0,a.jsx)(i.a,{href:"../chatmodels/more/bedrock-converse#_multimodal",children:"AWS Bedrock Converse"})}),"\n",(0,a.jsx)(i.li,{children:(0,a.jsx)(i.a,{href:"../chatmodels/more/azure-openai-chat#_multimodal",children:"Azure Open AI\uff08\u4f8b\u5982 GPT-4o model\uff09"})}),"\n",(0,a.jsx)(i.li,{children:(0,a.jsx)(i.a,{href:"../chatmodels/more/mistralai-chat#_multimodal",children:"Mistral AI\uff08\u4f8b\u5982 Mistral Pixtral model\uff09"})}),"\n",(0,a.jsx)(i.li,{children:(0,a.jsx)(i.a,{href:"../chatmodels/ollama-chat#_multimodal",children:"Ollama\uff08\u4f8b\u5982 LLaVA\u3001BakLLaVA\u3001Llama3.2 model\uff09"})}),"\n",(0,a.jsx)(i.li,{children:(0,a.jsx)(i.a,{href:"../chatmodels/openai-chat#_multimodal",children:"OpenAI\uff08\u4f8b\u5982 GPT-4 \u548c GPT-4o model\uff09"})}),"\n",(0,a.jsx)(i.li,{children:(0,a.jsx)(i.a,{href:"../chatmodels/gemini-chat#_multimodal",children:"Vertex AI Gemini\uff08\u4f8b\u5982 gemini-1.5-pro-001\u3001gemini-1.5-flash-001 model\uff09"})}),"\n"]})]})}function h(e={}){const{wrapper:i}={...(0,s.R)(),...e.components};return i?(0,a.jsx)(i,{...e,children:(0,a.jsx)(c,{...e})}):c(e)}},78709:(e,i,t)=>{t.d(i,{A:()=>n});const n=t.p+"assets/images/spring-ai-message-api-28ea8b962394e036bab091f1d08e922c.jpg"}}

1]);