https://hscarb.github.io/assets/20220301-rocketmq-indexfile.html-99f7f666.js

1import{_ as l}from"./plugin-vue_export-helper-c27b6911.js";import{r as o,o as i,c as u,a as n,b as s,d as a,w as p,e as c}from"./app-d4e6481d.js";const k={},d={href:"http://hscarb.github.io/rocketmq/20220301-rocketmq-indexfile.html",target:"_blank",rel:"noopener noreferrer"},r=n("h1",{id:"rocketmq-indexfile-索引文件",tabindex:"-1"},[n("a",{class:"header-anchor",href:"#rocketmq-indexfile-索引文件","aria-hidden":"true"},"#"),s(" RocketMQ IndexFile 索引文件")],-1),m={class:"table-of-contents"},v=c('<h2 id="_1-概述" tabindex="-1"><a class="header-anchor" href="#_1-概述" aria-hidden="true">#</a> 1. 概述</h2><h3 id="_1-1-索引文件是什么" tabindex="-1"><a class="header-anchor" href="#_1-1-索引文件是什么" aria-hidden="true">#</a> 1.1 索引文件是什么</h3><p>IndexFile，又可以称作索引文件，是 RocketMQ 保存在磁盘上的一种文件，属于 RocketMQ 存储的一部分。它的结构类似于类似 JDK中 HashMap。</p><p>可以通过<code>messageIndexEnable</code>属性配置打开或关闭 IndexFile 存储功能。</p><h3 id="_1-2-索引文件作用" tabindex="-1"><a class="header-anchor" href="#_1-2-索引文件作用" aria-hidden="true">#</a> 1.2 索引文件作用</h3><p>索引文件的应用场景其实比较局限，是为了提供<strong>按照 Message Key 查询消息</strong>的能力。索引文件可以通过 Message Key，查询到消息在 CommitLog 中的物理偏移量，进而从 CommitLog 中查询消息。</p><h2 id="_2-概要设计" tabindex="-1"><a class="header-anchor" href="#_2-概要设计" aria-hidden="true">#</a> 2. 概要设计</h2><h3 id="_2-1-索引文件结构" tabindex="-1"><a class="header-anchor" href="#_2-1-索引文件结构" aria-hidden="true">#</a> 2.1 索引文件结构</h3><p>上面说它的逻辑结构类似 HashMap，HashMap 以 Key-Value 形式存储数据，那么索引文件的存储格式也是 Key-Value</p><ul><li>Key：Message Key。索引文件的 Key 其实是 Message Key 经过 hash 得到的一个 Integer，</li><li>Value：physical offset。索引文件的 Value 主要是消息在 CommitLog 中的绝对物理偏移量。</li></ul><p>hash冲突时，Value以链表的方式存储，越新的消息在链表越前面。</p><p>它可以包含多个文件，每个文件的大小是固定的。这就意味着每个 IndexFile 包含的最大索引数量是相同的。</p><h3 id="_2-2-如何构建" tabindex="-1"><a class="header-anchor" href="#_2-2-如何构建" aria-hidden="true">#</a> 2.2 如何构建</h3><p>消息保存到 CommitLog 之后，会进行重投递。重投递消息的过程就是为了建立消息的索引文件（包括 ConsumeQueue 和 IndexFile）。</p><p>重投递线程会扫描是否有新消息被保存到 CommitLog，如果有则将这条消息查出来，执行重投递逻辑，构建该消息的索引。</p><h3 id="_2-3-如何查询消息" tabindex="-1"><a class="header-anchor" href="#_2-3-如何查询消息" aria-hidden="true">#</a> 2.3 如何查询消息</h3><p>索引文件中存储着 Message Key 对应消息在 CommitLog 中的偏移量，首先查询出这些偏移量信息，然后用偏移量从 CommitLog 中查询出消息。</p><h3 id="_2-4-刷盘机制" tabindex="-1"><a class="header-anchor" href="#_2-4-刷盘机制" aria-hidden="true">#</a> 2.4 刷盘机制</h3><p>索引文件的刷盘机制并不是采取定时刷盘机制，而是每写满一个索引文件时就新建一个文件，并且将上一个写满的索引文件刷盘。</p><h2 id="_3-详细设计" tabindex="-1"><a class="header-anchor" href="#_3-详细设计" aria-hidden="true">#</a> 3. 详细设计</h2><h3 id="_3-1-索引文件结构" tabindex="-1"><a class="header-anchor" href="#_3-1-索引文件结构" aria-hidden="true">#</a> 3.1 索引文件结构</h3><p>设计 IndexFile 最重要的是设计它的逻辑结构和文件存储结构。首先看一下 IndexFile 详细的逻辑结构</p><h4 id="_3-1-1-逻辑结构" tabindex="-1"><a class="header-anchor" href="#_3-1-1-逻辑结构" aria-hidden="true">#</a> 3.1.1 逻辑结构</h4><p>上面已经提到 IndexFile 是类似 JDK 的 HashMap 的结构。</p>',24),b=n("p",null,[s("Key：由 "),n("code",null,"IndexService#buildKey(String topic, String key)"),s("构建而成，具体为 "),n("code",null,'topic + "#" + messageKey'),s(" 经过hash（"),n("code",null,"IndexFile#

1indexKeyHashMethod(String Key)"),s("）得到。")],-1),f={href:"https://github.com/apache/rocketmq/issues/3613",target:"_blank",rel:"noopener noreferrer"},h=n("figure",null,[n("img",{src:"https://scarb-images.oss-cn-hangzhou.aliyuncs.com/img/202203152214095.png",alt:"Untitled",tabindex:"0",loading:"lazy"}),n("figcaption",null,"Untitled")],-1),y=n("li",null,[n("p",null,"Value：Hash 冲突时变成链表结构，包含："),n("ul",null,[n("li",null,"消息在 CommitLog 中的物理偏移量，用于到 CommitLog 中查询消息"),n("li",null,[n("code",null,"IndexFile#indexKeyHashMethod(String Key)"),s("得到的整数 Hash 值")]),n("li",null,"消息保存时间与索引文件最早消息保存时间的差值，用于搜索时间范围内的消息"),n("li",null,"指向下一条消息位置的指针（在时间上是前一条，越晚到达的消息在链表越前面）")])],-1),g=c(`<h4 id="_3-1-2-存储结构" tabindex="-1"><a class="header-anchor" href="#_3-1-2-存储结构" aria-hidden="true">#</a> 3.1.2 存储结构</h4><p>索引文件底层使用 RocketMQ 的 MappedFile 来存储，索引文件可以有多个，可以无限扩展。</p><p>每个索引文件以其创建的时间命名，举例：<code>20211209174133951</code></p><p>每个索引文件被设计为定长的，最多可以保存 500万个 Hash 槽和 2000万个索引项。当保存的数据超过上限时，会创建一个新的索引文件来保存。这就意味着同样 Hash 值的消息可能会被保存到不同的索引文件当中。</p><p>RocketMQ的存储文件都遵循一种通用的数据存储格式定义实践：<strong>Header + Body</strong>，通常 <strong>Header 部分是定长</strong>的，存放一些基本信息，Body 存放数据。</p><p>具体存储结构和内容如图所示：</p><figure><img src="https://scarb-images.oss-cn-hangzhou.aliyuncs.com/img/202203152214096.png" alt="Untitled" tabindex="0" loading="lazy"><figcaption>Untitled</figcaption></figure><ul><li><strong>Header</strong> 固定大小，包含一些基本信息 <ul><li>beginTimestamp：最早的消息存储时间（消息存储到 CommitLog 的时间）</li><li>endTimestamp：最晚的消息存储时间</li><li>beginPhyoffset：存储的消息的最小物理偏移量（在 CommitLog 中的偏移量）</li><li>endPhyoffset：存储的消息的最大物理偏移量</li><li>hashSlotCount：最大可存储的 hash 槽个数</li><li>indexCount：当前已经使用的索引条目个数。注意这个值是从 1 开始的</li></ul></li><li><strong>Hash Slot</strong> 部分存储固定数量的 Message Key hash槽（500万个，该数值可以通过 Broker 配置项 <code>maxHashSlotNum</code> 来配置） <ul><li>存储的每个值是在索引文件中 索引的逻辑下标。因为索引文件的 Header 和 Hash Slot 部分长度都是固定的，每个索引的长度也是固定的，所以可以通过逻辑下标计算出索引项在索引文件中的绝对偏移量</li></ul></li><li><strong>Index Item</strong> 部分存储固定数量的索引项（2000万个，该数值可以通过 Broker 配置项 <code>maxIndexNum</code> 来配置）。每个索引项包含如下信息 <ul><li>Key Hash：消息的 Topic 和 Message Key 经过哈希得到的整数</li><li>Commit Log Offset：消息在 CommitLog 中的物理偏移量，用于到 CommitLog 中查询消息</li><li>Time Diff：从该索引文件到消息保存时间的时间差（精确到秒），用于根据时间范围查询消息</li><li>Next Index Offset：链表下一项的逻辑下标（这里的逻辑下标的含义跟 Hash Slot 中存储的逻辑下标含义相同） <ul><li>每次插入新的消息，都会从链表的头部插入。链表越往后，消息越老。因为一般来说消息队列会更关心新的消息。</li></ul></li></ul></li></ul><h3 id="_3-2-索引文件涉及到的类" tabindex="-1"><a class="header-anchor" href="#_3-2-索引文件涉及到的类" aria-hidden="true">#</a> 3.2 索引文件涉及到的类</h3><h4 id="indexservice" tabindex="-1"><a class="header-anchor" href="#indexservice" aria-hidden="true">

1#</a> IndexService</h4><p>索引服务，用于管理和控制所有索引文件。包括索引文件的加载、创建、刷盘、删除等。是索引文件操作的入口。</p><ul><li><p><code>private final ArrayList&lt;IndexFile&gt; indexFileList</code>：索引文件列表。</p></li><li><p><code>buildIndex(DispatchRequest req)</code>：根据消息分发请求构建索引。注意这里会创建 msgId 的索引和消息 Key 的索引</p><ol><li>创建或获取最新的索引文件</li><li>调用该索引文件的 <code>IndexFile#putKey</code> 方法创建索引</li></ol></li><li><p><code>queryOffset(String topic, String key, int maxNum, long begin, long end)</code>：根据topic和message key，从IndexFile中查找消息。按时间查询：查询保存时间在 begin 到 end 内的消息</p><ol><li>从后往前遍历 <code>indexFileList</code> 列表中的索引文件，查找索引对应的 message 符合时间的 IndexFile（[beginTimestamp, endTimestamp] 与 [begin, end] 有交集的索引文件）</li><li>调用符合条件的索引文件 <code>IndexFile#selectPhyOffset()</code> 方法查找所有 offset</li></ol></li><li><p><code>retryGetAndCreateIndexFile()</code>：获取最新的索引文件，如果不存在，则创建一个新的索引文件。</p><ul><li>调用 <code>getAndCreateLastIndexFile()</code> 方法创建或者获取最新的索引文件</li><li>创建新索引文件时如果创建失败，会重试创建3次</li></ul></li><li><p><code>getAndCreateLastIndexFile()</code>：获取最后一个索引文件。如果没有索引文件或者最后一个索引文件满了，那么创建一个新的文件</p><ol><li>检查索引文件列表最后一个文件是否存在以及是否满</li><li>如果不存在或者已经满了，创建新的文件，并且把上一个索引文件<strong>异步刷盘</strong></li><li>如果最后一个文件存在并且未满，直接返回该文件</li></ol></li><li><p><code>flush()</code>：将一个索引文件强制刷盘，并且刷新 checkpoint 文件中的 indexMsgTimestamp，checkpoint文件刷盘。</p><ul><li>如果开启 <code>MessageStoreConfig#messageIndexSafe</code> 配置项，那么下次 Broker 异常恢复时，会从 checkpoint 保存的 indexMsgTimestamp 即索引文件记录的强制刷盘时间来恢复。</li><li>当一个索引文件写满后创建新索引文件时调用，强制将写满的文件刷盘</li></ul><figure><img src="https://scarb-images.oss-cn-hangzhou.aliyuncs.com/img/202203152214097.png" alt="Untitled" tabindex="0" loading="lazy"><figcaption>Untitled</figcaption></figure></li></ul><h4 id="indexfile" tabindex="-1"><a class="header-anchor" href="#indexfile" aria-hidden="true">#</a> IndexFile</h4><p>索引文件，包含索引文件的存储结构和一系列操作。</p><p>底层使用内存映射文件 MappedFile 存储。</p><ul><li><code>MappedFile mappedFile</code>：底层存储实现</li><li><code>putKey(final String key, final long phyOffset, final long storeTimestamp)</code>：添加一个索引到索引文件</li><li><code>selectPhyOffset(final List&lt;Long&gt; phyOffsets, final String key, final int maxNum, final long begin, final long end, boolean lock)</code>：从该索引文件中根据 Key 查找索引对应的 offset</li></ul><h2 id="_4-源码解析" tabindex="-1"><a class="header-anchor" href="#_4-源码解析" aria-hidden="true">#</a> 4. 源码解析</h2><h3 id="_4-1-indexservice" tabindex="-1"><a class="header-anchor" href="#_4-1-indexservice" aria-hidden="true">#</a> 4.1 IndexService</h3><h4 id="_4-1-1-创建" tabindex="-1"><a class="header-anchor" href="#_4-1-1-创建" aria-hidden="true">#</a> 4.1.1 创建</h4><ul><li><strong>load：重新加载索引文件</strong></li></ul><div class="language-java line-numbers-mode" data-ext="java"><pre class="language-java"><code><span class="token doc-comment comment">/**
2 * 重新加载索引文件
3 *
4 * <span class="token keyword">@param</span> <span class="token parameter">lastExitOK</span> 上次是否是正常退出
5 * <span class="token keyword">@return</span> 加载是否成功
6 */</span>
7<span class="token keyword">public</span> <span class="token keyword">boolean</span> <span class="token function">load</span><span class="token punctuation">(</span><span class="token keyword">final</span> <span class="token keyword">boolean</span> lastExitOK<span class="token punctuation">)</span> <span class="token punctuation">{</span>
8    <span class="token class-name">File</span> dir <span class="token operator">=</span> <span class="token keyword">new</span> <span class="token class-name">File</span><span class="token punctuation">(</span><span class="token keyword">this</span><span class="token punctuation">.</span>storePath<span class="token punctuation">)</span><span class="token punctuation">;</span>
9    <span class="token class-name">File</span><span class="token punctuation">[</span><span class="token punctuation">]</span> files <span class="token operator">=</span> dir<span class="token punctuation">.</span><span class="token function">listFiles</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
10    <span class="token keyword">if</span> <span class="token punctuation">(</span>files <span class="token operator">!=</span> <span class="token keyword">null</span><span class="token punctuation">)</span> <span class="token punctuation">{</span>

11        <span class="token comment">// ascending order, 将索引文件按照创建时间升序排序</span>
12        <span class="token class-name">Arrays</span><span class="token punctuation">.</span><span class="token function">sort</span><span class="token punctuation">(</span>files<span class="token punctuation">)</span><span class="token punctuation">;</span>
13        <span class="token keyword">for</span> <span class="token punctuation">(</span><span class="token class-name">File</span> file <span class="token operator">:</span> files<span class="token punctuation">)</span> <span class="token punctuation">{</span>
14            <span class="token comment">// 依次加载每个索引文件</span>
15            <span class="token keyword">try</span> <span class="token punctuation">{</span>
16                <span class="token class-name">IndexFile</span> f <span class="token operator">=</span> <span class="token keyword">new</span> <span class="token class-name">IndexFile</span><span class="token punctuation">(</span>file<span class="token punctuation">.</span><span class="token function">getPath</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">,</span> <span class="token keyword">this</span><span class="token punctuation">.</span>hashSlotNum<span class="token punctuation">,</span> <span class="token keyword">this</span><span class="token punctuation">.</span>indexNum<span class="token punctuation">,</span> <span class="token number">0</span><span class="token punctuation">,</span> <span class="token number">0</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
17                f<span class="token punctuation">.</span><span class="token function">load</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
18                <span class="token comment">// 如果上一次是异常退出，则删除check point之后的所有索引文件</span>
19                <span class="token keyword">if</span> <span class="token punctuation">(</span><span class="token operator">!</span>lastExitOK<span class="token punctuation">)</span> <span class="token punctuation">{</span>
20                    <span class="token keyword">if</span> <span class="token punctuation">(</span>f<span class="token punctuation">.</span><span class="token function">getEndTimestamp</span><span class="token punctuation">(</span><span class="token punctuation">)</span> <span class="token operator">&gt;</span> <span class="token keyword">this</span><span class="token punctuation">.</span>defaultMessageStore<span class="token punctuation">.</span><span class="token function">getStoreCheckpoint</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">.</span><span class="token function">getIndexMsgTimestamp</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">)</span> <span class="token punctuation">{</span>
21                        f<span class="token punctuation">.</span><span class="token function">destroy</span><span class="token punctuation">(</span><span class="token number">0</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
22                        <span class="token keyword">continue</span><span class="token punctuation">;</span>
23                    <span class="token punctuation">}</span>
24                <span class="token punctuation">}</span>
25
26                <span class="token keyword">this</span><span class="token punctuation">.</span>indexFileList<span class="token punctuation">.</span><span class="token function">add</span><span class="token punctuation">(</span>f<span class="token punctuation">)</span><span class="token punctuation">;</span>
27            <span class="token punctuation">}</span> <span class="token keyword">catch</span> 
28              <span class="token comment">// ...</span>
29        <span class="token punctuation">}</span>
30    <span class="token punctuation">}</span>
31    <span class="token keyword">return</span> <span class="token boolean">true</span><span class="token punctuation">;</span>
32<span class="token punctuation">}</span>
33</code></pre><div class="line-numbers" aria-hidden="true"><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div></div></div><hr><ul><li>getAndCreateLastIndexFile()：获取最后一个索引文件，如果集合为空或者最后一个文件写满了，则新建一个文件 <ol><li>先判断文件是否存在、是否写满</li><li>如果不存在或者最后一个文件写满，则创建一个文件</li><li>如果存在，直接返回该文件</li><li>如果创建了新文件，启动一个线程将前一个写满的文件异步刷盘。 <ul><li>刷盘线程会将该文件刷盘</li><li>然后更新 <code>StoreCheckpoint#indexMsgTimestamp</code> 为该写满的索引文件中 indexHeader 的 endTimestamp</li></ul></li></ol></li></ul><div class="language-java line-numbers-mode" data-ext="java"><pre class="language-java"><code><span class="token doc-comment comment">/**
34 * 获取最后一个索引文件，如果集合为空或者最后一个文件写满了，则新建一个文件<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>br</span><span class="token punctuation">&gt;</span></span>
35 * 只有一个线程调用，所以不存在写竟争问题
36 */</span>
37<span class="token keyword">public</span> <span class="token class-name">IndexFile</span> <span class="token function">getAndCreateLastIndexFile</span><span class="token punctuation">(</span><span class="token punctuation">)</span> <span class="token punctuation">{</span>
38    <span class="token class-name">IndexFile</span> indexFile <span class="token operator">=</span> <span class="token keyword">null</span><span class="token punctuation">;</span>
39    <span class="token class-name">IndexFile</span> prevIndexFile <span class="token operator">=</span> <span class="token keyword">null</span><span class="token punctuation">;</span>
40    <span class="token keyword">long</span> lastUpdateEndPhyOffset <span class="token operator">=</span> <span class="token number">0</span><span class="token punctuation">;</span>
41    <span class="token keyword">long</span> lastUpdateIndexTimestamp <span class="token operator">=</span> <span class="token number">0</span><span class="token punctuation">;</span>
42    <span class="token comment">// 先尝试使用读锁</span>
43    <span class="token punctuation">{</span>

44        <span class="token keyword">this</span><span class="token punctuation">.</span>readWriteLock<span class="token punctuation">.</span><span class="token function">readLock</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">.</span><span class="token function">lock</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
45        <span class="token comment">// 判断文件列表是否为空</span>
46        <span class="token keyword">if</span> <span class="token punctuation">(</span><span class="token operator">!</span><span class="token keyword">this</span><span class="token punctuation">.</span>indexFileList<span class="token punctuation">.</span><span class="token function">isEmpty</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">)</span> <span class="token punctuation">{</span>
47            <span class="token class-name">IndexFile</span> tmp <span class="token operator">=</span> <span class="token keyword">this</span><span class="token punctuation">.</span>indexFileList<span class="token punctuation">.</span><span class="token function">get</span><span class="token punctuation">(</span><span class="token keyword">this</span><span class="token punctuation">.</span>indexFileList<span class="token punctuation">.</span><span class="token function">size</span><span class="token punctuation">(</span><span class="token punctuation">)</span> <span class="token operator">-</span> <span class="token number">1</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
48            <span class="token comment">// 判断最后一个文件是否写满</span>
49            <span class="token keyword">if</span> <span class="token punctuation">(</span><span class="token operator">!</span>tmp<span class="token punctuation">.</span><span class="token function">isWriteFull</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">)</span> <span class="token punctuation">{</span>
50                indexFile <span class="token operator">=</span> tmp<span class="token punctuation">;</span>
51            <span class="token punctuation">}</span> <span class="token keyword">else</span> <span class="token punctuation">{</span>
52                lastUpdateEndPhyOffset <span class="token operator">=</span> tmp<span class="token punctuation">.</span><span class="token function">getEndPhyOffset</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
53                lastUpdateIndexTimestamp <span class="token operator">=</span> tmp<span class="token punctuation">.</span><span class="token function">getEndTimestamp</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
54                prevIndexFile <span class="token operator">=</span> tmp<span class="token punctuation">;</span>
55            <span class="token punctuation">}</span>
56        <span class="token punctuation">}</span>
57
58        <span class="token keyword">this</span><span class="token punctuation">.</span>readWriteLock<span class="token punctuation">.</span><span class="token function">readLock</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">.</span><span class="token function">unlock</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
59    <span class="token punctuation">}</span>
60
61    <span class="token comment">// 如果文件列表为空或者最后一个文件写满了，使用写锁创建文件</span>
62    <span class="token keyword">if</span> <span class="token punctuation">(</span>indexFile <span class="token operator">==</span> <span class="token keyword">null</span><span class="token punctuation">)</span> <span class="token punctuation">{</span>

63        <span class="token keyword">try</span> <span class="token punctuation">{</span>
64            <span class="token class-name">String</span> fileName <span class="token operator">=</span>
65                <span class="token keyword">this</span><span class="token punctuation">.</span>storePath <span class="token operator">+</span> <span class="token class-name">File</span><span class="token punctuation">.</span>separator
66                    <span class="token operator">+</span> <span class="token class-name">UtilAll</span><span class="token punctuation">.</span><span class="token function">timeMillisToHumanString</span><span class="token punctuation">(</span><span class="token class-name">System</span><span class="token punctuation">.</span><span class="token function">currentTimeMillis</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
67            indexFile <span class="token operator">=</span>
68                <span class="token keyword">new</span> <span class="token class-name">IndexFile</span><span class="token punctuation">(</span>fileName<span class="token punctuation">,</span> <span class="token keyword">this</span><span class="token punctuation">.</span>hashSlotNum<span class="token punctuation">,</span> <span class="token keyword">this</span><span class="token punctuation">.</span>indexNum<span class="token punctuation">,</span> lastUpdateEndPhyOffset<span class="token punctuation">,</span>
69                    lastUpdateIndexTimestamp<span class="token punctuation">)</span><span class="token punctuation">;</span>
70            <span class="token keyword">this</span><span class="token punctuation">.</span>readWriteLock<span class="token punctuation">.</span><span class="token function">writeLock</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">.</span><span class="token function">lock</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
71            <span class="token keyword">this</span><span class="token punctuation">.</span>indexFileList<span class="token punctuation">.</span><span class="token function">add</span><span class="token punctuation">(</span>indexFile<span class="token punctuation">)</span><span class="token punctuation">;</span>
72        <span class="token punctuation">}</span> <span class="token keyword">catch</span> <span class="token punctuation">(</span><span class="token class-name">Exception</span> e<span class="token punctuation">)</span> <span class="token punctuation">{</span>
73            log<span class="token punctuation">.</span><span class="token function">error</span><span class="token punctuation">(</span><span class="token string">&quot;getLastIndexFile exception &quot;</span><span class="token punctuation">,</span> e<span class="token punctuation">)</span><span class="token punctuation">;</span>
74        <span class="token punctuation">}</span> <span class="token keyword">finally</span> <span class="token punctuation">{</span>
75            <span class="token keyword">this</span><span class="token punctuation">.</span>readWriteLock<span class="token punctuation">.</span><span class="token function">writeLock</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">.</span><span class="token function">unlock</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
76        <span class="token punctuation">}</span>
77
78        <span class="token comment">// 每创建一个新文件，前一个文件异步刷盘</span>
79        <span class="token keyword">if</span> <span class="token punctuation">(</span>indexFile <span class="token operator">!=</span> <span class="token keyword">null</span><span class="token punctuation">)</span> <span class="token punctuation">{</span>
80            <span class="token keyword">final</span> <span class="token class-name">IndexFile</span> flushThisFile <span class="token operator">=</span> prevIndexFile<span class="token punctuation">;</span>

81            <span class="token class-name">Thread</span> flushThread <span class="token operator">=</span> <span class="token keyword">new</span> <span class="token class-name">Thread</span><span class="token punctuation">(</span><span class="token keyword">new</span> <span class="token class-name">Runnable</span><span class="token punctuation">(</span><span class="token punctuation">)</span> <span class="token punctuation">{</span>
82                <span class="token annotation punctuation">@Override</span>
83                <span class="token keyword">public</span> <span class="token keyword">void</span> <span class="token function">run</span><span class="token punctuation">(</span><span class="token punctuation">)</span> <span class="token punctuation">{</span>
84                    <span class="token class-name">IndexService</span><span class="token punctuation">.</span><span class="token keyword">this</span><span class="token punctuation">.</span><span class="token function">flush</span><span class="token punctuation">(</span>flushThisFile<span class="token punctuation">)</span><span class="token punctuation">;</span>
85                <span class="token punctuation">}</span>
86            <span class="token punctuation">}</span><span class="token punctuation">,</span> <span class="token string">&quot;FlushIndexFileThread&quot;</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
87
88            flushThread<span class="token punctuation">.</span><span class="token function">setDaemon</span><span class="token punctuation">(</span><span class="token boolean">true</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
89            flushThread<span class="token punctuation">.</span><span class="token function">start</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
90        <span class="token punctuation">}</span>
91    <span class="token punctuation">}</span>
92
93    <span class="token keyword">return</span> indexFile<span class="token punctuation">;</span>
94<span class="token punctuation">}</span>
95</code></pre><div class="line-numbers" aria-hidden="true"><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div></div></div>

95<h4 id="_4-1-2-插入和查询" tabindex="-1"><a class="header-anchor" href="#_4-1-2-插入和查询" aria-hidden="true">#</a> 4.1.2 插入和查询</h4><ul><li><code>buildIndex(DispatchRequest req)</code>：根据消息分发请求构建索引。注意这里会创建 msgId 的索引和消息 Key 的索引 <ol><li>创建或获取最新的索引文件</li><li>调用该索引文件的 <code>IndexFile#putKey</code> 方法创建索引 <ol><li>获取 uniqKey（也就是 msgId），创建索引</li><li>获取消息的所有 key，分别创建索引</li></ol></li></ol></li></ul><div class="language-java line-numbers-mode" data-ext="java"><pre class="language-java"><code><span class="token doc-comment comment">/**
96 * 根据 DispatchRequest 构建索引
97 *
98 * <span class="token keyword">@param</span> <span class="token parameter">req</span> 消息存入CommitLog之后重新分发到Index文件的 DispatchRequest
99 */</span>
100<span class="token keyword">public</span> <span class="token keyword">void</span> <span class="token function">buildIndex</span><span class="token punctuation">(</span><span class="token class-name">DispatchRequest</span> req<span class="token punctuation">)</span> <span class="token punctuation">{</span>
101    <span class="token class-name">IndexFile</span> indexFile <span class="token operator">=</span> <span class="token function">retryGetAndCreateIndexFile</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
102    <span class="token keyword">if</span> <span class="token punctuation">(</span>indexFile <span class="token operator">!=</span> <span class="token keyword">null</span><span class="token punctuation">)</span> <span class="token punctuation">{</span>
103        <span class="token keyword">long</span> endPhyOffset <span class="token operator">=</span> indexFile<span class="token punctuation">.</span><span class="token function">getEndPhyOffset</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
104        <span class="token class-name">DispatchRequest</span> msg <span class="token operator">=</span> req<span class="token punctuation">;</span>
105        <span class="token class-name">String</span> topic <span class="token operator">=</span> msg<span class="token punctuation">.</span><span class="token function">getTopic</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
106        <span class="token class-name">String</span> keys <span class="token operator">=</span> msg<span class="token punctuation">.</span><span class="token function">getKeys</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
107        <span class="token keyword">if</span> <span class="token punctuation">(</span>msg<span class="token punctuation">.</span><span class="token function">getCommitLogOffset</span><span class="token punctuation">(</span><span class="token punctuation">)</span> <span class="token operator">&lt;</span> endPhyOffset<span class="token punctuation">)</span> <span class="token punctuation">{</span>
108            <span class="token keyword">return</span><span class="token punctuation">;</span>
109        <span class="token punctuation">}</span>
110
111        <span class="token comment">// 如果是事务消息的回滚消息，不需要创建索引，直接返回</span>
112        <span class="token keyword">final</span> <span class="token keyword">int</span> tranType <span class="token operator">=</span> <span class="token class-name">MessageSysFlag</span><span class="token punctuation">.</span><span class="token function">getTransactionValue</span><span class="token punctuation">(</span>msg<span class="token punctuation">.</span><span class="token function">getSysFlag</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
113        <span class="token keyword">switch</span> <span class="token punctuation">(</span>tranType<span class="token punctuation">)</span> <span class="token punctuation">{</span>

114            <span class="token keyword">case</span> <span class="token class-name">MessageSysFlag</span><span class="token punctuation">.</span><span class="token constant">TRANSACTION_NOT_TYPE</span><span class="token operator">:</span>
115            <span class="token keyword">case</span> <span class="token class-name">MessageSysFlag</span><span class="token punctuation">.</span><span class="token constant">TRANSACTION_PREPARED_TYPE</span><span class="token operator">:</span>
116            <span class="token keyword">case</span> <span class="token class-name">MessageSysFlag</span><span class="token punctuation">.</span><span class="token constant">TRANSACTION_COMMIT_TYPE</span><span class="token operator">:</span>
117                <span class="token keyword">break</span><span class="token punctuation">;</span>
118            <span class="token keyword">case</span> <span class="token class-name">MessageSysFlag</span><span class="token punctuation">.</span><span class="token constant">TRANSACTION_ROLLBACK_TYPE</span><span class="token operator">:</span>
119                <span class="token keyword">return</span><span class="token punctuation">;</span>
120        <span class="token punctuation">}</span>
121
122        <span class="token keyword">if</span> <span class="token punctuation">(</span>req<span class="token punctuation">.</span><span class="token function">getUniqKey</span><span class="token punctuation">(</span><span class="token punctuation">)</span> <span class="token operator">!=</span> <span class="token keyword">null</span><span class="token punctuation">)</span> <span class="token punctuation">{</span>
123            <span class="token comment">// 创建UniqueKey的索引，也就是msgId的索引</span>
124            indexFile <span class="token operator">=</span> <span class="token function">putKey</span><span class="token punctuation">(</span>indexFile<span class="token punctuation">,</span> msg<span class="token punctuation">,</span> <span class="token function">buildKey</span><span class="token punctuation">(</span>topic<span class="token punctuation">,</span> req<span class="token punctuation">.</span><span class="token function">getUniqKey</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">)</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
125            <span class="token keyword">if</span> <span class="token punctuation">(</span>indexFile <span class="token operator">==</span> <span class="token keyword">null</span><span class="token punctuation">)</span> <span class="token punctuation">{</span>
126                log<span class="token punctuation">.</span><span class="token function">error</span><span class="token punctuation">(</span><span class="token string">&quot;putKey error commitlog {} uniqkey {}&quot;</span><span class="token punctuation">,</span> req<span class="token punctuation">.</span><span class="token function">getCommitLogOffset</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">,</span> req<span class="token punctuation">.</span><span class="token function">getUniqKey</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
127                <span class="token keyword">return</span><span class="token punctuation">;</span>
128            <span class="token punctuation">}</span>
129        <span class="token punctuation">}</span>
130
131        <span class="token comment">// 创建消息key的索引，这里key可以有多个</span>
132        <span class="token keyword">if</span> <span class="token punctuation">(</span>keys <span class="token operator">!=</span> <span class="token keyword">null</span> <span class="token operator">&amp;&amp;</span> keys<span class="token punctuation">.</span><span class="token function">length</span><span class="token punctuation">(</span><span class="token punctuation">)</span> <span class="token operator">&gt;</span> <span class="token number">0</span><span class="token punctuation">)</span> <span class="token punctuation">{</span>

133            <span class="token class-name">String</span><span class="token punctuation">[</span><span class="token punctuation">]</span> keyset <span class="token operator">=</span> keys<span class="token punctuation">.</span><span class="token function">split</span><span class="token punctuation">(</span><span class="token class-name">MessageConst</span><span class="token punctuation">.</span><span class="token constant">KEY_SEPARATOR</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
134            <span class="token keyword">for</span> <span class="token punctuation">(</span><span class="token keyword">int</span> i <span class="token operator">=</span> <span class="token number">0</span><span class="token punctuation">;</span> i <span class="token operator">&lt;</span> keyset<span class="token punctuation">.</span>length<span class="token punctuation">;</span> i<span class="token operator">++</span><span class="token punctuation">)</span> <span class="token punctuation">{</span>
135                <span class="token class-name">String</span> key <span class="token operator">=</span> keyset<span class="token punctuation">[</span>i<span class="token punctuation">]</span><span class="token punctuation">;</span>
136                <span class="token keyword">if</span> <span class="token punctuation">(</span>key<span class="token punctuation">.</span><span class="token function">length</span><span class="token punctuation">(</span><span class="token punctuation">)</span> <span class="token operator">&gt;</span> <span class="token number">0</span><span class="token punctuation">)</span> <span class="token punctuation">{</span>
137                    indexFile <span class="token operator">=</span> <span class="token function">putKey</span><span class="token punctuation">(</span>indexFile<span class="token punctuation">,</span> msg<span class="token punctuation">,</span> <span class="token function">buildKey</span><span class="token punctuation">(</span>topic<span class="token punctuation">,</span> key<span class="token punctuation">)</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
138                    <span class="token keyword">if</span> <span class="token punctuation">(</span>indexFile <span class="token operator">==</span> <span class="token keyword">null</span><span class="token punctuation">)</span> <span class="token punctuation">{</span>
139                        log<span class="token punctuation">.</span><span class="token function">error</span><span class="token punctuation">(</span><span class="token string">&quot;putKey error commitlog {} uniqkey {}&quot;</span><span class="token punctuation">,</span> req<span class="token punctuation">.</span><span class="token function">getCommitLogOffset</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">,</span> req<span class="token punctuation">.</span><span class="token function">getUniqKey</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
140                        <span class="token keyword">return</span><span class="token punctuation">;</span>
141                    <span class="token punctuation">}</span>
142                <span class="token punctuation">}</span>
143            <span class="token punctuation">}</span>
144        <span class="token punctuation">}</span>
145    <span class="token punctuation">}</span> <span class="token keyword">else</span> <span class="token punctuation">{</span>
146        log<span class="token punctuation">.</span><span class="token function">error</span><span class="token punctuation">(</span><span class="token string">&quot;build index error, stop building index&quot;</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
147    <span class="token punctuation">}</span>
148<span class="token punctuation">}</span>
149</code></pre><div class="line-numbers" aria-hidden="true"><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div></div></div><ul><li><code>queryOffset(String topic, String key, int maxNum, long begin, long end)</code>：根据topic和message key，从IndexFile中查找消息</li></ul><div class="language-java line-numbers-mode" data-ext="java"><pre class="language-java"><code><span class="token doc-comment comment">/**
150     * 根据topic和message key，从IndexFile中查找消息
151     *
152     * <span class="token keyword">@param</span> <span class="token parameter">topic</span>
153     * <span class="token keyword">@param</span> <span class="token parameter">key</span>
154     * <span class="token keyword">@param</span> <span class="token parameter">maxNum</span> 最大查找消息数量
155     * <span class="token keyword">@param</span> <span class="token parameter">begin</span> 查找消息最小时间
156     * <span class="token keyword">@param</span> <span class="token parameter">end</span>

156 查找消息最大时间
157     * <span class="token keyword">@return</span>
158     */</span>
159    <span class="token keyword">public</span> <span class="token class-name">QueryOffsetResult</span> <span class="token function">queryOffset</span><span class="token punctuation">(</span><span class="token class-name">String</span> topic<span class="token punctuation">,</span> <span class="token class-name">String</span> key<span class="token punctuation">,</span> <span class="token keyword">int</span> maxNum<span class="token punctuation">,</span> <span class="token keyword">long</span> begin<span class="token punctuation">,</span> <span class="token keyword">long</span> end<span class="token punctuation">)</span> <span class="token punctuation">{</span>
160        <span class="token class-name">List</span><span class="token generics"><span class="token punctuation">&lt;</span><span class="token class-name">Long</span><span class="token punctuation">&gt;</span></span> phyOffsets <span class="token operator">=</span> <span class="token keyword">new</span> <span class="token class-name">ArrayList</span><span class="token generics"><span class="token punctuation">&lt;</span><span class="token class-name">Long</span><span class="token punctuation">&gt;</span></span><span class="token punctuation">(</span>maxNum<span class="token punctuation">)</span><span class="token punctuation">;</span>
161
162        <span class="token keyword">long</span> indexLastUpdateTimestamp <span class="token operator">=</span> <span class="token number">0</span><span class="token punctuation">;</span>
163        <span class="token keyword">long</span> indexLastUpdatePhyoffset <span class="token operator">=</span> <span class="token number">0</span><span class="token punctuation">;</span>
164        maxNum <span class="token operator">=</span> <span class="token class-name">Math</span><span class="token punctuation">.</span><span class="token function">min</span><span class="token punctuation">(</span>maxNum<span class="token punctuation">,</span> <span class="token keyword">this</span><span class="token punctuation">.</span>defaultMessageStore<span class="token punctuation">.</span><span class="token function">getMessageStoreConfig</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">.</span><span class="token function">getMaxMsgsNumBatch</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
165        <span class="token keyword">try</span> <span class="token punctuation">{</span>
166            <span class="token keyword">this</span><span class="token punctuation">.</span>readWriteLock<span class="token punctuation">.</span><span class="token function">readLock</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">.</span><span class="token function">lock</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
167            <span class="token keyword">if</span> <span class="token punctuation">(</span><span class="token operator">!</span><span class="token keyword">this</span><span class="token punctuation">.</span>indexFileList<span class="token punctuation">.</span><span class="token function">isEmpty</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">)</span> <span class="token punctuation">{</span>
168                <span class="token comment">// 从后往前遍历IndexFile，查找索引对应的message符合时间的IndexFile</span>
169                <span class="token keyword">for</span> <span class="token punctuation">(</span><span class="token keyword">int</span> i <span class="token operator">=</span> <span class="token keyword">this</span><span class="token punctuation">.</span>indexFileList<span class="token punctuation">.</span><span class="token function">size</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">;</span> i <span class="token operator">&gt;</span> <span class="token number">0</span><span class="token punctuation">;</span> i<span class="token operator">--</span><span class="token punctuation">)</span> <span class="token punctuation">{</span>

170                    <span class="token class-name">IndexFile</span> f <span class="token operator">=</span> <span class="token keyword">this</span><span class="token punctuation">.</span>indexFileList<span class="token punctuation">.</span><span class="token function">get</span><span class="token punctuation">(</span>i <span class="token operator">-</span> <span class="token number">1</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
171                    <span class="token keyword">boolean</span> lastFile <span class="token operator">=</span> i <span class="token operator">==</span> <span class="token keyword">this</span><span class="token punctuation">.</span>indexFileList<span class="token punctuation">.</span><span class="token function">size</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
172                    <span class="token keyword">if</span> <span class="token punctuation">(</span>lastFile<span class="token punctuation">)</span> <span class="token punctuation">{</span>
173                        indexLastUpdateTimestamp <span class="token operator">=</span> f<span class="token punctuation">.</span><span class="token function">getEndTimestamp</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
174                        indexLastUpdatePhyoffset <span class="token operator">=</span> f<span class="token punctuation">.</span><span class="token function">getEndPhyOffset</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
175                    <span class="token punctuation">}</span>
176
177                    <span class="token keyword">if</span> <span class="token punctuation">(</span>f<span class="token punctuation">.</span><span class="token function">isTimeMatched</span><span class="token punctuation">(</span>begin<span class="token punctuation">,</span> end<span class="token punctuation">)</span><span class="token punctuation">)</span> <span class="token punctuation">{</span>
178                        <span class="token comment">// 最后一个文件需要加锁</span>
179                        f<span class="token punctuation">.</span><span class="token function">selectPhyOffset</span><span class="token punctuation">(</span>phyOffsets<span class="token punctuation">,</span> <span class="token function">buildKey</span><span class="token punctuation">(</span>topic<span class="token punctuation">,</span> key<span class="token punctuation">)</span><span class="token punctuation">,</span> maxNum<span class="token punctuation">,</span> begin<span class="token punctuation">,</span> end<span class="token punctuation">,</span> lastFile<span class="token punctuation">)</span><span class="token punctuation">;</span>
180                    <span class="token punctuation">}</span>
181
182                    <span class="token comment">// 再往前遍历时间更不符合</span>
183                    <span class="token keyword">if</span> <span class="token punctuation">(</span>f<span class="token punctuation">.</span><span class="token function">getBeginTimestamp</span><span class="token punctuation">(</span><span class="token punctuation">)</span> <span class="token operator">&lt;</span> begin<span class="token punctuation">)</span> <span class="token punctuation">{</span>
184                        <span class="token keyword">break</span><span class="token punctuation">;</span>
185                    <span class="token punctuation">}</span>
186
187                    <span class="token keyword">if</span> <span class="token punctuation">(</span>phyOffsets<span class="token punctuation">.</span><span class="token function">size</span><span class="token punctuation">(</span><span class="token punctuation">)</span> <span class="token operator">&gt;=</span> maxNum<span class="token punctuation">)</span> <span class="token punctuation">{</span>
188                        <span class="token keyword">break</span><span class="token punctuation">;</span>

189                    <span class="token punctuation">}</span>
190                <span class="token punctuation">}</span>
191            <span class="token punctuation">}</span>
192        <span class="token punctuation">}</span> <span class="token keyword">catch</span> <span class="token punctuation">(</span><span class="token class-name">Exception</span> e<span class="token punctuation">)</span> <span class="token punctuation">{</span>
193            log<span class="token punctuation">.</span><span class="token function">error</span><span class="token punctuation">(</span><span class="token string">&quot;queryMsg exception&quot;</span><span class="token punctuation">,</span> e<span class="token punctuation">)</span><span class="token punctuation">;</span>
194        <span class="token punctuation">}</span> <span class="token keyword">finally</span> <span class="token punctuation">{</span>
195            <span class="token keyword">this</span><span class="token punctuation">.</span>readWriteLock<span class="token punctuation">.</span><span class="token function">readLock</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">.</span><span class="token function">unlock</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
196        <span class="token punctuation">}</span>
197
198        <span class="token keyword">return</span> <span class="token keyword">new</span> <span class="token class-name">QueryOffsetResult</span><span class="token punctuation">(</span>phyOffsets<span class="token punctuation">,</span> indexLastUpdateTimestamp<span class="token punctuation">,</span> indexLastUpdatePhyoffset<span class="token punctuation">)</span><span class="token punctuation">;</span>
199    <span class="token punctuation">}</span>
200</code></pre><div class="line-numbers" aria-hidden="true"><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div></div></div><h4 id="_4-1-3-过期删除" tabindex="-1"><a class="header-anchor" href="#_4-1-3-过期删除" aria-hidden="true">#</a> 4.1.3 过期删除</h4><ul><li><code>deleteExpiredFile(long offset)</code>：删除消息CommitLog偏移量offset之前的所有IndexFile文件</li></ul><h4 id="_4-1-4-刷盘" tabindex="-1"><a class="header-anchor" href="#_4-1-4-刷盘" aria-hidden="true">#</a> 4.1.4 刷盘</h4><ul><li><code>flush()</code>：强制刷盘，会把内存映射文件中的数据强制写到磁盘。在一个索引文件写满后调用</li></ul><div class="language-java line-numbers-mode" data-ext="java"><pre class="language-java"><code><span class="token doc-comment comment">/**
201 * 索引文件刷盘，在一个文件写满后调用
202 * 
203 * <span class="token keyword">@param</span> <span class="token parameter">f</span> 需要刷盘的索引文件
204 */</span>
205<span class="token keyword">public</span> <span class="token keyword">void</span> <span class="token function">flush</span><span class="token punctuation">(</span><span class="token keyword">final</span> <span class="token class-name">IndexFile</span> f<span class="token punctuation">)</span> <span class="token punctuation">{</span>
206    <span class="token keyword">if</span> <span class="token punctuation">(</span><span class="token keyword">null</span> <span class="token operator">==</span> f<span class="token punctuation">)</span>

207        <span class="token keyword">return</span><span class="token punctuation">;</span>
208
209    <span class="token keyword">long</span> indexMsgTimestamp <span class="token operator">=</span> <span class="token number">0</span><span class="token punctuation">;</span>
210
211    <span class="token keyword">if</span> <span class="token punctuation">(</span>f<span class="token punctuation">.</span><span class="token function">isWriteFull</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">)</span> <span class="token punctuation">{</span>
212        indexMsgTimestamp <span class="token operator">=</span> f<span class="token punctuation">.</span><span class="token function">getEndTimestamp</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
213    <span class="token punctuation">}</span>
214
215    <span class="token comment">// 索引文件刷盘</span>
216    f<span class="token punctuation">.</span><span class="token function">flush</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
217
218    <span class="token comment">// checkpoint文件刷盘</span>
219    <span class="token keyword">if</span> <span class="token punctuation">(</span>indexMsgTimestamp <span class="token operator">&gt;</span> <span class="token number">0</span><span class="token punctuation">)</span> <span class="token punctuation">{</span>
220        <span class="token keyword">this</span><span class="token punctuation">.</span>defaultMessageStore<span class="token punctuation">.</span><span class="token function">getStoreCheckpoint</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">.</span><span class="token function">setIndexMsgTimestamp</span><span class="token punctuation">(</span>indexMsgTimestamp<span class="token punctuation">)</span><span class="token punctuation">;</span>
221        <span class="token keyword">this</span><span class="token punctuation">.</span>defaultMessageStore<span class="token punctuation">.</span><span class="token function">getStoreCheckpoint</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">.</span><span class="token function">flush</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
222    <span class="token punctuation">}</span>
223<span class="token punctuation">}</span>
224</code></pre><div class="line-numbers" aria-hidden="true"><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div></div></div><h3 id="_4-2-indexfile" tabindex="-1"><a class="header-anchor" href="#_4-2-indexfile" aria-hidden="true">#</a> 4.2 IndexFile</h3><ul><li><code>putKey(final String key, final long phyOffset, final long storeTimestamp)</code>：向索引文件插入新的索引项</li></ul><ol><li>根据 key 的 Hash 值计算出 hash槽绝对位置 <code>absSlotPos</code></li><li>获取当前 hash槽的值，为该 hash槽对应的最新的索引的逻辑下标</li><li>在 hash槽对应的链表头部插入索引</li><li>hash槽指向最新创建的索引的逻辑下标</li><li>更新文件头</li></ol><div class="language-java line-numbers-mode" data-ext="java"><pre class="language-java"><code><span class="token doc-comment comment">/**
225 * 向索引文件插入新的索引项
226 * 如果返回false，表示需要创建新的索引文件
227 */</span>
228<span class="token keyword">public</span> <span class="token keyword">boolean</span> <span class="token function">putKey</span><span class="token punctuation">(</span><span class="token keyword">final</span> <span class="token class-name">String</span> key<span class="token punctuation">,</span> <span class="token keyword">final</span> <span class="token keyword">long</span> phyOffset<span class="token punctuation">,</span> <span class="token keyword">final</span> <span class="token keyword">long</span> storeTimestamp<span class="token punctuation">)</span> <span class="token punctuation">{</span>
229    <span class="token comment">// 判断当前索引数量是否小于最大索引数量，如果小于则直接退出，说明需要创建新的索引文件</span>
230    <span class="token keyword">if</span> <span class="token punctuation">(</span><span class="token keyword">this</span><span class="token punctuation">.</span>indexHeader<span class="token punctuation">.</span><span class="token function">getIndexCount</span><span class="token punctuation">(</span><span class="token punctuation">)</span> <span class="token operator">&lt;</span> <span class="token keyword">this</span><span class="token punctuation">.</span>indexNum<span class="token punctuation">)</span> <span class="token punctuation">{</span>

231        <span class="token comment">// 计算key的hash值</span>
232        <span class="token keyword">int</span> keyHash <span class="token operator">=</span> <span class="token function">indexKeyHashMethod</span><span class="token punctuation">(</span>key<span class="token punctuation">)</span><span class="token punctuation">;</span>
233        <span class="token comment">// 获取hash槽位置（下标）。通过 keyHash % hashSlotNum 的方式再次哈希，这里会加大查询消息错误的概率。</span>
234        <span class="token keyword">int</span> slotPos <span class="token operator">=</span> keyHash <span class="token operator">%</span> <span class="token keyword">this</span><span class="token punctuation">.</span>hashSlotNum<span class="token punctuation">;</span>
235        <span class="token comment">// 通过hash槽下表计算出hash槽的绝对位置</span>
236        <span class="token keyword">int</span> absSlotPos <span class="token operator">=</span> <span class="token class-name">IndexHeader</span><span class="token punctuation">.</span><span class="token constant">INDEX_HEADER_SIZE</span> <span class="token operator">+</span> slotPos <span class="token operator">*</span> hashSlotSize<span class="token punctuation">;</span>
237
238        <span class="token class-name">FileLock</span> fileLock <span class="token operator">=</span> <span class="token keyword">null</span><span class="token punctuation">;</span>
239
240        <span class="token keyword">try</span> <span class="token punctuation">{</span>
241
242            <span class="token comment">// fileLock = this.fileChannel.lock(absSlotPos, hashSlotSize,</span>
243            <span class="token comment">// false);</span>
244            <span class="token comment">// 通过hash槽绝对位置，获取hash槽的值，如果有值说明这个hash key已经存在，如果不存在则需要填入</span>
245            <span class="token keyword">int</span> slotValue <span class="token operator">=</span> <span class="token keyword">this</span><span class="token punctuation">.</span>mappedByteBuffer<span class="token punctuation">.</span><span class="token function">getInt</span><span class="token punctuation">(</span>absSlotPos<span class="token punctuation">)</span><span class="token punctuation">;</span>
246            <span class="token keyword">if</span> <span class="token punctuation">(</span>slotValue <span class="token operator">&lt;=</span> invalidIndex <span class="token operator">||</span> slotValue <span class="token operator">&gt;</span> <span class="token keyword">this</span><span class="token punctuation">.</span>indexHeader<span class="token punctuation">.</span><span class="token function">getIndexCount</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">)</span> <span class="token punctuation">{</span>
247                slotValue <span class="token operator">=</span> invalidIndex<span class="token punctuation">;</span>
248            <span class="token punctuation">}</span>
249
250            <span class="token keyword">long</span> timeDiff <span class="token operator">=</span> storeTimestamp <span class="token operator">-</span> <span class="token keyword">this</span><span class="token punctuation">.</span>indexHeader<span class="token punctuation">.</span><span class="token function">getBeginTimestamp</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
251
252            timeDiff <span class="token operator">=</span> timeDiff <span class="token operator">/</span> <span class="token number">1000</span><span class="token punctuation">;</span>
253
254            <span class="token keyword">if</span> <span class="token punctuation">(</span><span class="token keyword">this</span><span class="token punctuation">.</span>indexHeader<span class="token punctuation">.</span><span class="token function">getBeginTimestamp</span><span class="token punctuation">(</span><span class="token punctuation">)</span> <span class="token operator">&lt;=</span> <span class="token number">0</span><span class="token punctuation">)</span> <span class="token punctuation">{</span>
255                timeDiff <span class="token operator">=</span> <span class="token number">0</span><span class="token punctuation">;</span>

256            <span class="token punctuation">}</span> <span class="token keyword">else</span> <span class="token keyword">if</span> <span class="token punctuation">(</span>timeDiff <span class="token operator">&gt;</span> <span class="token class-name">Integer</span><span class="token punctuation">.</span><span class="token constant">MAX_VALUE</span><span class="token punctuation">)</span> <span class="token punctuation">{</span>
257                timeDiff <span class="token operator">=</span> <span class="token class-name">Integer</span><span class="token punctuation">.</span><span class="token constant">MAX_VALUE</span><span class="token punctuation">;</span>
258            <span class="token punctuation">}</span> <span class="token keyword">else</span> <span class="token keyword">if</span> <span class="token punctuation">(</span>timeDiff <span class="token operator">&lt;</span> <span class="token number">0</span><span class="token punctuation">)</span> <span class="token punctuation">{</span>
259                timeDiff <span class="token operator">=</span> <span class="token number">0</span><span class="token punctuation">;</span>
260            <span class="token punctuation">}</span>
261
262            <span class="token comment">// 计算放置索引的绝对偏移量</span>
263            <span class="token keyword">int</span> absIndexPos <span class="token operator">=</span>
264                <span class="token class-name">IndexHeader</span><span class="token punctuation">.</span><span class="token constant">INDEX_HEADER_SIZE</span> <span class="token operator">+</span> <span class="token keyword">this</span><span class="token punctuation">.</span>hashSlotNum <span class="token operator">*</span> hashSlotSize
265                    <span class="token operator">+</span> <span class="token keyword">this</span><span class="token punctuation">.</span>indexHeader<span class="token punctuation">.</span><span class="token function">getIndexCount</span><span class="token punctuation">(</span><span class="token punctuation">)</span> <span class="token operator">*</span> indexSize<span class="token punctuation">;</span>
266
267            <span class="token comment">// 在链表头部插入最新的索引项</span>
268            <span class="token comment">// 将索引存入文件，最后一个是指针，指向下一个链表元素</span>
269            <span class="token keyword">this</span><span class="token punctuation">.</span>mappedByteBuffer<span class="token punctuation">.</span><span class="token function">putInt</span><span class="token punctuation">(</span>absIndexPos<span class="token punctuation">,</span> keyHash<span class="token punctuation">)</span><span class="token punctuation">;</span>
270            <span class="token keyword">this</span><span class="token punctuation">.</span>mappedByteBuffer<span class="token punctuation">.</span><span class="token function">putLong</span><span class="token punctuation">(</span>absIndexPos <span class="token operator">+</span> <span class="token number">4</span><span class="token punctuation">,</span> phyOffset<span class="token punctuation">)</span><span class="token punctuation">;</span>
271            <span class="token keyword">this</span><span class="token punctuation">.</span>mappedByteBuffer<span class="token punctuation">.</span><span class="token function">putInt</span><span class="token punctuation">(</span>absIndexPos <span class="token operator">+</span> <span class="token number">4</span> <span class="token operator">+</span> <span class="token number">8</span><span class="token punctuation">,</span> <span class="token punctuation">(</span><span class="token keyword">int</span><span class="token punctuation">)</span> timeDiff<span class="token punctuation">)</span><span class="token punctuation">;</span>
272            <span class="token keyword">this</span><span class="token punctuation">.</span>mappedByteBuffer<span class="token punctuation">.</span><span class="token function">putInt</span><span class="token punctuation">(</span>absIndexPos <span class="token operator">+</span> <span class="token number">4</span> <span class="token operator">+</span> <span class="token number">8</span> <span class="token operator">+</span> <span class="token number">4</span><span class="token punctuation">,</span> slotValue<span class="token punctuation">)</span><span class="token punctuation">;</span>
273

274            <span class="token comment">// 写入hash槽，每个hash槽的值是最新写入的索引文件的逻辑下标</span>
275            <span class="token keyword">this</span><span class="token punctuation">.</span>mappedByteBuffer<span class="token punctuation">.</span><span class="token function">putInt</span><span class="token punctuation">(</span>absSlotPos<span class="token punctuation">,</span> <span class="token keyword">this</span><span class="token punctuation">.</span>indexHeader<span class="token punctuation">.</span><span class="token function">getIndexCount</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
276
277            <span class="token keyword">if</span> <span class="token punctuation">(</span><span class="token keyword">this</span><span class="token punctuation">.</span>indexHeader<span class="token punctuation">.</span><span class="token function">getIndexCount</span><span class="token punctuation">(</span><span class="token punctuation">)</span> <span class="token operator">&lt;=</span> <span class="token number">1</span><span class="token punctuation">)</span> <span class="token punctuation">{</span>
278                <span class="token keyword">this</span><span class="token punctuation">.</span>indexHeader<span class="token punctuation">.</span><span class="token function">setBeginPhyOffset</span><span class="token punctuation">(</span>phyOffset<span class="token punctuation">)</span><span class="token punctuation">;</span>
279                <span class="token keyword">this</span><span class="token punctuation">.</span>indexHeader<span class="token punctuation">.</span><span class="token function">setBeginTimestamp</span><span class="token punctuation">(</span>storeTimestamp<span class="token punctuation">)</span><span class="token punctuation">;</span>
280            <span class="token punctuation">}</span>
281
282            <span class="token keyword">if</span> <span class="token punctuation">(</span>invalidIndex <span class="token operator">==</span> slotValue<span class="token punctuation">)</span> <span class="token punctuation">{</span>
283                <span class="token keyword">this</span><span class="token punctuation">.</span>indexHeader<span class="token punctuation">.</span><span class="token function">incHashSlotCount</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
284            <span class="token punctuation">}</span>
285            <span class="token comment">// 更新索引文件头，索引项个数+1</span>
286            <span class="token keyword">this</span><span class="token punctuation">.</span>indexHeader<span class="token punctuation">.</span><span class="token function">incIndexCount</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
287            <span class="token keyword">this</span><span class="token punctuation">.</span>indexHeader<span class="token punctuation">.</span><span class="token function">setEndPhyOffset</span><span class="token punctuation">(</span>phyOffset<span class="token punctuation">)</span><span class="token punctuation">;</span>
288            <span class="token keyword">this</span><span class="token punctuation">.</span>indexHeader<span class="token punctuation">.</span><span class="token function">setEndTimestamp</span><span class="token punctuation">(</span>storeTimestamp<span class="token punctuation">)</span><span class="token punctuation">;</span>
289
290            <span class="token keyword">return</span> <span class="token boolean">true</span><span class="token punctuation">;</span>
291        <span class="token punctuation">}</span> <span class="token keyword">catch</span> <span class="token punctuation">(</span><span class="token class-name">Exception</span> e<span class="token punctuation">)</span> <span class="token punctuation">{</span>
292            log<span class="token punctuation">.</span><span class="token function">error</span><span class="token punctuation">(</span><span class="token string">&quot;putKey exception, Key: &quot;</span> <span class="token operator">+</span> key <span class="token operator">+</span> <span class="token string">&quot; KeyHashCode: &quot;</span> <span class="token operator">+</span> key<span class="token punctuation">.</span><span class="token function">hashCode</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">,</span>

292 e<span class="token punctuation">)</span><span class="token punctuation">;</span>
293        <span class="token punctuation">}</span> <span class="token keyword">finally</span> <span class="token punctuation">{</span>
294            <span class="token keyword">if</span> <span class="token punctuation">(</span>fileLock <span class="token operator">!=</span> <span class="token keyword">null</span><span class="token punctuation">)</span> <span class="token punctuation">{</span>
295                <span class="token keyword">try</span> <span class="token punctuation">{</span>
296                    fileLock<span class="token punctuation">.</span><span class="token function">release</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
297                <span class="token punctuation">}</span> <span class="token keyword">catch</span> <span class="token punctuation">(</span><span class="token class-name">IOException</span> e<span class="token punctuation">)</span> <span class="token punctuation">{</span>
298                    log<span class="token punctuation">.</span><span class="token function">error</span><span class="token punctuation">(</span><span class="token string">&quot;Failed to release the lock&quot;</span><span class="token punctuation">,</span> e<span class="token punctuation">)</span><span class="token punctuation">;</span>
299                <span class="token punctuation">}</span>
300            <span class="token punctuation">}</span>
301        <span class="token punctuation">}</span>
302    <span class="token punctuation">}</span> <span class="token keyword">else</span> <span class="token punctuation">{</span>
303        log<span class="token punctuation">.</span><span class="token function">warn</span><span class="token punctuation">(</span><span class="token string">&quot;Over index file capacity: index count = &quot;</span> <span class="token operator">+</span> <span class="token keyword">this</span><span class="token punctuation">.</span>indexHeader<span class="token punctuation">.</span><span class="token function">getIndexCount</span><span class="token punctuation">(</span><span class="token punctuation">)</span>
304            <span class="token operator">+</span> <span class="token string">&quot;; index max num = &quot;</span> <span class="token operator">+</span> <span class="token keyword">this</span><span class="token punctuation">.</span>indexNum<span class="token punctuation">)</span><span class="token punctuation">;</span>
305    <span class="token punctuation">}</span>
306
307    <span class="token keyword">return</span> <span class="token boolean">false</span><span class="token punctuation">;</span>
308<span class="token punctuation">}</span>
309</code></pre><div class="line-numbers" aria-hidden="true"><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div></div></div><hr><ul><li><code>selectPhyOffset(final List&lt;Long&gt;

309 phyOffsets, final String key, final int maxNum, final long begin, final long end, boolean lock)</code>：从该索引文件中根据 Key 查找索引对应的 offset <ol><li>根据 key 的 Hash值计算 hash槽的绝对位置</li><li>通过 hash槽中存储的索引逻辑下标，找到索引链表绝对位置</li><li>遍历索引链表中的每个索引，获取索引数据，比较时间信息</li><li>将时间信息符合搜索条件的索引加入到结果列表中</li></ol></li></ul><div class="language-java line-numbers-mode" data-ext="java"><pre class="language-java"><code><span class="token doc-comment comment">/**
310 * 从该索引文件中根据key查找offsets
311 *
312 * <span class="token keyword">@param</span> <span class="token parameter">phyOffsets</span> offsets结果列表
313 * <span class="token keyword">@param</span> <span class="token parameter">key</span> 查找的key
314 * <span class="token keyword">@param</span> <span class="token parameter">maxNum</span> 最大返回结果数量
315 * <span class="token keyword">@param</span> <span class="token parameter">begin</span> 查找消息的开始时间
316 * <span class="token keyword">@param</span> <span class="token parameter">end</span> 查找消息的结束时间
317 * <span class="token keyword">@param</span> <span class="token parameter">lock</span> 查找时是否加锁（已废弃）
318 */</span>
319<span class="token keyword">public</span> <span class="token keyword">void</span> <span class="token function">selectPhyOffset</span><span class="token punctuation">(</span><span class="token keyword">final</span> <span class="token class-name">List</span><span class="token generics"><span class="token punctuation">&lt;</span><span class="token class-name">Long</span><span class="token punctuation">&gt;</span></span> phyOffsets<span class="token punctuation">,</span> <span class="token keyword">final</span> <span class="token class-name">String</span> key<span class="token punctuation">,</span> <span class="token keyword">final</span> <span class="token keyword">int</span> maxNum<span class="token punctuation">,</span>
320    <span class="token keyword">final</span> <span class="token keyword">long</span> begin<span class="token punctuation">,</span> <span class="token keyword">final</span> <span class="token keyword">long</span> end<span class="token punctuation">,</span> <span class="token keyword">boolean</span> lock<span class="token punctuation">)</span> <span class="token punctuation">{</span>
321    <span class="token keyword">if</span> <span class="token punctuation">(</span><span class="token keyword">this</span><span class="token punctuation">.</span>mappedFile<span class="token punctuation">.</span><span class="token function">hold</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">)</span> <span class="token punctuation">{</span>
322        <span class="token comment">// 根据key的hash值计算hash槽的绝对位置</span>
323        <span class="token keyword">int</span> keyHash <span class="token operator">=</span> <span class="token function">indexKeyHashMethod</span><span class="token punctuation">(</span>key<span class="token punctuation">)</span><span class="token punctuation">;</span>
324        <span class="token keyword">int</span> slotPos <span class="token operator">=</span> keyHash <span class="token operator">%</span> <span class="token keyword">this</span><span class="token punctuation">.</span>hashSlotNum<span class="token punctuation">;</span>
325        <span class="token keyword">int</span> absSlotPos <span class="token operator">=</span> <span class="token class-name">IndexHeader</span><span class="token punctuation">.</span><span class="token constant">INDEX_HEADER_SIZE</span> <span class="token operator">+</span> slotPos <span class="token operator">*</span> hashSlotSize<span class="token punctuation">;</span>
326
327        <span class="token class-name">FileLock</span> fileLock <span class="token operator">=</span> <span class="token keyword">null</span><span class="token punctuation">;</span>

328        <span class="token keyword">try</span> <span class="token punctuation">{</span>
329            <span class="token keyword">if</span> <span class="token punctuation">(</span>lock<span class="token punctuation">)</span> <span class="token punctuation">{</span>
330                <span class="token comment">// fileLock = this.fileChannel.lock(absSlotPos,</span>
331                <span class="token comment">// hashSlotSize, true);</span>
332            <span class="token punctuation">}</span>
333
334            <span class="token comment">// 获取hash槽的值</span>
335            <span class="token keyword">int</span> slotValue <span class="token operator">=</span> <span class="token keyword">this</span><span class="token punctuation">.</span>mappedByteBuffer<span class="token punctuation">.</span><span class="token function">getInt</span><span class="token punctuation">(</span>absSlotPos<span class="token punctuation">)</span><span class="token punctuation">;</span>
336            <span class="token comment">// if (fileLock != null) {</span>
337            <span class="token comment">// fileLock.release();</span>
338            <span class="token comment">// fileLock = null;</span>
339            <span class="token comment">// }</span>
340            <span class="token comment">// 如果该hash槽的值有效则查找，否则查找失败</span>
341            <span class="token keyword">if</span> <span class="token punctuation">(</span>slotValue <span class="token operator">&lt;=</span> invalidIndex <span class="token operator">||</span> slotValue <span class="token operator">&gt;</span> <span class="token keyword">this</span><span class="token punctuation">.</span>indexHeader<span class="token punctuation">.</span><span class="token function">getIndexCount</span><span class="token punctuation">(</span><span class="token punctuation">)</span>
342                <span class="token operator">||</span> <span class="token keyword">this</span><span class="token punctuation">.</span>indexHeader<span class="token punctuation">.</span><span class="token function">getIndexCount</span><span class="token punctuation">(</span><span class="token punctuation">)</span> <span class="token operator">&lt;=</span> <span class="token number">1</span><span class="token punctuation">)</span> <span class="token punctuation">{</span>
343            <span class="token punctuation">}</span> <span class="token keyword">else</span> <span class="token punctuation">{</span>
344                <span class="token keyword">for</span> <span class="token punctuation">(</span><span class="token keyword">int</span> nextIndexToRead <span class="token operator">=</span> slotValue<span class="token punctuation">;</span> <span class="token punctuation">;</span> <span class="token punctuation">)</span> <span class="token punctuation">{</span>
345                    <span class="token keyword">if</span> <span class="token punctuation">(</span>phyOffsets<span class="token punctuation">.</span><span class="token function">size</span><span class="token punctuation">(</span><span class="token punctuation">)</span> <span class="token operator">&gt;=</span> maxNum<span class="token punctuation">)</span> <span class="token punctuation">{</span>
346                        <span class="token keyword">break</span><span class="token punctuation">;</span>
347                    <span class="token punctuation">}</span>
348
349                    <span class="token keyword">int</span> absIndexPos <span class="token operator">=</span>
350                        <span class="token class-name">IndexHeader</span><span class="token punctuation">.</span><span class="token constant">INDEX_HEADER_SIZE</span> <span class="token operator">+</span> <span class="token keyword">this</span><span class="token punctuation">.</span>hashSlotNum <span class="token operator">*</span> hashSlotSize
351                            <span class="token operator">+</span> nextIndexToRead <span class="token operator">*</span> indexSize<span class="token punctuation">;</span>
352
353                    <span class="token keyword">int</span> keyHashRead <span class="token operator">=</span> <span class="token keyword">this</span><span class="token punctuation">.</span>mappedByteBuffer<span class="token punctuation">.</span><span class="token function">getInt</span><span class="token punctuation">(</span>absIndexPos<span class="token punctuation">)</span><span class="token punctuation">;</span>

354                    <span class="token keyword">long</span> phyOffsetRead <span class="token operator">=</span> <span class="token keyword">this</span><span class="token punctuation">.</span>mappedByteBuffer<span class="token punctuation">.</span><span class="token function">getLong</span><span class="token punctuation">(</span>absIndexPos <span class="token operator">+</span> <span class="token number">4</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
355
356                    <span class="token keyword">long</span> timeDiff <span class="token operator">=</span> <span class="token punctuation">(</span><span class="token keyword">long</span><span class="token punctuation">)</span> <span class="token keyword">this</span><span class="token punctuation">.</span>mappedByteBuffer<span class="token punctuation">.</span><span class="token function">getInt</span><span class="token punctuation">(</span>absIndexPos <span class="token operator">+</span> <span class="token number">4</span> <span class="token operator">+</span> <span class="token number">8</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
357                    <span class="token keyword">int</span> prevIndexRead <span class="token operator">=</span> <span class="token keyword">this</span><span class="token punctuation">.</span>mappedByteBuffer<span class="token punctuation">.</span><span class="token function">getInt</span><span class="token punctuation">(</span>absIndexPos <span class="token operator">+</span> <span class="token number">4</span> <span class="token operator">+</span> <span class="token number">8</span> <span class="token operator">+</span> <span class="token number">4</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
358
359                    <span class="token keyword">if</span> <span class="token punctuation">(</span>timeDiff <span class="token operator">&lt;</span> <span class="token number">0</span><span class="token punctuation">)</span> <span class="token punctuation">{</span>
360                        <span class="token keyword">break</span><span class="token punctuation">;</span>
361                    <span class="token punctuation">}</span>
362
363                    timeDiff <span class="token operator">*=</span> <span class="token number">1000L</span><span class="token punctuation">;</span>
364
365                    <span class="token keyword">long</span> timeRead <span class="token operator">=</span> <span class="token keyword">this</span><span class="token punctuation">.</span>indexHeader<span class="token punctuation">.</span><span class="token function">getBeginTimestamp</span><span class="token punctuation">(</span><span class="token punctuation">)</span> <span class="token operator">+</span> timeDiff<span class="token punctuation">;</span>
366                    <span class="token keyword">boolean</span> timeMatched <span class="token operator">=</span> <span class="token punctuation">(</span>timeRead <span class="token operator">&gt;=</span> begin<span class="token punctuation">)</span> <span class="token operator">&amp;&amp;</span> <span class="token punctuation">(</span>timeRead <span class="token operator">&lt;=</span> end<span class="token punctuation">)</span><span class="token punctuation">;</span>
367
368                    <span class="token keyword">if</span> <span class="token punctuation">(</span>keyHash <span class="token operator">==</span> keyHashRead <span class="token operator">&amp;&amp;</span> timeMatched<span class="token punctuation">)</span> <span class="token punctuation">{</span>
369                        phyOffsets<span class="token punctuation">.</span><span class="token function">add</span><span class="token punctuation">(</span>phyOffsetRead<span class="token punctuation">)</span><span class="token punctuation">;</span>
370                    <span class="token punctuation">}</span>
371
372                    <span class="token keyword">if</span> <span class="token punctuation">(</span>prevIndexRead <span class="token operator">&lt;=</span> invalidIndex

373                        <span class="token operator">||</span> prevIndexRead <span class="token operator">&gt;</span> <span class="token keyword">this</span><span class="token punctuation">.</span>indexHeader<span class="token punctuation">.</span><span class="token function">getIndexCount</span><span class="token punctuation">(</span><span class="token punctuation">)</span>
374                        <span class="token operator">||</span> prevIndexRead <span class="token operator">==</span> nextIndexToRead <span class="token operator">||</span> timeRead <span class="token operator">&lt;</span> begin<span class="token punctuation">)</span> <span class="token punctuation">{</span>
375                        <span class="token keyword">break</span><span class="token punctuation">;</span>
376                    <span class="token punctuation">}</span>
377
378                    nextIndexToRead <span class="token operator">=</span> prevIndexRead<span class="token punctuation">;</span>
379                <span class="token punctuation">}</span>
380            <span class="token punctuation">}</span>
381        <span class="token punctuation">}</span> <span class="token keyword">catch</span> <span class="token punctuation">(</span><span class="token class-name">Exception</span> e<span class="token punctuation">)</span> <span class="token punctuation">{</span>
382            log<span class="token punctuation">.</span><span class="token function">error</span><span class="token punctuation">(</span><span class="token string">&quot;selectPhyOffset exception &quot;</span><span class="token punctuation">,</span> e<span class="token punctuation">)</span><span class="token punctuation">;</span>
383        <span class="token punctuation">}</span> <span class="token keyword">finally</span> <span class="token punctuation">{</span>
384            <span class="token keyword">if</span> <span class="token punctuation">(</span>fileLock <span class="token operator">!=</span> <span class="token keyword">null</span><span class="token punctuation">)</span> <span class="token punctuation">{</span>
385                <span class="token keyword">try</span> <span class="token punctuation">{</span>
386                    fileLock<span class="token punctuation">.</span><span class="token function">release</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
387                <span class="token punctuation">}</span> <span class="token keyword">catch</span> <span class="token punctuation">(</span><span class="token class-name">IOException</span> e<span class="token punctuation">)</span> <span class="token punctuation">{</span>
388                    log<span class="token punctuation">.</span><span class="token function">error</span><span class="token punctuation">(</span><span class="token string">&quot;Failed to release the lock&quot;</span><span class="token punctuation">,</span> e<span class="token punctuation">)</span><span class="token punctuation">;</span>
389                <span class="token punctuation">}</span>
390            <span class="token punctuation">}</span>
391
392            <span class="token keyword">this</span><span class="token punctuation">.</span>mappedFile<span class="token punctuation">.</span><span class="token function">release</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
393        <span class="token punctuation">}</span>
394    <span class="token punctuation">}</span>
395</code></pre><div class="line-numbers" aria-hidden="true"><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div></div></div><h2 id="参考资料" tabindex="-1"><a class="header-anchor" href="#参考资料" aria-hidden="true">

395#</a> 参考资料</h2>`,42),x={href:"https://github.com/apache/rocketmq/blob/master/docs/cn/design.md",target:"_blank",rel:"noopener noreferrer"},w={href:"https://blog.csdn.net/meilong_whpu/article/details/76921583",target:"_blank",rel:"noopener noreferrer"},I=n("hr",null,null,-1),_=n("p",null,"欢迎关注公众号【消息中间件】（middleware-mq），更新消息中间件的源码解析和最新动态！",-1),F=n("figure",null,[n("img",{src:"https://scarb-images.oss-cn-hangzhou.aliyuncs.com/img/202205170102971.jpg",alt:"",tabindex:"0",loading:"lazy"}),n("figcaption")],-1);function S(L,T){const e=o("ExternalLinkIcon"),t=o("router-link");return i(),u("div",null,[n("p",null,[s("原文地址："),n("a",d,[s("http://hscarb.github.io/rocketmq/20220301-rocketmq-indexfile.html"),a(e)])]),r,n("nav",m,[n("ul",null,[n("li",null,[a(t,{to:"#_1-概述"},{default:p(()=>[s("1. 概述")]),_:1}),n("ul",null,[n("li",null,[a(t,{to:"#_1-1-索引文件是什么"},{default:p(()=>[s("1.1 索引文件是什么")]),_:1})]),n("li",null,[a(t,{to:"#_1-2-索引文件作用"},{default:p(()=>[s("1.2 索引文件作用")]),_:1})])])]),n("li",null,[a(t,{to:"#_2-概要设计"},{default:p(()=>[s("2. 概要设计")]),_:1}),n("ul",null,[n("li",null,[a(t,{to:"#_2-1-索引文件结构"},{default:p(()=>[s("2.1 索引文件结构")]),_:1})]),n("li",null,[a(t,{to:"#_2-2-如何构建"},{default:p(()=>[s("2.2 如何构建")]),_:1})]),n("li",null,[a(t,{to:"#_2-3-如何查询消息"},{default:p(()=>[s("2.3 如何查询消息")]),_:1})]),n("li",null,[a(t,{to:"#_2-4-刷盘机制"},{default:p(()=>[s("2.4 刷盘机制")]),_:1})])])]),n("li",null,[a(t,{to:"#_3-详细设计"},{default:p(()=>[s("3. 详细设计")]),_:1}),n("ul",null,[n("li",null,[a(t,{to:"#_3-1-索引文件结构"},{default:p(()=>[s("3.1 索引文件结构")]),_:1})]),n("li",null,[a(t,{to:"#_3-2-索引文件涉及到的类"},{default:p(()=>[s("3.2 索引文件涉及到的类")]),_:1})])])]),n("li",null,[a(t,{to:"#_4-源码解析"},{default:p(()=>[s("4. 源码解析")]),_:1}),n("ul",null,[n("li",null,[a(t,{to:"#_4-1-indexservice"},{default:p(()=>[s("4.1 IndexService")]),_:1})]),n("li",null,[a(t,{to:"#_4-2-indexfile"},{default:p(()=>[s("4.2 IndexFile")]),_:1})])])]),n("li",null,[a(t,{to:"#参考资料"},{default:p(()=>[s("参考资料")]),_:1})])])]),v,n("ul",null,[n("li",null,[b,n("blockquote",null,[n("p",null,[s("注意：这里存在 Hash 冲突的可能，两个 Topic 和 Key 不同的消息可能得到相同的 Hash 值，会导致查询结果错误。社区已经提出这个错误 "),n("a",f,[s("ISSUE#3613"),a(e)]),s("，但目前还并未解决。")]),h])]),y]),g,n("ul",null,[n("li",null,[n("a",x,[s("RocketMQ 文档：设计"),a(e)])]),n("li",null,[n("a",w,[s("RocketMQ存储篇——IndexFile和IndexService"),a(e)])])]),I,_,F])}const M=l(k,[["render",S],["__file","20220301-rocketmq-indexfile.html.vue"]]);export{M as default};