1import{_ as s,c as a,e,o as t}from"./app-CjUkjRdW.js";const p="/assets/psi-tree-BfzXRRWs.png",l="/assets/psi-error-C3vs59-e.png",i={};function o(c,n){return t(),a("div",null,n[0]||(n[0]=[e(`<h1 id="parser-implementation" tabindex="-1"><a class="header-anchor" href="#parser-implementation"><span>Parser Implementation</span></a></h1><p>In this section, we will implement a simple parser by hand to parse the following Nim statement:</p><div class="language-nim line-numbers-mode" data-highlighter="prismjs" data-ext="nim" data-title="nim"><pre><code><span class="line">echo <span class="token string">"Hello, World!"</span></span> 2<span class="line"></span></code></pre><div class="line-numbers" aria-hidden="true" style="counter-reset:line-number 0;"><div class="line-number"></div></div></div><p>The result of the parsing will be:</p><ul><li>an AST representing the structure of the statement, and</li><li>an equivalent PSI tree that the IDE recognizes and can work with.</li></ul><p>We will rely on the built-in PSI viewer to inspect the PSI tree generated by our parser.</p><h2 id="token-element-and-psi-types" tabindex="-1"><a class="header-anchor" href="#token-element-and-psi-types"><span>Token, Element, and PSI Types</span></a></h2><p>Before we start parsing, we need to define the token and element types we will be using. Let's start with the token types:</p><div class="language-kotlin line-numbers-mode" data-highlighter="prismjs" data-ext="kt" data-title="kt"><pre><code><span class="line"><span class="token comment">// src/main/kotlin/khaledh/nimjet/NimTokenType.kt</span></span> 3<span class="line"><span class="token operator">..</span><span class="token punctuation">.</span></span> 4<span class="line"></span> 5<span class="line"><span class="token keyword">class</span> <span class="token function">NimTokenType</span><span class="token punctuation">(</span>debugName<span class="token operator">:</span> String<span class="token punctuation">)</span> <span class="token operator">:</span> <span class="token function">IElementType</span><span class="token punctuation">(</span>debugName<span class="token punctuation">,</span> NimLanguage<span class="token punctuation">)</span></span> 6<span class="line"></span> 7<span class="line"><span class="token keyword">interface</span> NimToken <span class="token punctuation">{</span></span> 8<span class="line"> <span class="token keyword">companion</span> <span class="token keyword">object</span> <span class="token punctuation">{</span></span> 9<span class="line"> <span class="token annotation builtin">@JvmField</span> <span class="token keyword">val</span> IDENTIFIER <span class="token operator">=</span> <span class="token function">NimTokenType</span><span class="token punctuation">(</span><span class="token string-literal singleline"><span class="token string">"IDENTIFIER"</span></span><span class="token punctuation">)</span></span> 10<span class="line"> <span class="token annotation builtin">@JvmField</span> <span class="token keyword">val</span> STRING_LIT <span class="token operator">=</span> <span class="token function">NimTokenType</span><span class="token punctuation">(</span><span class="token string-literal singleline"><span class="token string">"STRING_LIT"</span></span><span class="token punctuation">)</span></span> 11<span class="line"> <span class="token punctuation">}</span></span> 12<span class="line"><span class="token punctuation">}</span></span> 13<span class="line"></span></code></pre><div class="line-numbers" aria-hidden="true" style="counter-reset:line-number 0;"><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div></div></div><p>We define two token types: <code>IDENTIFIER</code> and <code>STRING_LIT</code>, which are instances of <code>NimTokenType</code> (which extends <code>IElementType</code>). We will use <code>IDENTIFIER</code> to represent the <code>echo</code> proc name, and <code>STRING_LIT</code> to represent the string literal <code>"Hello, World!"</code>.</p><p style="font-size:80%;"><b>Note:</b> The <code>@JvmField</code> annotation is used to expose the companion object fields as static fields so that we can refer to them as <code>NimToken.IDENTIFIER</code> instead of <code>NimToken.Companion.IDENTIFIER</code>. </p><p>Next, we define the element types that will be used to represent the nodes in the AST:</p><div class="language-kotlin line-numbers-mode" data-highlighter="prismjs" data-ext="kt" data-title="kt"><pre><code><span class="line"><span class="token comment">// src/main/kotlin/khaledh/nimjet/NimElementType.kt</span></span> 14<span class="line"><span class="token operator">..</span><span class="token punctuation">.</span></span> 15<span class="line"></span> 16<span class="line"><span class="token keyword">class</span> <span class="token function">NimElementType</span><span class="token punctuation">(</span>debugName<span class="token operator">:</span> String<span class="token punctuation">)</span> <span class="token operator">:</span> <span class="token function">IElementType</span><span class="token punctuation">(</span>debugName<span class="token punctuation">,</span> NimLanguage<span class="token punctuation">)</span></span> 17<span class="line"></span> 18<span class="line"><span class="token keyword">interface</span> NimElement <span class="token punctuation">{</span></span> 19<span class="line"> <span class="token keyword">companion</span> <span class="token keyword">object</span> <span class="token punctuation">{</span></span> 20<span class="line">
20 <span class="token annotation builtin">@JvmField</span> <span class="token keyword">val</span> FILE <span class="token operator">=</span> <span class="token function">IFileElementType</span><span class="token punctuation">(</span>NimLanguage<span class="token punctuation">)</span></span> 21<span class="line"> <span class="token annotation builtin">@JvmField</span> <span class="token keyword">val</span> STMT <span class="token operator">=</span> <span class="token function">NimElementType</span><span class="token punctuation">(</span><span class="token string-literal singleline"><span class="token string">"STMT"</span></span><span class="token punctuation">)</span></span> 22<span class="line"> <span class="token punctuation">}</span></span> 23<span class="line"><span class="token punctuation">}</span></span> 24<span class="line"></span></code></pre><div class="line-numbers" aria-hidden="true" style="counter-reset:line-number 0;"><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div></div></div><p>We have two element types: <code>FILE</code>, which is an instance of <code>IFileElementType</code> (the root element type), and <code>STMT</code>, which is an instance of <code>NimElementType</code>. We will use the <code>STMT</code> element to represent the whole statement.</p><p>Finally, we define the PSI element types that will be used to represent the nodes in the PSI tree. We'll use a separate class/file for each PSI element, since we'll be adding more functionality to them later. First, let's define the root PSI element, <code>NimFile</code>:</p><div class="language-kotlin line-numbers-mode" data-highlighter="prismjs" data-ext="kt" data-title="kt"><pre><code><span class="line"><span class="token comment">// src/main/kotlin/khaledh/nimjet/psi/NimFile.kt</span></span> 25<span class="line"><span class="token operator">..</span><span class="token punctuation">.</span></span> 26<span class="line"></span> 27<span class="line"><span class="token keyword">class</span> <span class="token function">NimFile</span><span class="token punctuation">(</span>viewProvider<span class="token operator">:</span> FileViewProvider<span class="token punctuation">)</span> <span class="token operator">:</span> <span class="token function">PsiFileBase</span><span class="token punctuation">(</span>viewProvider<span class="token punctuation">,</span> NimLanguage<span class="token punctuation">)</span> <span class="token punctuation">{</span></span> 28<span class="line"> <span class="token keyword">override</span> <span class="token keyword">fun</span> <span class="token function">getFileType</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token operator">:</span> FileType <span class="token operator">=</span> NimFileType</span> 29<span class="line"> <span class="token keyword">override</span> <span class="token keyword">fun</span> <span class="token function">toString</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token operator">:</span> String <span class="token operator">=</span> <span class="token string-literal singleline"><span class="token string">"NimFile"</span></span></span> 30<span class="line"><span class="token punctuation">}</span></span> 31<span class="line"></span></code></pre><div class="line-numbers" aria-hidden="true" style="counter-reset:line-number 0;"><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div></div></div><p>The <code>NimFile</code> class extends <code>PsiFileBase</code>, which is a base class that provides a default implementation for the <code>PsiFile</code> interface methods. We override <code>getFileType</code> to return our <code>NimFileType</code> instance, and <code>toString</code> to return a string representation of the PSI file node.</p><p>Next, we define the PSI element for the statement, <code>NimStmt</code>:</p><div class="language-kotlin line-numbers-mode" data-highlighter="prismjs" data-ext="kt" data-title="kt"><pre><code><span class="line"><span class="token comment">// src/main/kotlin/khaledh/nimjet/psi/NimStmt.kt</span></span> 32<span class="line"><span class="token operator">..</span><span class="token punctuation">.</span></span> 33<span class="line"></span> 34<span class="line"><span class="token keyword">class</span> <span class="token function">
34NimStmt</span><span class="token punctuation">(</span>node<span class="token operator">:</span> ASTNode<span class="token punctuation">)</span> <span class="token operator">:</span> <span class="token function">ASTWrapperPsiElement</span><span class="token punctuation">(</span>node<span class="token punctuation">)</span></span> 35<span class="line"></span></code></pre><div class="line-numbers" aria-hidden="true" style="counter-reset:line-number 0;"><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div></div></div><p>All PSI elements will extend <code>ASTWrapperPsiElement</code>, which is a base class that wraps an <code>ASTNode</code> and provides a default implementation for the <code>PsiElement</code> interface methods.</p><p>This completes the definition of the token, element, and PSI types we will be using in our simple parser.</p><h2 id="lexer" tabindex="-1"><a class="header-anchor" href="#lexer"><span>Lexer</span></a></h2><p>Let's update our JFlex lexer spec to recognize the two types of tokens we introduced:</p><div class="language-java line-numbers-mode" data-highlighter="prismjs" data-ext="java" data-title="java"><pre><code><span class="line"><span class="token comment">// src/main/kotlin/khaledh/nimjet/Nim.flex</span></span> 36<span class="line"><span class="token punctuation">.</span><span class="token punctuation">.</span><span class="token punctuation">.</span></span> 37<span class="line"><span class="token keyword">import</span> <span class="token import"><span class="token namespace">khaledh<span class="token punctuation">.</span>nimjet<span class="token punctuation">.</span></span><span class="token class-name">NimTokenType</span></span><span class="token punctuation">;</span></span> 38<span class="line"></span> 39<span class="line"><span class="token operator">%</span><span class="token operator">%</span></span> 40<span class="line"><span class="token punctuation">.</span><span class="token punctuation">.</span><span class="token punctuation">.</span></span> 41<span class="line"></span> 42<span class="line"><span class="token operator">%</span><span class="token operator">%</span></span> 43<span class="line"></span> 44<span class="line"><span class="token generics"><span class="token punctuation"><</span>YYINITIAL<span class="token punctuation">></span></span> <span class="token punctuation">{</span></span> 45<span class="line"> <span class="token punctuation">[</span>a<span class="token operator">-</span>zA<span class="token operator">-</span><span class="token class-name">Z</span><span class="token punctuation">]</span><span class="token punctuation">[</span>a<span class="token operator">-</span>zA<span class="token operator">-</span><span class="token constant">Z0</span><span class="token operator">-</span><span class="token number">9</span><span class="token punctuation">]</span><span class="token operator">*</span> <span class="token punctuation">{</span> <span class="token keyword">return</span> <span class="token class-name">NimToken</span><span class="token punctuation">.</span><span class="token constant">IDENTIFIER</span><span class="token punctuation">;</span> <span class="token punctuation">}</span></span> 46<span class="line"> \\"<span class="token punctuation">(</span><span class="token punctuation">[</span><span class="token operator">^</span>\\"\\\\<span class="token punctuation">]</span><span class="token operator">|</span>\\\\<span class="token punctuation">.</span><span class="token punctuation">)</span><span class="token operator">*</span>\\" <span class="token punctuation">{</span> <span class="token keyword">return</span> <span class="token class-name">NimToken</span><span class="token punctuation">.</span><span class="token constant">STRING_LIT</span><span class="token punctuation">;</span> <span class="token punctuation">}</span></span> 47<span class="line"> <span class="token punctuation">[</span> \\t\\n\\r<span class="token punctuation">]</span><span class="token operator">+</span>
47 <span class="token punctuation">{</span> <span class="token keyword">return</span> <span class="token class-name">TokenType</span><span class="token punctuation">.</span><span class="token constant">WHITE_SPACE</span><span class="token punctuation">;</span> <span class="token punctuation">}</span></span> 48<span class="line"></span> 49<span class="line"> <span class="token comment">// error fallback</span></span> 50<span class="line"> <span class="token punctuation">[</span><span class="token operator">^</span><span class="token punctuation">]</span> <span class="token punctuation">{</span> <span class="token keyword">return</span> <span class="token class-name">TokenType</span><span class="token punctuation">.</span><span class="token constant">BAD_CHARACTER</span><span class="token punctuation">;</span> <span class="token punctuation">}</span></span> 51<span class="line"><span class="token punctuation">}</span></span> 52<span class="line"></span></code></pre><div class="line-numbers" aria-hidden="true" style="counter-reset:line-number 0;"><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div></div></div><p>We added two rules to match the two token types we defined:</p><ul><li><code>IDENTIFIER</code>: matches any sequence of letters and digits starting with a letter</li><li><code>STRING_LIT</code>: matches a string literal enclosed in double quotes (with support for escape sequences)</li></ul><h2 id="parser" tabindex="-1"><a class="header-anchor" href="#parser"><span>Parser</span></a></h2><p>Now, let's implement the parser. We will create a <code>NimParser</code> class that extends <code>PsiParser</code>, and drives a <code>PsiBuilder</code> to create the AST by marking regions of tokens as nodes in the tree. The parsing approach we will use is called <em>recursive descent</em>, where each grammar rule is implemented as a method in the parser class. Let's start with a simple implementation without error handling to focus on the parsing logic:</p><div class="language-kotlin line-numbers-mode" data-highlighter="prismjs" data-ext="kt" data-title="kt"><pre><code><span class="line"><span class="token comment">// src/main/kotlin/khaledh/nimjet/NimParser.kt</span></span> 53<span class="line"><span class="token operator">..</span><span class="token punctuation">.</span></span> 54<span class="line"></span> 55<span class="line"><span class="token keyword">class</span> NimParser <span class="token operator">:</span> PsiParser <span class="token punctuation">{</span></span> 56<span class="line"></span> 57<span class="line"> <span class="token comment">// Grammar:</span></span> 58<span class="line"> <span class="token comment">// file ::= stmt</span></span> 59<span class="line"> <span class="token comment">// stmt ::= IDENTIFIER STRING_LIT</span></span> 60<span class="line"></span> 61<span class="line"> <span class="token keyword">override</span> <span class="token keyword">fun</span> <span class="token function">parse</span><span class="token punctuation">(</span>root<span class="token operator">:</span> IElementType<span class="token punctuation">,</span> builder<span class="token operator">:</span> PsiBuilder<span class="token punctuation">)</span><span class="token operator">:</span> ASTNode <span class="token punctuation">{</span></span> 62<span class="line"> <span class="token function">parseFile</span><span class="token punctuation">(</span>root<span class="token punctuation">,</span> builder<span class="token punctuation">)</span></span> 63<span class="line"> <span class="token keyword">return</span> builder<span class="token punctuation">.</span>treeBuilt</span> 64<span class="line"> <span class="token punctuation">}</span></span> 65<span class="line"></span> 66<span class="line"> <span class="token keyword">private</span> <span class="token keyword">fun</span> <span class="token function">parseFile</span><span class="token punctuation">(</span>root<span class="token operator">:</span> IElementType<span class="token punctuation">,</span> builder<span class="token operator">:</span> PsiBuilder<span class="token punctuation">)</span> <span class="token punctuation">{</span></span> 67<span class="line">
67 <span class="token keyword">var</span> marker <span class="token operator">=</span> builder<span class="token punctuation">.</span><span class="token function">mark</span><span class="token punctuation">(</span><span class="token punctuation">)</span></span> 68<span class="line"></span> 69<span class="line"> <span class="token function">parseStmt</span><span class="token punctuation">(</span>builder<span class="token punctuation">)</span></span> 70<span class="line"></span> 71<span class="line"> marker<span class="token punctuation">.</span><span class="token function">done</span><span class="token punctuation">(</span>NimElement<span class="token punctuation">.</span>FILE<span class="token punctuation">)</span></span> 72<span class="line"> <span class="token punctuation">}</span></span> 73<span class="line"></span> 74<span class="line"> <span class="token keyword">private</span> <span class="token keyword">fun</span> <span class="token function">parseStmt</span><span class="token punctuation">(</span>builder<span class="token operator">:</span> PsiBuilder<span class="token punctuation">)</span> <span class="token punctuation">{</span></span> 75<span class="line"> <span class="token keyword">var</span> marker <span class="token operator">=</span> builder<span class="token punctuation">.</span><span class="token function">mark</span><span class="token punctuation">(</span><span class="token punctuation">)</span></span> 76<span class="line"></span> 77<span class="line"> <span class="token function">assert</span><span class="token punctuation">(</span>builder<span class="token punctuation">.</span>tokenType <span class="token operator">==</span> NimToken<span class="token punctuation">.</span>IDENTIFIER<span class="token punctuation">)</span></span> 78<span class="line"> builder<span class="token punctuation">.</span><span class="token function">advanceLexer</span><span class="token punctuation">(</span><span class="token punctuation">)</span></span> 79<span class="line"></span> 80<span class="line"> <span class="token function">assert</span><span class="token punctuation">(</span>builder<span class="token punctuation">.</span>tokenType <span class="token operator">==</span> NimToken<span class="token punctuation">.</span>STRING_LIT<span class="token punctuation">)</span></span> 81<span class="line"> builder<span class="token punctuation">.</span><span class="token function">advanceLexer</span><span class="token punctuation">(</span><span class="token punctuation">)</span></span> 82<span class="line"></span> 83<span class="line"> marker<span class="token punctuation">.</span><span class="token function">done</span><span class="token punctuation">(</span>NimElement<span class="token punctuation">.</span>STMT<span class="token punctuation">)</span></span> 84<span class="line"> <span class="token punctuation">}</span></span> 85<span class="line"><span class="token punctuation">}</span></span> 86<span class="line"></span></code></pre><div class="line-numbers" aria-hidden="true" style="counter-reset:line-number 0;"><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div></div></div><p>The parser is very simple: it defines two grammar rules:</p><ul><li><code>file ::= stmt</code>: a file consists of a single statement</li><li><code>stmt ::= IDENTIFIER STRING_LIT</code>: a statement consists of an identifier followed by a string literal</li></ul><p>The recursive descent parsing is done by starting with the <code>parseFile</code> method, which represents the <code>file</code> rule. Since the rule consists of a single statement, we <em>descend</em> into the <code>stmt</code> rule by calling <code>parseStmt</code>. The <code>parseStmt</code> method parses its rule by asserting that there is an <code>IDENTIFIER</code> token followed by a <code>STRING_LIT</code> token (advancing the lexer after each token). A proper implementation would include error handling and reporting, but they're intentionally omitted here for simplicity.</p><p>To tell the <code>PsiBuilder</code>
86 that a region of tokens should be converted into a node in the AST, we create a <em>marker</em> by calling <code>builder.mark()</code> at the beginning of the region, and then call <code>done</code> on the marker at the end of the region to convert it into a node. You'll see this being done in the <code>parseFile</code> and <code>parseStmt</code> methods, where a marker is set at the beginning of the rule, and then marked as <em>done</em> when the rule is successfully parsed. The <code>done</code> method takes the element type to be associated with the node, which is <code>NimElement.FILE</code> (the root element) for the file node, and <code>NimElement.STMT</code> for the statement node.</p><p>Needless to say, this parser will crash if the input doesn't exactly match the grammar rules. We'll improve it later to handle errors gracefully.</p><h2 id="parser-definition" tabindex="-1"><a class="header-anchor" href="#parser-definition"><span>Parser Definition</span></a></h2><p>Let's now create the <code>NimParserDefinition</code> class, which we will register as an extension to the <code>com.intellij.lang.parserDefinition</code> extension point. This class provides the IDE with the necessary methods to create the lexer, parser, and a few other things:</p><div class="language-kotlin line-numbers-mode" data-highlighter="prismjs" data-ext="kt" data-title="kt"><pre><code><span class="line"><span class="token comment">// src/main/kotlin/khaledh/nimjet/NimParserDefinition.kt</span></span> 87<span class="line"><span class="token operator">..</span><span class="token punctuation">.</span></span> 88<span class="line"></span> 89<span class="line"><span class="token keyword">class</span> NimParserDefinition <span class="token operator">:</span> ParserDefinition <span class="token punctuation">{</span></span> 90<span class="line"> <span class="token keyword">override</span> <span class="token keyword">fun</span> <span class="token function">createLexer</span><span class="token punctuation">(</span>project<span class="token operator">:</span> Project<span class="token operator">?</span><span class="token punctuation">)</span><span class="token operator">:</span> Lexer <span class="token operator">=</span> <span class="token function">NimLexerAdapter</span><span class="token punctuation">(</span><span class="token punctuation">)</span></span> 91<span class="line"> <span class="token keyword">override</span> <span class="token keyword">fun</span> <span class="token function">createParser</span><span class="token punctuation">(</span>project<span class="token operator">:</span> Project<span class="token operator">?</span><span class="token punctuation">)</span><span class="token operator">:</span> PsiParser <span class="token operator">=</span> <span class="token function">NimParser</span><span class="token punctuation">(</span><span class="token punctuation">)</span></span> 92<span class="line"></span> 93<span class="line"> <span class="token comment">// AST</span></span> 94<span class="line"> <span class="token keyword">override</span> <span class="token keyword">fun</span> <span class="token function">getFileNodeType</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token operator">:</span> IFileElementType <span class="token operator">=</span> NimElement<span class="token punctuation">.</span>FILE</span> 95<span class="line"> <span class="token keyword">override</span> <span class="token keyword">fun</span> <span class="token function">getWhitespaceTokens</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token operator">:</span> TokenSet <span class="token operator">=</span> TokenSet<span class="token punctuation">.</span>WHITE_SPACE</span> 96<span class="line"> <span class="token keyword">override</span> <span class="token keyword">fun</span> <span class="token function">getCommentTokens</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token operator">:</span> TokenSet <span class="token operator">=</span> TokenSet<span class="token punctuation">.</span>EMPTY</span> 97<span class="line"> <span class="token keyword">
97override</span> <span class="token keyword">fun</span> <span class="token function">getStringLiteralElements</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token operator">:</span> TokenSet <span class="token operator">=</span> TokenSet<span class="token punctuation">.</span><span class="token function">create</span><span class="token punctuation">(</span>NimToken<span class="token punctuation">.</span>STRING_LIT<span class="token punctuation">)</span></span> 98<span class="line"></span> 99<span class="line"> <span class="token comment">// PSI</span></span> 100<span class="line"> <span class="token keyword">override</span> <span class="token keyword">fun</span> <span class="token function">createFile</span><span class="token punctuation">(</span>viewProvider<span class="token operator">:</span> FileViewProvider<span class="token punctuation">)</span><span class="token operator">:</span> PsiFile <span class="token operator">=</span> <span class="token function">NimFile</span><span class="token punctuation">(</span>viewProvider<span class="token punctuation">)</span></span> 101<span class="line"> <span class="token keyword">override</span> <span class="token keyword">fun</span> <span class="token function">createElement</span><span class="token punctuation">(</span>node<span class="token operator">:</span> ASTNode<span class="token punctuation">)</span><span class="token operator">:</span> PsiElement <span class="token punctuation">{</span></span> 102<span class="line"> <span class="token keyword">return</span> <span class="token keyword">when</span> <span class="token punctuation">(</span>node<span class="token punctuation">.</span>elementType<span class="token punctuation">)</span> <span class="token punctuation">{</span></span> 103<span class="line"> STMT <span class="token operator">-></span> <span class="token function">NimStmt</span><span class="token punctuation">(</span>node<span class="token punctuation">)</span></span> 104<span class="line"> <span class="token keyword">else</span> <span class="token operator">-></span> <span class="token keyword">throw</span> <span class="token function">AssertionError</span><span class="token punctuation">(</span><span class="token string-literal singleline"><span class="token string">"Unknown element type: </span><span class="token interpolation"><span class="token interpolation-punctuation punctuation">\${</span><span class="token expression">node<span class="token punctuation">.</span>elementType</span><span class="token interpolation-punctuation punctuation">}</span></span><span class="token string">"</span></span><span class="token punctuation">)</span></span> 105<span class="line"> <span class="token punctuation">}</span></span> 106<span class="line"> <span class="token punctuation">}</span></span> 107<span class="line"><span class="token punctuation">}</span></span> 108<span class="line"></span></code></pre><div class="line-numbers" aria-hidden="true" style="counter-reset:line-number 0;"><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div></div></div><p>Most of the methods should be self-explanatory. The last method, <code>createElement</code>, is responsible for creating the appropriate PSI element for a given AST node. In our case, we only have one type of element, <code>STMT</code>, so we create a <code>NimStmt</code> instance for that element type, and throw an error if we encounter an unknown element type (for now).</p><p>Finally, we need to register the <code>NimParserDefinition</code> class in the <code>plugin.xml</code> file:</p><div class="language-xml line-numbers-mode" data-highlighter="prismjs" data-ext="xml" data-title="xml"><pre><code><span class="line"><span class="token comment"><!-- src/main/resources/META-INF/plugin.xml --></span></span> 109<span class="line"></span> 110<span class="line"><span class="token tag"><span class="token tag"><span class="token punctuation"><</span>idea-plugin</span><span class="token punctuation">></span></span></span> 111<span class="line"> ...</span> 112<span class="line"></span> 113<span class="line"> <span class="token tag"><span class="token tag"><span class="token punctuation"><</span>extensions</span> <span class="token attr-name">defaultExtensionNs</span><span class="token attr-value"><span class="token punctuation attr-equals">=</span><span class="token punctuation">"</span>com.intellij<span class="token punctuation">"</span></span><span class="token punctuation">></span></span></span> 114<span class="line"> ...</span> 115<span class="line"></span> 116<span class="line">
116 <span class="token tag"><span class="token tag"><span class="token punctuation"><</span>lang.parserDefinition</span></span> 117<span class="line"> <span class="token attr-name">language</span><span class="token attr-value"><span class="token punctuation attr-equals">=</span><span class="token punctuation">"</span>Nim<span class="token punctuation">"</span></span></span> 118<span class="line"> <span class="token attr-name">implementationClass</span><span class="token attr-value"><span class="token punctuation attr-equals">=</span><span class="token punctuation">"</span>khaledh.nimjet.NimParserDefinition<span class="token punctuation">"</span></span><span class="token punctuation">/></span></span></span> 119<span class="line"></span> 120<span class="line"> <span class="token tag"><span class="token tag"><span class="token punctuation"></</span>extensions</span><span class="token punctuation">></span></span></span> 121<span class="line"></span> 122<span class="line"><span class="token tag"><span class="token tag"><span class="token punctuation"></</span>idea-plugin</span><span class="token punctuation">></span></span></span> 123<span class="line"></span></code></pre><div class="line-numbers" aria-hidden="true" style="counter-reset:line-number 0;"><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div></div></div><p>This completes the implementation of the parser. We can now test it by running the plugin and inspecting the PSI tree using the built-in PSI viewer.</p><h2 id="testing-the-parser" tabindex="-1"><a class="header-anchor" href="#testing-the-parser"><span>Testing the Parser</span></a></h2><p>Let's run the <code>Run Plugin</code> gradle task to start the IDE with our plugin. Once the IDE starts, create a new file with the following content:</p><div class="language-nim line-numbers-mode" data-highlighter="prismjs" data-ext="nim" data-title="nim"><pre><code><span class="line">echo <span class="token string">"hello, world!"</span></span> 124<span class="line"></span></code></pre><div class="line-numbers" aria-hidden="true" style="counter-reset:line-number 0;"><div class="line-number"></div></div></div><p>Now, open the PSI viewer from the <strong>Tools | View PSI Structure of Current File</strong> menu.</p><p><img src="`+p+`" alt="PSI Tree"></p><p>Great! We can see the PSI tree with the <code>NimFile</code> and <code>NimStmt</code> nodes, in addition to the <code>IDENTIFIER</code> and <code>STRING_LIT</code> tokens under the <code>NimStmt</code> node. Our simple parser is working as expected.</p><h2 id="error-handling" tabindex="-1"><a class="header-anchor" href="#error-handling"><span>Error Handling</span></a></h2><p>Let's add error handling to our parser to make it more robust. We'll change the parser to detect and recover from errors, and report them to the user. We'll make each parsing method return a boolean indicating whether the rule was successfully parsed. If a rule fails, we'll mark the region as <em>error</em> and return early. Since returning early means that not all tokens are consumed, we need to advance the lexer to the end of the token stream to ensure that the PSI tree is properly built (this is a requirement of the <code>PsiBuilder</code>).</p><p>Here's the updated parser:</p><div class="language-kotlin line-numbers-mode" data-highlighter="prismjs" data-ext="kt" data-title="kt"><pre><code><span class="line"><span class="token comment">// src/main/kotlin/khaledh/nimjet/NimParser.kt</span></span> 125<span class="line"><span class="token operator">..</span><span class="token punctuation">.</span></span> 126<span class="line"></span> 127<span class="line"><span class="token keyword">class</span> NimParser <span class="token operator">:</span> PsiParser <span class="token punctuation">{</span></span> 128<span class="line"></span> 129<span class="line"> <span class="token comment">// Grammar:</span></span> 130<span class="line"> <span class="token comment">
130// file ::= stmt</span></span> 131<span class="line"> <span class="token comment">// stmt ::= IDENTIFIER STRING_LIT</span></span> 132<span class="line"></span> 133<span class="line"> <span class="token keyword">override</span> <span class="token keyword">fun</span> <span class="token function">parse</span><span class="token punctuation">(</span>root<span class="token operator">:</span> IElementType<span class="token punctuation">,</span> builder<span class="token operator">:</span> PsiBuilder<span class="token punctuation">)</span><span class="token operator">:</span> ASTNode <span class="token punctuation">{</span></span> 134<span class="line"> <span class="token function">parseFile</span><span class="token punctuation">(</span>root<span class="token punctuation">,</span> builder<span class="token punctuation">)</span></span> 135<span class="line"> <span class="token keyword">return</span> builder<span class="token punctuation">.</span>treeBuilt</span> 136<span class="line"> <span class="token punctuation">}</span></span> 137<span class="line"></span> 138<span class="line"> <span class="token keyword">private</span> <span class="token keyword">fun</span> <span class="token function">parseFile</span><span class="token punctuation">(</span>root<span class="token operator">:</span> IElementType<span class="token punctuation">,</span> builder<span class="token operator">:</span> PsiBuilder<span class="token punctuation">)</span> <span class="token punctuation">{</span></span> 139<span class="line"> <span class="token keyword">var</span> marker <span class="token operator">=</span> builder<span class="token punctuation">.</span><span class="token function">mark</span><span class="token punctuation">(</span><span class="token punctuation">)</span></span> 140<span class="line"></span> 141<span class="line"> <span class="token keyword">if</span> <span class="token punctuation">(</span><span class="token operator">!</span>builder<span class="token punctuation">.</span><span class="token function">eof</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">)</span> <span class="token punctuation">{</span></span> 142<span class="line"> <span class="token function">parseStmt</span><span class="token punctuation">(</span>builder<span class="token punctuation">)</span></span> 143<span class="line"> <span class="token punctuation">}</span></span> 144<span class="line"></span> 145<span class="line"> <span class="token comment">// consume any remaining tokens (in case of error)</span></span> 146<span class="line"> <span class="token keyword">while</span> <span class="token punctuation">(</span><span class="token operator">!</span>builder<span class="token punctuation">.</span><span class="token function">eof</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">)</span> <span class="token punctuation">{</span></span> 147<span class="line"> builder<span class="token punctuation">.</span><span class="token function">advanceLexer</span><span class="token punctuation">(</span><span class="token punctuation">)</span></span> 148<span class="line"> <span class="token punctuation">}</span></span> 149<span class="line"></span> 150<span class="line"> marker<span class="token punctuation">.</span><span class="token function">done</span><span class="token punctuation">(</span>NimElement<span class="token punctuation">.</span>FILE<span class="token punctuation">)</span></span> 151<span class="line"> <span class="token punctuation">}</span></span> 152<span class="line"></span> 153<span class="line"> <span class="token keyword">private</span> <span class="token keyword">fun</span> <span class="token function">parseStmt</span><span class="token punctuation">(</span>builder<span class="token operator">:</span> PsiBuilder<span class="token punctuation">)</span><span class="token operator">:</span> Boolean <span class="token punctuation">{</span></span> 154<span class="line"> <span class="token keyword">var</span> marker <span class="token operator">=</span> builder<span class="token punctuation">.</span><span class="token function">mark</span><span class="token punctuation">(</span><span class="token punctuation">)</span></span> 155<span class="line"></span> 156<span class="line">
156 <span class="token keyword">if</span> <span class="token punctuation">(</span>builder<span class="token punctuation">.</span>tokenType <span class="token operator">!=</span> NimToken<span class="token punctuation">.</span>IDENTIFIER<span class="token punctuation">)</span> <span class="token punctuation">{</span></span> 157<span class="line"> <span class="token function">reportError</span><span class="token punctuation">(</span>builder<span class="token punctuation">,</span> <span class="token string-literal singleline"><span class="token string">"Expecting an identifier"</span></span><span class="token punctuation">)</span></span> 158<span class="line"> marker<span class="token punctuation">.</span><span class="token function">drop</span><span class="token punctuation">(</span><span class="token punctuation">)</span></span> 159<span class="line"> <span class="token keyword">return</span> <span class="token boolean">false</span></span> 160<span class="line"> <span class="token punctuation">}</span></span> 161<span class="line"> builder<span class="token punctuation">.</span><span class="token function">advanceLexer</span><span class="token punctuation">(</span><span class="token punctuation">)</span></span> 162<span class="line"></span> 163<span class="line"> <span class="token keyword">if</span> <span class="token punctuation">(</span>builder<span class="token punctuation">.</span>tokenType <span class="token operator">!=</span> NimToken<span class="token punctuation">.</span>STRING_LIT<span class="token punctuation">)</span> <span class="token punctuation">{</span></span> 164<span class="line"> <span class="token function">reportError</span><span class="token punctuation">(</span>builder<span class="token punctuation">,</span> <span class="token string-literal singleline"><span class="token string">"Expecting a string literal"</span></span><span class="token punctuation">)</span></span> 165<span class="line"> marker<span class="token punctuation">.</span><span class="token function">drop</span><span class="token punctuation">(</span><span class="token punctuation">)</span></span> 166<span class="line"> <span class="token keyword">return</span> <span class="token boolean">false</span></span> 167<span class="line"> <span class="token punctuation">}</span></span> 168<span class="line"> builder<span class="token punctuation">.</span><span class="token function">advanceLexer</span><span class="token punctuation">(</span><span class="token punctuation">)</span></span> 169<span class="line"></span> 170<span class="line"> marker<span class="token punctuation">.</span><span class="token function">done</span><span class="token punctuation">(</span>NimElement<span class="token punctuation">.</span>STMT<span class="token punctuation">)</span></span> 171<span class="line"> <span class="token keyword">return</span> <span class="token boolean">true</span></span> 172<span class="line"> <span class="token punctuation">}</span></span> 173<span class="line"></span> 174<span class="line"> <span class="token keyword">private</span> <span class="token keyword">fun</span> <span class="token function">reportError</span><span class="token punctuation">(</span>builder<span class="token operator">:</span> PsiBuilder<span class="token punctuation">,</span> message<span class="token operator">:</span> String<span class="token punctuation">)</span> <span class="token punctuation">{</span></span> 175<span class="line"> <span class="token keyword">var</span> marker <span class="token operator">=</span> builder<span class="token punctuation">.</span><span class="token function">mark</span><span class="token punctuation">(</span><span class="token punctuation">)</span></span> 176<span class="line"> builder<span class="token punctuation">.</span><span class="token function">advanceLexer</span><span class="token punctuation">(</span><span class="token punctuation">)</span></span> 177<span class="line"> marker<span class="token punctuation">.</span><span class="token function">error</span><span class="token punctuation">(</span>message<span class="token punctuation">)</span></span> 178<span class="line"> <span class="token punctuation">}</span></span> 179<span class="line"><span class="token punctuation">}</span></span> 180<span class="line"></span></code></pre><div class="line-numbers" aria-hidden="true" style="counter-reset:line-number 0;"><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div><div class="line-number"></div></div></div><p>We added a check for the end of the token stream in the <code>parseFile</code> method to ensure that we don't try to parse an empty file. The <code>reportError</code> method is used to mark the region as an error and report the error message to the user. If a rule fails, we drop the marker and return <code>false</code> to indicate that the rule was not successfully parsed.</p><p>Let's run the plugin again and test the error handling by creating a file with the contents <code>echo hello</code> (missing the string quotation marks).</p><p style="text-align:center;">
180<img src="`+l+'" alt="PSI Error Reporting" width="550"></p><p>And it works! The token <code>hello</code> is marked as an error with a red squiggly line, and our error message is displayed when we hover over the token.</p><p>OK, we now have a good understanding of how to implement a parser by hand. In the next section, we will look at how to generate a parser using Grammar-Kit by defining the grammar in a BNF-like format. This will allow us to focus on the language grammar rules without worrying about the details of the lexer and parser implementation.</p>',56)]))}const u=s(i,[["render",o],["__file","06-parser-p2.html.vue"]]),d=JSON.parse('{"path":"/nimjet/06-parser-p2.html","title":"Parser Implementation","lang":"en-US","frontmatter":{},"headers":[{"level":2,"title":"Token, Element, and PSI Types","slug":"token-element-and-psi-types","link":"#token-element-and-psi-types","children":[]},{"level":2,"title":"Lexer","slug":"lexer","link":"#lexer","children":[]},{"level":2,"title":"Parser","slug":"parser","link":"#parser","children":[]},{"level":2,"title":"Parser Definition","slug":"parser-definition","link":"#parser-definition","children":[]},{"level":2,"title":"Testing the Parser","slug":"testing-the-parser","link":"#testing-the-parser","children":[]},{"level":2,"title":"Error Handling","slug":"error-handling","link":"#error-handling","children":[]}],"git":{"updatedTime":1725855037000},"filePathRelative":"nimjet/06-parser-p2.md","excerpt":"\\n<p>In this section, we will implement a simple parser by hand to parse the following Nim\\nstatement:</p>\\n<div class=\\"language-nim line-numbers-mode\\" data-highlighter=\\"prismjs\\" data-ext=\\"nim\\" data-title=\\"nim\\"><pre><code><span class=\\"line\\">echo <span class=\\"token string\\">\\"Hello, World!\\"</span></span>\\n<span class=\\"line\\"></span></code></pre>\\n<div class=\\"line-numbers\\" aria-hidden=\\"true\\" style=\\"counter-reset:line-number 0\\"><div class=\\"line-number\\"></div></div></div>"}');export{u as comp,d as data};
Line numbers count LF bytes from the start of the resource, as the search results do. Vendor segments are library code the classifier recognised; they are stored but not indexed. Bytes are shown as Latin1 characters, one per byte.