<!DOCTYPE html>
<html>
  <head>
  </head>
  <body>
    <dl style="line-height: 1.5; margin-left: 2em">
      <dt style="clear: both; float: left; font-weight: bold; width: 8em">Author</dt>
      <dd style="margin-left: 8.5em">Kouhei Sutou &lt;kou****@clear*****&gt;</dd>
      <dt style="clear: both; float: left; font-weight: bold; width: 8em">Date</dt>
      <dd style="margin-left: 8.5em">2014-06-05 23:34:56 +0900 (Thu, 05 Jun 2014)</dd>
      <dt style="clear: both; float: left; font-weight: bold; width: 8em">New Revision</dt>
      <dd style="margin-left: 8.5em"><a href="https://github.com/groonga/groonga/commit/7d11c2f6960f244ed0ed5c0408dd80beed668fa2">7d11c2f6960f244ed0ed5c0408dd80beed668fa2</a></dd>
      <dt style="clear: both; float: left; font-weight: bold; width: 8em">Message</dt>
      <dd style="margin-left: 8.5em"><pre style="border: 1px solid #aaa; font-family: Consolas, Menlo, &quot;Liberation Mono&quot;, Courier, monospace; line-height: 1.2; padding: 0.5em; width: auto">tokenize: add valid tokenizer check</pre></dd>
      <dt style="clear: both; float: left; font-weight: bold; width: 8em">Added files</dt>
      <dd style="margin-left: 8.5em">
        <ul>
          <li><a href="https://github.com/groonga/groonga/commit/7d11c2f6960f244ed0ed5c0408dd80beed668fa2#diff-1">test/command/suite/tokenize/invalid/tokenizer/invalid.expected</a></li>
          <li><a href="https://github.com/groonga/groonga/commit/7d11c2f6960f244ed0ed5c0408dd80beed668fa2#diff-2">test/command/suite/tokenize/invalid/tokenizer/invalid.test</a></li>
          <li><a href="https://github.com/groonga/groonga/commit/7d11c2f6960f244ed0ed5c0408dd80beed668fa2#diff-3">test/command/suite/tokenize/invalid/tokenizer/nonexistent.expected</a></li>
          <li><a href="https://github.com/groonga/groonga/commit/7d11c2f6960f244ed0ed5c0408dd80beed668fa2#diff-4">test/command/suite/tokenize/invalid/tokenizer/nonexistent.test</a></li>
        </ul>
      </dd>
      <dt style="clear: both; float: left; font-weight: bold; width: 8em">Removed files</dt>
      <dd style="margin-left: 8.5em">
        <ul>
          <li><a href="https://github.com/groonga/groonga/commit/7d11c2f6960f244ed0ed5c0408dd80beed668fa2#diff-5">test/command/suite/tokenize/invalid/tokenizer/unknown.expected</a></li>
          <li><a href="https://github.com/groonga/groonga/commit/7d11c2f6960f244ed0ed5c0408dd80beed668fa2#diff-6">test/command/suite/tokenize/invalid/tokenizer/unknown.test</a></li>
        </ul>
      </dd>
      <dt style="clear: both; float: left; font-weight: bold; width: 8em">Modified files</dt>
      <dd style="margin-left: 8.5em">
        <ul>
          <li><a href="https://github.com/groonga/groonga/commit/7d11c2f6960f244ed0ed5c0408dd80beed668fa2#diff-0">lib/proc.c</a></li>
        </ul>
      </dd>
    </dl>

    <div class="diff-section" style="clear: both">
      <table style="border-collapse: collapse; border: 1px solid #aaa">
        <thead>
          <tr class="diff-header" style="border: 1px solid #aaa">
            <td colspan="3">
<pre style="border: 0; font-family: Consolas, Menlo, &quot;Liberation Mono&quot;, Courier, monospace; line-height: 1.2; margin: 0; padding: 0.5em; white-space: normal; width: auto"><span class="diff-header" style="background-color: #eaf2f5; color: #999999; display: block; white-space: pre">  Modified: lib/proc.c (+28 -1)</span>
<span class="diff-header-mark" style="background-color: #eaf2f5; color: #999999; display: block; white-space: pre">===================================================================</span>
</pre>
            </td>
          </tr>
        </thead>
        <tbody>
          <tr>
            <th class="diff-line-number" style="border: 1px solid #aaa">
<pre style="border: 0; font-family: Consolas, Menlo, &quot;Liberation Mono&quot;, Courier, monospace; line-height: 1.2; margin: 0; padding: 0.5em; white-space: normal; width: auto"><span class="diff-line-number-hunk-header" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/7d11c2f6960f244ed0ed5c0408dd80beed668fa2#L0L2957">...</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/7d11c2f6960f244ed0ed5c0408dd80beed668fa2#L0L2958">2958</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/7d11c2f6960f244ed0ed5c0408dd80beed668fa2#L0L2959">2959</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/7d11c2f6960f244ed0ed5c0408dd80beed668fa2#L0L2960">2960</a></span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/7d11c2f6960f244ed0ed5c0408dd80beed668fa2#L0L2961">2961</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/7d11c2f6960f244ed0ed5c0408dd80beed668fa2#L0L2962">2962</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/7d11c2f6960f244ed0ed5c0408dd80beed668fa2#L0L2963">2963</a></span>
<span class="diff-line-number-hunk-header" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/7d11c2f6960f244ed0ed5c0408dd80beed668fa2#L0L3189">...</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/7d11c2f6960f244ed0ed5c0408dd80beed668fa2#L0L3190">3190</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/7d11c2f6960f244ed0ed5c0408dd80beed668fa2#L0L3191">3191</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/7d11c2f6960f244ed0ed5c0408dd80beed668fa2#L0L3192">3192</a></span>
<span class="diff-line-number-deleted" style="background-color: #ffaaaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/7d11c2f6960f244ed0ed5c0408dd80beed668fa2#L0L3193">3193</a></span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/7d11c2f6960f244ed0ed5c0408dd80beed668fa2#L0L3194">3194</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/7d11c2f6960f244ed0ed5c0408dd80beed668fa2#L0L3195">3195</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/7d11c2f6960f244ed0ed5c0408dd80beed668fa2#L0L3196">3196</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/7d11c2f6960f244ed0ed5c0408dd80beed668fa2#L0L3197">3197</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/7d11c2f6960f244ed0ed5c0408dd80beed668fa2#L0L3198">3198</a></span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/7d11c2f6960f244ed0ed5c0408dd80beed668fa2#L0L3199">3199</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/7d11c2f6960f244ed0ed5c0408dd80beed668fa2#L0L3200">3200</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/7d11c2f6960f244ed0ed5c0408dd80beed668fa2#L0L3201">3201</a></span>
</pre>
            </th>
            <th class="diff-line-number" style="border: 1px solid #aaa">
<pre style="border: 0; font-family: Consolas, Menlo, &quot;Liberation Mono&quot;, Courier, monospace; line-height: 1.2; margin: 0; padding: 0.5em; white-space: normal; width: auto"><span class="diff-line-number-hunk-header" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/7d11c2f6960f244ed0ed5c0408dd80beed668fa2#L0R2957">...</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/7d11c2f6960f244ed0ed5c0408dd80beed668fa2#L0R2958">2958</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/7d11c2f6960f244ed0ed5c0408dd80beed668fa2#L0R2959">2959</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/7d11c2f6960f244ed0ed5c0408dd80beed668fa2#L0R2960">2960</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/7d11c2f6960f244ed0ed5c0408dd80beed668fa2#L0R2961">2961</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/7d11c2f6960f244ed0ed5c0408dd80beed668fa2#L0R2962">2962</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/7d11c2f6960f244ed0ed5c0408dd80beed668fa2#L0R2963">2963</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/7d11c2f6960f244ed0ed5c0408dd80beed668fa2#L0R2964">2964</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/7d11c2f6960f244ed0ed5c0408dd80beed668fa2#L0R2965">2965</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/7d11c2f6960f244ed0ed5c0408dd80beed668fa2#L0R2966">2966</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/7d11c2f6960f244ed0ed5c0408dd80beed668fa2#L0R2967">2967</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/7d11c2f6960f244ed0ed5c0408dd80beed668fa2#L0R2968">2968</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/7d11c2f6960f244ed0ed5c0408dd80beed668fa2#L0R2969">2969</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/7d11c2f6960f244ed0ed5c0408dd80beed668fa2#L0R2970">2970</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/7d11c2f6960f244ed0ed5c0408dd80beed668fa2#L0R2971">2971</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/7d11c2f6960f244ed0ed5c0408dd80beed668fa2#L0R2972">2972</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/7d11c2f6960f244ed0ed5c0408dd80beed668fa2#L0R2973">2973</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/7d11c2f6960f244ed0ed5c0408dd80beed668fa2#L0R2974">2974</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/7d11c2f6960f244ed0ed5c0408dd80beed668fa2#L0R2975">2975</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/7d11c2f6960f244ed0ed5c0408dd80beed668fa2#L0R2976">2976</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/7d11c2f6960f244ed0ed5c0408dd80beed668fa2#L0R2977">2977</a></span>
<span class="diff-line-number-hunk-header" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/7d11c2f6960f244ed0ed5c0408dd80beed668fa2#L0R3203">...</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/7d11c2f6960f244ed0ed5c0408dd80beed668fa2#L0R3204">3204</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/7d11c2f6960f244ed0ed5c0408dd80beed668fa2#L0R3205">3205</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/7d11c2f6960f244ed0ed5c0408dd80beed668fa2#L0R3206">3206</a></span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/7d11c2f6960f244ed0ed5c0408dd80beed668fa2#L0R3207">3207</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/7d11c2f6960f244ed0ed5c0408dd80beed668fa2#L0R3208">3208</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/7d11c2f6960f244ed0ed5c0408dd80beed668fa2#L0R3209">3209</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/7d11c2f6960f244ed0ed5c0408dd80beed668fa2#L0R3210">3210</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/7d11c2f6960f244ed0ed5c0408dd80beed668fa2#L0R3211">3211</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/7d11c2f6960f244ed0ed5c0408dd80beed668fa2#L0R3212">3212</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/7d11c2f6960f244ed0ed5c0408dd80beed668fa2#L0R3213">3213</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/7d11c2f6960f244ed0ed5c0408dd80beed668fa2#L0R3214">3214</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/7d11c2f6960f244ed0ed5c0408dd80beed668fa2#L0R3215">3215</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/7d11c2f6960f244ed0ed5c0408dd80beed668fa2#L0R3216">3216</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/7d11c2f6960f244ed0ed5c0408dd80beed668fa2#L0R3217">3217</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/7d11c2f6960f244ed0ed5c0408dd80beed668fa2#L0R3218">3218</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/7d11c2f6960f244ed0ed5c0408dd80beed668fa2#L0R3219">3219</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/7d11c2f6960f244ed0ed5c0408dd80beed668fa2#L0R3220">3220</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/7d11c2f6960f244ed0ed5c0408dd80beed668fa2#L0R3221">3221</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/7d11c2f6960f244ed0ed5c0408dd80beed668fa2#L0R3222">3222</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/7d11c2f6960f244ed0ed5c0408dd80beed668fa2#L0R3223">3223</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/7d11c2f6960f244ed0ed5c0408dd80beed668fa2#L0R3224">3224</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/7d11c2f6960f244ed0ed5c0408dd80beed668fa2#L0R3225">3225</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/7d11c2f6960f244ed0ed5c0408dd80beed668fa2#L0R3226">3226</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/7d11c2f6960f244ed0ed5c0408dd80beed668fa2#L0R3227">3227</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/7d11c2f6960f244ed0ed5c0408dd80beed668fa2#L0R3228">3228</a></span>
</pre>
            </th>
            <td class="diff-content" style="border: 1px solid #aaa">
<pre style="border: 0; font-family: Consolas, Menlo, &quot;Liberation Mono&quot;, Courier, monospace; line-height: 1.2; margin: 0; padding: 0.5em; white-space: normal; width: auto"><span class="diff-hunk-header" style="background-color: #eaf2f5; color: #999999; display: block; white-space: pre">@@ -2958,6 +2958,20 @@ <span class="diff-context" style="background-color: #ffffaa; color: #000000">is_normalizer(grn_ctx *ctx, grn_obj *object)</span></span>
<span class="diff-not-changed" style="display: block; white-space: pre">   return GRN_TRUE;</span>
<span class="diff-not-changed" style="display: block; white-space: pre"> }</span>
<span class="diff-not-changed" style="display: block; white-space: pre"> </span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+static grn_bool</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+is_tokenizer(grn_ctx *ctx, grn_obj *object)</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+{</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+  if (object-&gt;header.type != GRN_PROC) {</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+    return GRN_FALSE;</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+  }</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+  if (grn_proc_get_type(ctx, object) != GRN_PROC_TOKENIZER) {</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+    return GRN_FALSE;</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+  }</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+  return GRN_TRUE;</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+}</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+</span>
<span class="diff-not-changed" style="display: block; white-space: pre"> static const char *</span>
<span class="diff-not-changed" style="display: block; white-space: pre"> char_type_name(grn_char_type type)</span>
<span class="diff-not-changed" style="display: block; white-space: pre"> {</span>
<span class="diff-hunk-header" style="background-color: #eaf2f5; color: #999999; display: block; white-space: pre">@@ -3190,12 +3204,25 @@ <span class="diff-context" style="background-color: #ffffaa; color: #000000">create_lexicon_for_tokenize(grn_ctx *ctx,</span></span>
<span class="diff-not-changed" style="display: block; white-space: pre">                           GRN_TEXT_LEN(tokenizer_name));</span>
<span class="diff-not-changed" style="display: block; white-space: pre">   if (!tokenizer) {</span>
<span class="diff-not-changed" style="display: block; white-space: pre">     ERR(GRN_INVALID_ARGUMENT,</span>
<span class="diff-deleted" style="background-color: #ffaaaa; color: #000000; display: block; white-space: pre">-        &quot;[tokenize] unknown tokenizer: &lt;%.*s&gt;&quot;,</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+        &quot;[tokenize] nonexistent tokenizer: &lt;%.*s&gt;&quot;,</span>
<span class="diff-not-changed" style="display: block; white-space: pre">         (int)GRN_TEXT_LEN(tokenizer_name),</span>
<span class="diff-not-changed" style="display: block; white-space: pre">         GRN_TEXT_VALUE(tokenizer_name));</span>
<span class="diff-not-changed" style="display: block; white-space: pre">     return NULL;</span>
<span class="diff-not-changed" style="display: block; white-space: pre">   }</span>
<span class="diff-not-changed" style="display: block; white-space: pre"> </span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+  if (!is_tokenizer(ctx, tokenizer)) {</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+    grn_obj inspected;</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+    GRN_TEXT_INIT(&amp;inspected, 0);</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+    grn_inspect(ctx, &amp;inspected, tokenizer);</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+    ERR(GRN_INVALID_ARGUMENT,</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+        &quot;[tokenize] not tokenizer: %.*s&quot;,</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+        (int)GRN_TEXT_LEN(&amp;inspected),</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+        GRN_TEXT_VALUE(&amp;inspected));</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+    GRN_OBJ_FIN(ctx, &amp;inspected);</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+    grn_obj_unlink(ctx, tokenizer);</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+    return NULL;</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+  }</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+</span>
<span class="diff-not-changed" style="display: block; white-space: pre">   if (GRN_TEXT_LEN(normalizer_name) &gt; 0) {</span>
<span class="diff-not-changed" style="display: block; white-space: pre">     normalizer = grn_ctx_get(ctx,</span>
<span class="diff-not-changed" style="display: block; white-space: pre">                              GRN_TEXT_VALUE(normalizer_name),</span>
</pre>
            </td>
          </tr>
        </tbody>
      </table>
      <table style="border-collapse: collapse; border: 1px solid #aaa">
        <thead>
          <tr class="diff-header" style="border: 1px solid #aaa">
            <td colspan="3">
<pre style="border: 0; font-family: Consolas, Menlo, &quot;Liberation Mono&quot;, Courier, monospace; line-height: 1.2; margin: 0; padding: 0.5em; white-space: normal; width: auto"><span class="diff-header" style="background-color: #eaf2f5; color: #999999; display: block; white-space: pre">  Added: test/command/suite/tokenize/invalid/tokenizer/invalid.expected (+15 -0) 100644</span>
<span class="diff-header-mark" style="background-color: #eaf2f5; color: #999999; display: block; white-space: pre">===================================================================</span>
</pre>
            </td>
          </tr>
        </thead>
        <tbody>
          <tr>
            <th class="diff-line-number" style="border: 1px solid #aaa">
<pre style="border: 0; font-family: Consolas, Menlo, &quot;Liberation Mono&quot;, Courier, monospace; line-height: 1.2; margin: 0; padding: 0.5em; white-space: normal; width: auto"><span class="diff-line-number-hunk-header" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/7d11c2f6960f244ed0ed5c0408dd80beed668fa2#L1L-1">...</a></span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
</pre>
            </th>
            <th class="diff-line-number" style="border: 1px solid #aaa">
<pre style="border: 0; font-family: Consolas, Menlo, &quot;Liberation Mono&quot;, Courier, monospace; line-height: 1.2; margin: 0; padding: 0.5em; white-space: normal; width: auto"><span class="diff-line-number-hunk-header" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/7d11c2f6960f244ed0ed5c0408dd80beed668fa2#L1R0">...</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/7d11c2f6960f244ed0ed5c0408dd80beed668fa2#L1R1">1</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/7d11c2f6960f244ed0ed5c0408dd80beed668fa2#L1R2">2</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/7d11c2f6960f244ed0ed5c0408dd80beed668fa2#L1R3">3</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/7d11c2f6960f244ed0ed5c0408dd80beed668fa2#L1R4">4</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/7d11c2f6960f244ed0ed5c0408dd80beed668fa2#L1R5">5</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/7d11c2f6960f244ed0ed5c0408dd80beed668fa2#L1R6">6</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/7d11c2f6960f244ed0ed5c0408dd80beed668fa2#L1R7">7</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/7d11c2f6960f244ed0ed5c0408dd80beed668fa2#L1R8">8</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/7d11c2f6960f244ed0ed5c0408dd80beed668fa2#L1R9">9</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/7d11c2f6960f244ed0ed5c0408dd80beed668fa2#L1R10">10</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/7d11c2f6960f244ed0ed5c0408dd80beed668fa2#L1R11">11</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/7d11c2f6960f244ed0ed5c0408dd80beed668fa2#L1R12">12</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/7d11c2f6960f244ed0ed5c0408dd80beed668fa2#L1R13">13</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/7d11c2f6960f244ed0ed5c0408dd80beed668fa2#L1R14">14</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/7d11c2f6960f244ed0ed5c0408dd80beed668fa2#L1R15">15</a></span>
</pre>
            </th>
            <td class="diff-content" style="border: 1px solid #aaa">
<pre style="border: 0; font-family: Consolas, Menlo, &quot;Liberation Mono&quot;, Courier, monospace; line-height: 1.2; margin: 0; padding: 0.5em; white-space: normal; width: auto"><span class="diff-hunk-header" style="background-color: #eaf2f5; color: #999999; display: block; white-space: pre">@@ -0,0 +1,15 @@</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+tokenize NormalizerAuto &quot;aBcDe 123&quot;</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+[</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+  [</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+    [</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+      -22,</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+      0.0,</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+      0.0</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+    ],</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+    &quot;[tokenize] not tokenizer: #&lt;proc:normalizer NormalizerAuto arguments:[$1]&gt;&quot;</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+  ],</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+  [</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+  ]</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+]</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+#|e| [tokenize] not tokenizer: #&lt;proc:normalizer NormalizerAuto arguments:[$1]&gt;</span>
</pre>
            </td>
          </tr>
        </tbody>
      </table>
      <table style="border-collapse: collapse; border: 1px solid #aaa">
        <thead>
          <tr class="diff-header" style="border: 1px solid #aaa">
            <td colspan="3">
<pre style="border: 0; font-family: Consolas, Menlo, &quot;Liberation Mono&quot;, Courier, monospace; line-height: 1.2; margin: 0; padding: 0.5em; white-space: normal; width: auto"><span class="diff-header" style="background-color: #eaf2f5; color: #999999; display: block; white-space: pre">  Added: test/command/suite/tokenize/invalid/tokenizer/invalid.test (+1 -0) 100644</span>
<span class="diff-header-mark" style="background-color: #eaf2f5; color: #999999; display: block; white-space: pre">===================================================================</span>
</pre>
            </td>
          </tr>
        </thead>
        <tbody>
          <tr>
            <th class="diff-line-number" style="border: 1px solid #aaa">
<pre style="border: 0; font-family: Consolas, Menlo, &quot;Liberation Mono&quot;, Courier, monospace; line-height: 1.2; margin: 0; padding: 0.5em; white-space: normal; width: auto"><span class="diff-line-number-hunk-header" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/7d11c2f6960f244ed0ed5c0408dd80beed668fa2#L2L-1">...</a></span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
</pre>
            </th>
            <th class="diff-line-number" style="border: 1px solid #aaa">
<pre style="border: 0; font-family: Consolas, Menlo, &quot;Liberation Mono&quot;, Courier, monospace; line-height: 1.2; margin: 0; padding: 0.5em; white-space: normal; width: auto"><span class="diff-line-number-hunk-header" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/7d11c2f6960f244ed0ed5c0408dd80beed668fa2#L2R0">...</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/7d11c2f6960f244ed0ed5c0408dd80beed668fa2#L2R1">1</a></span>
</pre>
            </th>
            <td class="diff-content" style="border: 1px solid #aaa">
<pre style="border: 0; font-family: Consolas, Menlo, &quot;Liberation Mono&quot;, Courier, monospace; line-height: 1.2; margin: 0; padding: 0.5em; white-space: normal; width: auto"><span class="diff-hunk-header" style="background-color: #eaf2f5; color: #999999; display: block; white-space: pre">@@ -0,0 +1 @@</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+tokenize NormalizerAuto &quot;aBcDe 123&quot;</span>
</pre>
            </td>
          </tr>
        </tbody>
      </table>
      <table style="border-collapse: collapse; border: 1px solid #aaa">
        <thead>
          <tr class="diff-header" style="border: 1px solid #aaa">
            <td colspan="3">
<pre style="border: 0; font-family: Consolas, Menlo, &quot;Liberation Mono&quot;, Courier, monospace; line-height: 1.2; margin: 0; padding: 0.5em; white-space: normal; width: auto"><span class="diff-header" style="background-color: #eaf2f5; color: #999999; display: block; white-space: pre">  Added: test/command/suite/tokenize/invalid/tokenizer/nonexistent.expected (+3 -0) 100644</span>
<span class="diff-header-mark" style="background-color: #eaf2f5; color: #999999; display: block; white-space: pre">===================================================================</span>
</pre>
            </td>
          </tr>
        </thead>
        <tbody>
          <tr>
            <th class="diff-line-number" style="border: 1px solid #aaa">
<pre style="border: 0; font-family: Consolas, Menlo, &quot;Liberation Mono&quot;, Courier, monospace; line-height: 1.2; margin: 0; padding: 0.5em; white-space: normal; width: auto"><span class="diff-line-number-hunk-header" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/7d11c2f6960f244ed0ed5c0408dd80beed668fa2#L3L-1">...</a></span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
</pre>
            </th>
            <th class="diff-line-number" style="border: 1px solid #aaa">
<pre style="border: 0; font-family: Consolas, Menlo, &quot;Liberation Mono&quot;, Courier, monospace; line-height: 1.2; margin: 0; padding: 0.5em; white-space: normal; width: auto"><span class="diff-line-number-hunk-header" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/7d11c2f6960f244ed0ed5c0408dd80beed668fa2#L3R0">...</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/7d11c2f6960f244ed0ed5c0408dd80beed668fa2#L3R1">1</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/7d11c2f6960f244ed0ed5c0408dd80beed668fa2#L3R2">2</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/7d11c2f6960f244ed0ed5c0408dd80beed668fa2#L3R3">3</a></span>
</pre>
            </th>
            <td class="diff-content" style="border: 1px solid #aaa">
<pre style="border: 0; font-family: Consolas, Menlo, &quot;Liberation Mono&quot;, Courier, monospace; line-height: 1.2; margin: 0; padding: 0.5em; white-space: normal; width: auto"><span class="diff-hunk-header" style="background-color: #eaf2f5; color: #999999; display: block; white-space: pre">@@ -0,0 +1,3 @@</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+tokenize TokenNonexistent &quot;aBcDe 123&quot;</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+[[[-22,0.0,0.0],&quot;[tokenize] nonexistent tokenizer: &lt;TokenNonexistent&gt;&quot;],[]]</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+#|e| [tokenize] nonexistent tokenizer: &lt;TokenNonexistent&gt;</span>
</pre>
            </td>
          </tr>
        </tbody>
      </table>
      <table style="border-collapse: collapse; border: 1px solid #aaa">
        <thead>
          <tr class="diff-header" style="border: 1px solid #aaa">
            <td colspan="3">
<pre style="border: 0; font-family: Consolas, Menlo, &quot;Liberation Mono&quot;, Courier, monospace; line-height: 1.2; margin: 0; padding: 0.5em; white-space: normal; width: auto"><span class="diff-header" style="background-color: #eaf2f5; color: #999999; display: block; white-space: pre">  Added: test/command/suite/tokenize/invalid/tokenizer/nonexistent.test (+1 -0) 100644</span>
<span class="diff-header-mark" style="background-color: #eaf2f5; color: #999999; display: block; white-space: pre">===================================================================</span>
</pre>
            </td>
          </tr>
        </thead>
        <tbody>
          <tr>
            <th class="diff-line-number" style="border: 1px solid #aaa">
<pre style="border: 0; font-family: Consolas, Menlo, &quot;Liberation Mono&quot;, Courier, monospace; line-height: 1.2; margin: 0; padding: 0.5em; white-space: normal; width: auto"><span class="diff-line-number-hunk-header" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/7d11c2f6960f244ed0ed5c0408dd80beed668fa2#L4L-1">...</a></span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
</pre>
            </th>
            <th class="diff-line-number" style="border: 1px solid #aaa">
<pre style="border: 0; font-family: Consolas, Menlo, &quot;Liberation Mono&quot;, Courier, monospace; line-height: 1.2; margin: 0; padding: 0.5em; white-space: normal; width: auto"><span class="diff-line-number-hunk-header" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/7d11c2f6960f244ed0ed5c0408dd80beed668fa2#L4R0">...</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/7d11c2f6960f244ed0ed5c0408dd80beed668fa2#L4R1">1</a></span>
</pre>
            </th>
            <td class="diff-content" style="border: 1px solid #aaa">
<pre style="border: 0; font-family: Consolas, Menlo, &quot;Liberation Mono&quot;, Courier, monospace; line-height: 1.2; margin: 0; padding: 0.5em; white-space: normal; width: auto"><span class="diff-hunk-header" style="background-color: #eaf2f5; color: #999999; display: block; white-space: pre">@@ -0,0 +1 @@</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+tokenize TokenNonexistent &quot;aBcDe 123&quot;</span>
</pre>
            </td>
          </tr>
        </tbody>
      </table>
      <table style="border-collapse: collapse; border: 1px solid #aaa">
        <thead>
          <tr class="diff-header" style="border: 1px solid #aaa">
            <td colspan="3">
<pre style="border: 0; font-family: Consolas, Menlo, &quot;Liberation Mono&quot;, Courier, monospace; line-height: 1.2; margin: 0; padding: 0.5em; white-space: normal; width: auto"><span class="diff-header" style="background-color: #eaf2f5; color: #999999; display: block; white-space: pre">  Deleted: test/command/suite/tokenize/invalid/tokenizer/unknown.expected (+0 -3) 100644</span>
<span class="diff-header-mark" style="background-color: #eaf2f5; color: #999999; display: block; white-space: pre">===================================================================</span>
</pre>
            </td>
          </tr>
        </thead>
        <tbody>
          <tr>
            <th class="diff-line-number" style="border: 1px solid #aaa">
<pre style="border: 0; font-family: Consolas, Menlo, &quot;Liberation Mono&quot;, Courier, monospace; line-height: 1.2; margin: 0; padding: 0.5em; white-space: normal; width: auto"><span class="diff-line-number-hunk-header" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/7d11c2f6960f244ed0ed5c0408dd80beed668fa2#L5L0">...</a></span>
<span class="diff-line-number-deleted" style="background-color: #ffaaaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/7d11c2f6960f244ed0ed5c0408dd80beed668fa2#L5L1">1</a></span>
<span class="diff-line-number-deleted" style="background-color: #ffaaaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/7d11c2f6960f244ed0ed5c0408dd80beed668fa2#L5L2">2</a></span>
<span class="diff-line-number-deleted" style="background-color: #ffaaaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/7d11c2f6960f244ed0ed5c0408dd80beed668fa2#L5L3">3</a></span>
</pre>
            </th>
            <th class="diff-line-number" style="border: 1px solid #aaa">
<pre style="border: 0; font-family: Consolas, Menlo, &quot;Liberation Mono&quot;, Courier, monospace; line-height: 1.2; margin: 0; padding: 0.5em; white-space: normal; width: auto"><span class="diff-line-number-hunk-header" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/7d11c2f6960f244ed0ed5c0408dd80beed668fa2#L5R-1">...</a></span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
</pre>
            </th>
            <td class="diff-content" style="border: 1px solid #aaa">
<pre style="border: 0; font-family: Consolas, Menlo, &quot;Liberation Mono&quot;, Courier, monospace; line-height: 1.2; margin: 0; padding: 0.5em; white-space: normal; width: auto"><span class="diff-hunk-header" style="background-color: #eaf2f5; color: #999999; display: block; white-space: pre">@@ -1,3 +0,0 @@</span>
<span class="diff-deleted" style="background-color: #ffaaaa; color: #000000; display: block; white-space: pre">-tokenize TokenUnknown &quot;aBcDe 123&quot;</span>
<span class="diff-deleted" style="background-color: #ffaaaa; color: #000000; display: block; white-space: pre">-[[[-22,0.0,0.0],&quot;[tokenize] unknown tokenizer: &lt;TokenUnknown&gt;&quot;],[]]</span>
<span class="diff-deleted" style="background-color: #ffaaaa; color: #000000; display: block; white-space: pre">-#|e| [tokenize] unknown tokenizer: &lt;TokenUnknown&gt;</span>
</pre>
            </td>
          </tr>
        </tbody>
      </table>
      <table style="border-collapse: collapse; border: 1px solid #aaa">
        <thead>
          <tr class="diff-header" style="border: 1px solid #aaa">
            <td colspan="3">
<pre style="border: 0; font-family: Consolas, Menlo, &quot;Liberation Mono&quot;, Courier, monospace; line-height: 1.2; margin: 0; padding: 0.5em; white-space: normal; width: auto"><span class="diff-header" style="background-color: #eaf2f5; color: #999999; display: block; white-space: pre">  Deleted: test/command/suite/tokenize/invalid/tokenizer/unknown.test (+0 -1) 100644</span>
<span class="diff-header-mark" style="background-color: #eaf2f5; color: #999999; display: block; white-space: pre">===================================================================</span>
</pre>
            </td>
          </tr>
        </thead>
        <tbody>
          <tr>
            <th class="diff-line-number" style="border: 1px solid #aaa">
<pre style="border: 0; font-family: Consolas, Menlo, &quot;Liberation Mono&quot;, Courier, monospace; line-height: 1.2; margin: 0; padding: 0.5em; white-space: normal; width: auto"><span class="diff-line-number-hunk-header" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/7d11c2f6960f244ed0ed5c0408dd80beed668fa2#L6L0">...</a></span>
<span class="diff-line-number-deleted" style="background-color: #ffaaaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/7d11c2f6960f244ed0ed5c0408dd80beed668fa2#L6L1">1</a></span>
</pre>
            </th>
            <th class="diff-line-number" style="border: 1px solid #aaa">
<pre style="border: 0; font-family: Consolas, Menlo, &quot;Liberation Mono&quot;, Courier, monospace; line-height: 1.2; margin: 0; padding: 0.5em; white-space: normal; width: auto"><span class="diff-line-number-hunk-header" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/7d11c2f6960f244ed0ed5c0408dd80beed668fa2#L6R-1">...</a></span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
</pre>
            </th>
            <td class="diff-content" style="border: 1px solid #aaa">
<pre style="border: 0; font-family: Consolas, Menlo, &quot;Liberation Mono&quot;, Courier, monospace; line-height: 1.2; margin: 0; padding: 0.5em; white-space: normal; width: auto"><span class="diff-hunk-header" style="background-color: #eaf2f5; color: #999999; display: block; white-space: pre">@@ -1 +0,0 @@</span>
<span class="diff-deleted" style="background-color: #ffaaaa; color: #000000; display: block; white-space: pre">-tokenize TokenUnknown &quot;aBcDe 123&quot;</span>
</pre>
            </td>
          </tr>
        </tbody>
      </table>
    </div>
  </body>
</html>