<!DOCTYPE html>
<html>
  <head>
  </head>
  <body>
    <dl style="line-height: 1.5; margin-left: 2em">
      <dt style="clear: both; float: left; font-weight: bold; width: 8em">Author</dt>
      <dd style="margin-left: 8.5em">Kouhei Sutou &lt;kou****@clear*****&gt;</dd>
      <dt style="clear: both; float: left; font-weight: bold; width: 8em">Date</dt>
      <dd style="margin-left: 8.5em">2014-06-05 23:30:25 +0900 (Thu, 05 Jun 2014)</dd>
      <dt style="clear: both; float: left; font-weight: bold; width: 8em">New Revision</dt>
      <dd style="margin-left: 8.5em"><a href="https://github.com/groonga/groonga/commit/d0873c7056b5a98d704569bbf82daaa5af1865bc">d0873c7056b5a98d704569bbf82daaa5af1865bc</a></dd>
      <dt style="clear: both; float: left; font-weight: bold; width: 8em">Message</dt>
      <dd style="margin-left: 8.5em"><pre style="border: 1px solid #aaa; font-family: Consolas, Menlo, &quot;Liberation Mono&quot;, Courier, monospace; line-height: 1.2; padding: 0.5em; width: auto">tokenize: add valid normalizer check</pre></dd>
      <dt style="clear: both; float: left; font-weight: bold; width: 8em">Added files</dt>
      <dd style="margin-left: 8.5em">
        <ul>
          <li><a href="https://github.com/groonga/groonga/commit/d0873c7056b5a98d704569bbf82daaa5af1865bc#diff-1">test/command/suite/tokenize/invalid/normalizer/invalid.expected</a></li>
          <li><a href="https://github.com/groonga/groonga/commit/d0873c7056b5a98d704569bbf82daaa5af1865bc#diff-2">test/command/suite/tokenize/invalid/normalizer/invalid.test</a></li>
          <li><a href="https://github.com/groonga/groonga/commit/d0873c7056b5a98d704569bbf82daaa5af1865bc#diff-3">test/command/suite/tokenize/invalid/normalizer/nonexistent.expected</a></li>
          <li><a href="https://github.com/groonga/groonga/commit/d0873c7056b5a98d704569bbf82daaa5af1865bc#diff-4">test/command/suite/tokenize/invalid/normalizer/nonexistent.test</a></li>
        </ul>
      </dd>
      <dt style="clear: both; float: left; font-weight: bold; width: 8em">Removed files</dt>
      <dd style="margin-left: 8.5em">
        <ul>
          <li><a href="https://github.com/groonga/groonga/commit/d0873c7056b5a98d704569bbf82daaa5af1865bc#diff-5">test/command/suite/tokenize/invalid/normalizer/unknown.expected</a></li>
          <li><a href="https://github.com/groonga/groonga/commit/d0873c7056b5a98d704569bbf82daaa5af1865bc#diff-6">test/command/suite/tokenize/invalid/normalizer/unknown.test</a></li>
        </ul>
      </dd>
      <dt style="clear: both; float: left; font-weight: bold; width: 8em">Modified files</dt>
      <dd style="margin-left: 8.5em">
        <ul>
          <li><a href="https://github.com/groonga/groonga/commit/d0873c7056b5a98d704569bbf82daaa5af1865bc#diff-0">lib/proc.c</a></li>
        </ul>
      </dd>
    </dl>

    <div class="diff-section" style="clear: both">
      <table style="border-collapse: collapse; border: 1px solid #aaa">
        <thead>
          <tr class="diff-header" style="border: 1px solid #aaa">
            <td colspan="3">
<pre style="border: 0; font-family: Consolas, Menlo, &quot;Liberation Mono&quot;, Courier, monospace; line-height: 1.2; margin: 0; padding: 0.5em; white-space: normal; width: auto"><span class="diff-header" style="background-color: #eaf2f5; color: #999999; display: block; white-space: pre">  Modified: lib/proc.c (+14 -1)</span>
<span class="diff-header-mark" style="background-color: #eaf2f5; color: #999999; display: block; white-space: pre">===================================================================</span>
</pre>
            </td>
          </tr>
        </thead>
        <tbody>
          <tr>
            <th class="diff-line-number" style="border: 1px solid #aaa">
<pre style="border: 0; font-family: Consolas, Menlo, &quot;Liberation Mono&quot;, Courier, monospace; line-height: 1.2; margin: 0; padding: 0.5em; white-space: normal; width: auto"><span class="diff-line-number-hunk-header" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/d0873c7056b5a98d704569bbf82daaa5af1865bc#L0L3202">...</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/d0873c7056b5a98d704569bbf82daaa5af1865bc#L0L3203">3203</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/d0873c7056b5a98d704569bbf82daaa5af1865bc#L0L3204">3204</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/d0873c7056b5a98d704569bbf82daaa5af1865bc#L0L3205">3205</a></span>
<span class="diff-line-number-deleted" style="background-color: #ffaaaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/d0873c7056b5a98d704569bbf82daaa5af1865bc#L0L3206">3206</a></span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/d0873c7056b5a98d704569bbf82daaa5af1865bc#L0L3207">3207</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/d0873c7056b5a98d704569bbf82daaa5af1865bc#L0L3208">3208</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/d0873c7056b5a98d704569bbf82daaa5af1865bc#L0L3209">3209</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/d0873c7056b5a98d704569bbf82daaa5af1865bc#L0L3210">3210</a></span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/d0873c7056b5a98d704569bbf82daaa5af1865bc#L0L3211">3211</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/d0873c7056b5a98d704569bbf82daaa5af1865bc#L0L3212">3212</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/d0873c7056b5a98d704569bbf82daaa5af1865bc#L0L3213">3213</a></span>
</pre>
            </th>
            <th class="diff-line-number" style="border: 1px solid #aaa">
<pre style="border: 0; font-family: Consolas, Menlo, &quot;Liberation Mono&quot;, Courier, monospace; line-height: 1.2; margin: 0; padding: 0.5em; white-space: normal; width: auto"><span class="diff-line-number-hunk-header" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/d0873c7056b5a98d704569bbf82daaa5af1865bc#L0R3202">...</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/d0873c7056b5a98d704569bbf82daaa5af1865bc#L0R3203">3203</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/d0873c7056b5a98d704569bbf82daaa5af1865bc#L0R3204">3204</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/d0873c7056b5a98d704569bbf82daaa5af1865bc#L0R3205">3205</a></span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/d0873c7056b5a98d704569bbf82daaa5af1865bc#L0R3206">3206</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/d0873c7056b5a98d704569bbf82daaa5af1865bc#L0R3207">3207</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/d0873c7056b5a98d704569bbf82daaa5af1865bc#L0R3208">3208</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/d0873c7056b5a98d704569bbf82daaa5af1865bc#L0R3209">3209</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/d0873c7056b5a98d704569bbf82daaa5af1865bc#L0R3210">3210</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/d0873c7056b5a98d704569bbf82daaa5af1865bc#L0R3211">3211</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/d0873c7056b5a98d704569bbf82daaa5af1865bc#L0R3212">3212</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/d0873c7056b5a98d704569bbf82daaa5af1865bc#L0R3213">3213</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/d0873c7056b5a98d704569bbf82daaa5af1865bc#L0R3214">3214</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/d0873c7056b5a98d704569bbf82daaa5af1865bc#L0R3215">3215</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/d0873c7056b5a98d704569bbf82daaa5af1865bc#L0R3216">3216</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/d0873c7056b5a98d704569bbf82daaa5af1865bc#L0R3217">3217</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/d0873c7056b5a98d704569bbf82daaa5af1865bc#L0R3218">3218</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/d0873c7056b5a98d704569bbf82daaa5af1865bc#L0R3219">3219</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/d0873c7056b5a98d704569bbf82daaa5af1865bc#L0R3220">3220</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/d0873c7056b5a98d704569bbf82daaa5af1865bc#L0R3221">3221</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/d0873c7056b5a98d704569bbf82daaa5af1865bc#L0R3222">3222</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/d0873c7056b5a98d704569bbf82daaa5af1865bc#L0R3223">3223</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/d0873c7056b5a98d704569bbf82daaa5af1865bc#L0R3224">3224</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/d0873c7056b5a98d704569bbf82daaa5af1865bc#L0R3225">3225</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/d0873c7056b5a98d704569bbf82daaa5af1865bc#L0R3226">3226</a></span>
</pre>
            </th>
            <td class="diff-content" style="border: 1px solid #aaa">
<pre style="border: 0; font-family: Consolas, Menlo, &quot;Liberation Mono&quot;, Courier, monospace; line-height: 1.2; margin: 0; padding: 0.5em; white-space: normal; width: auto"><span class="diff-hunk-header" style="background-color: #eaf2f5; color: #999999; display: block; white-space: pre">@@ -3203,11 +3203,24 @@ <span class="diff-context" style="background-color: #ffffaa; color: #000000">create_lexicon_for_tokenize(grn_ctx *ctx,</span></span>
<span class="diff-not-changed" style="display: block; white-space: pre">     if (!normalizer) {</span>
<span class="diff-not-changed" style="display: block; white-space: pre">       grn_obj_unlink(ctx, tokenizer);</span>
<span class="diff-not-changed" style="display: block; white-space: pre">       ERR(GRN_INVALID_ARGUMENT,</span>
<span class="diff-deleted" style="background-color: #ffaaaa; color: #000000; display: block; white-space: pre">-          &quot;[tokenize] unknown normalizer: &lt;%.*s&gt;&quot;,</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+          &quot;[tokenize] nonexistent normalizer: &lt;%.*s&gt;&quot;,</span>
<span class="diff-not-changed" style="display: block; white-space: pre">           (int)GRN_TEXT_LEN(normalizer_name),</span>
<span class="diff-not-changed" style="display: block; white-space: pre">           GRN_TEXT_VALUE(normalizer_name));</span>
<span class="diff-not-changed" style="display: block; white-space: pre">       return NULL;</span>
<span class="diff-not-changed" style="display: block; white-space: pre">     }</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+    if (!is_normalizer(ctx, normalizer)) {</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+      grn_obj inspected;</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+      grn_obj_unlink(ctx, tokenizer);</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+      GRN_TEXT_INIT(&amp;inspected, 0);</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+      grn_inspect(ctx, &amp;inspected, normalizer);</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+      ERR(GRN_INVALID_ARGUMENT,</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+          &quot;[tokenize] not normalizer: %.*s&quot;,</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+          (int)GRN_TEXT_LEN(&amp;inspected),</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+          GRN_TEXT_VALUE(&amp;inspected));</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+      GRN_OBJ_FIN(ctx, &amp;inspected);</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+      return NULL;</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+    }</span>
<span class="diff-not-changed" style="display: block; white-space: pre">   }</span>
<span class="diff-not-changed" style="display: block; white-space: pre"> </span>
<span class="diff-not-changed" style="display: block; white-space: pre">   lexicon = grn_hash_create(ctx, NULL, GRN_TABLE_MAX_KEY_SIZE, 0,</span>
</pre>
            </td>
          </tr>
        </tbody>
      </table>
      <table style="border-collapse: collapse; border: 1px solid #aaa">
        <thead>
          <tr class="diff-header" style="border: 1px solid #aaa">
            <td colspan="3">
<pre style="border: 0; font-family: Consolas, Menlo, &quot;Liberation Mono&quot;, Courier, monospace; line-height: 1.2; margin: 0; padding: 0.5em; white-space: normal; width: auto"><span class="diff-header" style="background-color: #eaf2f5; color: #999999; display: block; white-space: pre">  Added: test/command/suite/tokenize/invalid/normalizer/invalid.expected (+15 -0) 100644</span>
<span class="diff-header-mark" style="background-color: #eaf2f5; color: #999999; display: block; white-space: pre">===================================================================</span>
</pre>
            </td>
          </tr>
        </thead>
        <tbody>
          <tr>
            <th class="diff-line-number" style="border: 1px solid #aaa">
<pre style="border: 0; font-family: Consolas, Menlo, &quot;Liberation Mono&quot;, Courier, monospace; line-height: 1.2; margin: 0; padding: 0.5em; white-space: normal; width: auto"><span class="diff-line-number-hunk-header" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/d0873c7056b5a98d704569bbf82daaa5af1865bc#L1L-1">...</a></span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
</pre>
            </th>
            <th class="diff-line-number" style="border: 1px solid #aaa">
<pre style="border: 0; font-family: Consolas, Menlo, &quot;Liberation Mono&quot;, Courier, monospace; line-height: 1.2; margin: 0; padding: 0.5em; white-space: normal; width: auto"><span class="diff-line-number-hunk-header" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/d0873c7056b5a98d704569bbf82daaa5af1865bc#L1R0">...</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/d0873c7056b5a98d704569bbf82daaa5af1865bc#L1R1">1</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/d0873c7056b5a98d704569bbf82daaa5af1865bc#L1R2">2</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/d0873c7056b5a98d704569bbf82daaa5af1865bc#L1R3">3</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/d0873c7056b5a98d704569bbf82daaa5af1865bc#L1R4">4</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/d0873c7056b5a98d704569bbf82daaa5af1865bc#L1R5">5</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/d0873c7056b5a98d704569bbf82daaa5af1865bc#L1R6">6</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/d0873c7056b5a98d704569bbf82daaa5af1865bc#L1R7">7</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/d0873c7056b5a98d704569bbf82daaa5af1865bc#L1R8">8</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/d0873c7056b5a98d704569bbf82daaa5af1865bc#L1R9">9</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/d0873c7056b5a98d704569bbf82daaa5af1865bc#L1R10">10</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/d0873c7056b5a98d704569bbf82daaa5af1865bc#L1R11">11</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/d0873c7056b5a98d704569bbf82daaa5af1865bc#L1R12">12</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/d0873c7056b5a98d704569bbf82daaa5af1865bc#L1R13">13</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/d0873c7056b5a98d704569bbf82daaa5af1865bc#L1R14">14</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/d0873c7056b5a98d704569bbf82daaa5af1865bc#L1R15">15</a></span>
</pre>
            </th>
            <td class="diff-content" style="border: 1px solid #aaa">
<pre style="border: 0; font-family: Consolas, Menlo, &quot;Liberation Mono&quot;, Courier, monospace; line-height: 1.2; margin: 0; padding: 0.5em; white-space: normal; width: auto"><span class="diff-hunk-header" style="background-color: #eaf2f5; color: #999999; display: block; white-space: pre">@@ -0,0 +1,15 @@</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+tokenize TokenBigram &quot;aBcDe 123&quot; TokenDelimit</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+[</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+  [</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+    [</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+      -22,</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+      0.0,</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+      0.0</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+    ],</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+    &quot;[tokenize] not normalizer: #&lt;proc:tokenizer TokenDelimit arguments:[$1, $2, $3]&gt;&quot;</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+  ],</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+  [</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+  ]</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+]</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+#|e| [tokenize] not normalizer: #&lt;proc:tokenizer TokenDelimit arguments:[$1, $2, $3]&gt;</span>
</pre>
            </td>
          </tr>
        </tbody>
      </table>
      <table style="border-collapse: collapse; border: 1px solid #aaa">
        <thead>
          <tr class="diff-header" style="border: 1px solid #aaa">
            <td colspan="3">
<pre style="border: 0; font-family: Consolas, Menlo, &quot;Liberation Mono&quot;, Courier, monospace; line-height: 1.2; margin: 0; padding: 0.5em; white-space: normal; width: auto"><span class="diff-header" style="background-color: #eaf2f5; color: #999999; display: block; white-space: pre">  Added: test/command/suite/tokenize/invalid/normalizer/invalid.test (+1 -0) 100644</span>
<span class="diff-header-mark" style="background-color: #eaf2f5; color: #999999; display: block; white-space: pre">===================================================================</span>
</pre>
            </td>
          </tr>
        </thead>
        <tbody>
          <tr>
            <th class="diff-line-number" style="border: 1px solid #aaa">
<pre style="border: 0; font-family: Consolas, Menlo, &quot;Liberation Mono&quot;, Courier, monospace; line-height: 1.2; margin: 0; padding: 0.5em; white-space: normal; width: auto"><span class="diff-line-number-hunk-header" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/d0873c7056b5a98d704569bbf82daaa5af1865bc#L2L-1">...</a></span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
</pre>
            </th>
            <th class="diff-line-number" style="border: 1px solid #aaa">
<pre style="border: 0; font-family: Consolas, Menlo, &quot;Liberation Mono&quot;, Courier, monospace; line-height: 1.2; margin: 0; padding: 0.5em; white-space: normal; width: auto"><span class="diff-line-number-hunk-header" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/d0873c7056b5a98d704569bbf82daaa5af1865bc#L2R0">...</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/d0873c7056b5a98d704569bbf82daaa5af1865bc#L2R1">1</a></span>
</pre>
            </th>
            <td class="diff-content" style="border: 1px solid #aaa">
<pre style="border: 0; font-family: Consolas, Menlo, &quot;Liberation Mono&quot;, Courier, monospace; line-height: 1.2; margin: 0; padding: 0.5em; white-space: normal; width: auto"><span class="diff-hunk-header" style="background-color: #eaf2f5; color: #999999; display: block; white-space: pre">@@ -0,0 +1 @@</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+tokenize TokenBigram &quot;aBcDe 123&quot; TokenDelimit</span>
</pre>
            </td>
          </tr>
        </tbody>
      </table>
      <table style="border-collapse: collapse; border: 1px solid #aaa">
        <thead>
          <tr class="diff-header" style="border: 1px solid #aaa">
            <td colspan="3">
<pre style="border: 0; font-family: Consolas, Menlo, &quot;Liberation Mono&quot;, Courier, monospace; line-height: 1.2; margin: 0; padding: 0.5em; white-space: normal; width: auto"><span class="diff-header" style="background-color: #eaf2f5; color: #999999; display: block; white-space: pre">  Added: test/command/suite/tokenize/invalid/normalizer/nonexistent.expected (+15 -0) 100644</span>
<span class="diff-header-mark" style="background-color: #eaf2f5; color: #999999; display: block; white-space: pre">===================================================================</span>
</pre>
            </td>
          </tr>
        </thead>
        <tbody>
          <tr>
            <th class="diff-line-number" style="border: 1px solid #aaa">
<pre style="border: 0; font-family: Consolas, Menlo, &quot;Liberation Mono&quot;, Courier, monospace; line-height: 1.2; margin: 0; padding: 0.5em; white-space: normal; width: auto"><span class="diff-line-number-hunk-header" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/d0873c7056b5a98d704569bbf82daaa5af1865bc#L3L-1">...</a></span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
</pre>
            </th>
            <th class="diff-line-number" style="border: 1px solid #aaa">
<pre style="border: 0; font-family: Consolas, Menlo, &quot;Liberation Mono&quot;, Courier, monospace; line-height: 1.2; margin: 0; padding: 0.5em; white-space: normal; width: auto"><span class="diff-line-number-hunk-header" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/d0873c7056b5a98d704569bbf82daaa5af1865bc#L3R0">...</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/d0873c7056b5a98d704569bbf82daaa5af1865bc#L3R1">1</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/d0873c7056b5a98d704569bbf82daaa5af1865bc#L3R2">2</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/d0873c7056b5a98d704569bbf82daaa5af1865bc#L3R3">3</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/d0873c7056b5a98d704569bbf82daaa5af1865bc#L3R4">4</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/d0873c7056b5a98d704569bbf82daaa5af1865bc#L3R5">5</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/d0873c7056b5a98d704569bbf82daaa5af1865bc#L3R6">6</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/d0873c7056b5a98d704569bbf82daaa5af1865bc#L3R7">7</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/d0873c7056b5a98d704569bbf82daaa5af1865bc#L3R8">8</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/d0873c7056b5a98d704569bbf82daaa5af1865bc#L3R9">9</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/d0873c7056b5a98d704569bbf82daaa5af1865bc#L3R10">10</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/d0873c7056b5a98d704569bbf82daaa5af1865bc#L3R11">11</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/d0873c7056b5a98d704569bbf82daaa5af1865bc#L3R12">12</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/d0873c7056b5a98d704569bbf82daaa5af1865bc#L3R13">13</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/d0873c7056b5a98d704569bbf82daaa5af1865bc#L3R14">14</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/d0873c7056b5a98d704569bbf82daaa5af1865bc#L3R15">15</a></span>
</pre>
            </th>
            <td class="diff-content" style="border: 1px solid #aaa">
<pre style="border: 0; font-family: Consolas, Menlo, &quot;Liberation Mono&quot;, Courier, monospace; line-height: 1.2; margin: 0; padding: 0.5em; white-space: normal; width: auto"><span class="diff-hunk-header" style="background-color: #eaf2f5; color: #999999; display: block; white-space: pre">@@ -0,0 +1,15 @@</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+tokenize TokenBigram &quot;aBcDe 123&quot; NormalizerNonexistent</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+[</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+  [</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+    [</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+      -22,</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+      0.0,</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+      0.0</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+    ],</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+    &quot;[tokenize] nonexistent normalizer: &lt;NormalizerNonexistent&gt;&quot;</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+  ],</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+  [</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+  ]</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+]</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+#|e| [tokenize] nonexistent normalizer: &lt;NormalizerNonexistent&gt;</span>
</pre>
            </td>
          </tr>
        </tbody>
      </table>
      <table style="border-collapse: collapse; border: 1px solid #aaa">
        <thead>
          <tr class="diff-header" style="border: 1px solid #aaa">
            <td colspan="3">
<pre style="border: 0; font-family: Consolas, Menlo, &quot;Liberation Mono&quot;, Courier, monospace; line-height: 1.2; margin: 0; padding: 0.5em; white-space: normal; width: auto"><span class="diff-header" style="background-color: #eaf2f5; color: #999999; display: block; white-space: pre">  Added: test/command/suite/tokenize/invalid/normalizer/nonexistent.test (+1 -0) 100644</span>
<span class="diff-header-mark" style="background-color: #eaf2f5; color: #999999; display: block; white-space: pre">===================================================================</span>
</pre>
            </td>
          </tr>
        </thead>
        <tbody>
          <tr>
            <th class="diff-line-number" style="border: 1px solid #aaa">
<pre style="border: 0; font-family: Consolas, Menlo, &quot;Liberation Mono&quot;, Courier, monospace; line-height: 1.2; margin: 0; padding: 0.5em; white-space: normal; width: auto"><span class="diff-line-number-hunk-header" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/d0873c7056b5a98d704569bbf82daaa5af1865bc#L4L-1">...</a></span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
</pre>
            </th>
            <th class="diff-line-number" style="border: 1px solid #aaa">
<pre style="border: 0; font-family: Consolas, Menlo, &quot;Liberation Mono&quot;, Courier, monospace; line-height: 1.2; margin: 0; padding: 0.5em; white-space: normal; width: auto"><span class="diff-line-number-hunk-header" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/d0873c7056b5a98d704569bbf82daaa5af1865bc#L4R0">...</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/d0873c7056b5a98d704569bbf82daaa5af1865bc#L4R1">1</a></span>
</pre>
            </th>
            <td class="diff-content" style="border: 1px solid #aaa">
<pre style="border: 0; font-family: Consolas, Menlo, &quot;Liberation Mono&quot;, Courier, monospace; line-height: 1.2; margin: 0; padding: 0.5em; white-space: normal; width: auto"><span class="diff-hunk-header" style="background-color: #eaf2f5; color: #999999; display: block; white-space: pre">@@ -0,0 +1 @@</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+tokenize TokenBigram &quot;aBcDe 123&quot; NormalizerNonexistent</span>
</pre>
            </td>
          </tr>
        </tbody>
      </table>
      <table style="border-collapse: collapse; border: 1px solid #aaa">
        <thead>
          <tr class="diff-header" style="border: 1px solid #aaa">
            <td colspan="3">
<pre style="border: 0; font-family: Consolas, Menlo, &quot;Liberation Mono&quot;, Courier, monospace; line-height: 1.2; margin: 0; padding: 0.5em; white-space: normal; width: auto"><span class="diff-header" style="background-color: #eaf2f5; color: #999999; display: block; white-space: pre">  Deleted: test/command/suite/tokenize/invalid/normalizer/unknown.expected (+0 -3) 100644</span>
<span class="diff-header-mark" style="background-color: #eaf2f5; color: #999999; display: block; white-space: pre">===================================================================</span>
</pre>
            </td>
          </tr>
        </thead>
        <tbody>
          <tr>
            <th class="diff-line-number" style="border: 1px solid #aaa">
<pre style="border: 0; font-family: Consolas, Menlo, &quot;Liberation Mono&quot;, Courier, monospace; line-height: 1.2; margin: 0; padding: 0.5em; white-space: normal; width: auto"><span class="diff-line-number-hunk-header" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/d0873c7056b5a98d704569bbf82daaa5af1865bc#L5L0">...</a></span>
<span class="diff-line-number-deleted" style="background-color: #ffaaaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/d0873c7056b5a98d704569bbf82daaa5af1865bc#L5L1">1</a></span>
<span class="diff-line-number-deleted" style="background-color: #ffaaaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/d0873c7056b5a98d704569bbf82daaa5af1865bc#L5L2">2</a></span>
<span class="diff-line-number-deleted" style="background-color: #ffaaaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/d0873c7056b5a98d704569bbf82daaa5af1865bc#L5L3">3</a></span>
</pre>
            </th>
            <th class="diff-line-number" style="border: 1px solid #aaa">
<pre style="border: 0; font-family: Consolas, Menlo, &quot;Liberation Mono&quot;, Courier, monospace; line-height: 1.2; margin: 0; padding: 0.5em; white-space: normal; width: auto"><span class="diff-line-number-hunk-header" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/d0873c7056b5a98d704569bbf82daaa5af1865bc#L5R-1">...</a></span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
</pre>
            </th>
            <td class="diff-content" style="border: 1px solid #aaa">
<pre style="border: 0; font-family: Consolas, Menlo, &quot;Liberation Mono&quot;, Courier, monospace; line-height: 1.2; margin: 0; padding: 0.5em; white-space: normal; width: auto"><span class="diff-hunk-header" style="background-color: #eaf2f5; color: #999999; display: block; white-space: pre">@@ -1,3 +0,0 @@</span>
<span class="diff-deleted" style="background-color: #ffaaaa; color: #000000; display: block; white-space: pre">-tokenize TokenBigram &quot;aBcDe 123&quot; NormalizerUnknown</span>
<span class="diff-deleted" style="background-color: #ffaaaa; color: #000000; display: block; white-space: pre">-[[[-22,0.0,0.0],&quot;[tokenize] unknown normalizer: &lt;NormalizerUnknown&gt;&quot;],[]]</span>
<span class="diff-deleted" style="background-color: #ffaaaa; color: #000000; display: block; white-space: pre">-#|e| [tokenize] unknown normalizer: &lt;NormalizerUnknown&gt;</span>
</pre>
            </td>
          </tr>
        </tbody>
      </table>
      <table style="border-collapse: collapse; border: 1px solid #aaa">
        <thead>
          <tr class="diff-header" style="border: 1px solid #aaa">
            <td colspan="3">
<pre style="border: 0; font-family: Consolas, Menlo, &quot;Liberation Mono&quot;, Courier, monospace; line-height: 1.2; margin: 0; padding: 0.5em; white-space: normal; width: auto"><span class="diff-header" style="background-color: #eaf2f5; color: #999999; display: block; white-space: pre">  Deleted: test/command/suite/tokenize/invalid/normalizer/unknown.test (+0 -1) 100644</span>
<span class="diff-header-mark" style="background-color: #eaf2f5; color: #999999; display: block; white-space: pre">===================================================================</span>
</pre>
            </td>
          </tr>
        </thead>
        <tbody>
          <tr>
            <th class="diff-line-number" style="border: 1px solid #aaa">
<pre style="border: 0; font-family: Consolas, Menlo, &quot;Liberation Mono&quot;, Courier, monospace; line-height: 1.2; margin: 0; padding: 0.5em; white-space: normal; width: auto"><span class="diff-line-number-hunk-header" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/d0873c7056b5a98d704569bbf82daaa5af1865bc#L6L0">...</a></span>
<span class="diff-line-number-deleted" style="background-color: #ffaaaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/d0873c7056b5a98d704569bbf82daaa5af1865bc#L6L1">1</a></span>
</pre>
            </th>
            <th class="diff-line-number" style="border: 1px solid #aaa">
<pre style="border: 0; font-family: Consolas, Menlo, &quot;Liberation Mono&quot;, Courier, monospace; line-height: 1.2; margin: 0; padding: 0.5em; white-space: normal; width: auto"><span class="diff-line-number-hunk-header" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/d0873c7056b5a98d704569bbf82daaa5af1865bc#L6R-1">...</a></span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
</pre>
            </th>
            <td class="diff-content" style="border: 1px solid #aaa">
<pre style="border: 0; font-family: Consolas, Menlo, &quot;Liberation Mono&quot;, Courier, monospace; line-height: 1.2; margin: 0; padding: 0.5em; white-space: normal; width: auto"><span class="diff-hunk-header" style="background-color: #eaf2f5; color: #999999; display: block; white-space: pre">@@ -1 +0,0 @@</span>
<span class="diff-deleted" style="background-color: #ffaaaa; color: #000000; display: block; white-space: pre">-tokenize TokenBigram &quot;aBcDe 123&quot; NormalizerUnknown</span>
</pre>
            </td>
          </tr>
        </tbody>
      </table>
    </div>
  </body>
</html>