<!DOCTYPE html>
<html>
  <head>
  </head>
  <body>
    <dl style="line-height: 1.5; margin-left: 2em">
      <dt style="clear: both; float: left; font-weight: bold; width: 8em">Author</dt>
      <dd style="margin-left: 8.5em">Kouhei Sutou &lt;kou****@clear*****&gt;</dd>
      <dt style="clear: both; float: left; font-weight: bold; width: 8em">Date</dt>
      <dd style="margin-left: 8.5em">2015-04-02 13:03:17 +0900 (Thu, 02 Apr 2015)</dd>
      <dt style="clear: both; float: left; font-weight: bold; width: 8em">New Revision</dt>
      <dd style="margin-left: 8.5em"><a href="https://github.com/groonga/groonga/commit/5f42bd915b68925aff9f367febe94c5e14cd4b42">5f42bd915b68925aff9f367febe94c5e14cd4b42</a></dd>
      <dt style="clear: both; float: left; font-weight: bold; width: 8em">Message</dt>
      <dd style="margin-left: 8.5em"><pre style="border: 1px solid #aaa; font-family: Consolas, Menlo, &quot;Liberation Mono&quot;, Courier, monospace; line-height: 1.2; padding: 0.5em; width: auto">tokenizers regexp: skip the last one character token</pre></dd>
      <dt style="clear: both; float: left; font-weight: bold; width: 8em">Modified files</dt>
      <dd style="margin-left: 8.5em">
        <ul>
          <li><a href="https://github.com/groonga/groonga/commit/5f42bd915b68925aff9f367febe94c5e14cd4b42#diff-0">lib/tokenizers.c</a></li>
        </ul>
      </dd>
    </dl>

    <div class="diff-section" style="clear: both">
      <table style="border-collapse: collapse; border: 1px solid #aaa">
        <thead>
          <tr class="diff-header" style="border: 1px solid #aaa">
            <td colspan="3">
<pre style="border: 0; font-family: Consolas, Menlo, &quot;Liberation Mono&quot;, Courier, monospace; line-height: 1.2; margin: 0; padding: 0.5em; white-space: normal; width: auto"><span class="diff-header" style="background-color: #eaf2f5; color: #999999; display: block; white-space: pre">  Modified: lib/tokenizers.c (+26 -6)</span>
<span class="diff-header-mark" style="background-color: #eaf2f5; color: #999999; display: block; white-space: pre">===================================================================</span>
</pre>
            </td>
          </tr>
        </thead>
        <tbody>
          <tr>
            <th class="diff-line-number" style="border: 1px solid #aaa">
<pre style="border: 0; font-family: Consolas, Menlo, &quot;Liberation Mono&quot;, Courier, monospace; line-height: 1.2; margin: 0; padding: 0.5em; white-space: normal; width: auto"><span class="diff-line-number-hunk-header" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/5f42bd915b68925aff9f367febe94c5e14cd4b42#L0L477">...</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/5f42bd915b68925aff9f367febe94c5e14cd4b42#L0L478">478</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/5f42bd915b68925aff9f367febe94c5e14cd4b42#L0L479">479</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/5f42bd915b68925aff9f367febe94c5e14cd4b42#L0L480">480</a></span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/5f42bd915b68925aff9f367febe94c5e14cd4b42#L0L481">481</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/5f42bd915b68925aff9f367febe94c5e14cd4b42#L0L482">482</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/5f42bd915b68925aff9f367febe94c5e14cd4b42#L0L483">483</a></span>
<span class="diff-line-number-hunk-header" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/5f42bd915b68925aff9f367febe94c5e14cd4b42#L0L514">...</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/5f42bd915b68925aff9f367febe94c5e14cd4b42#L0L515">515</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/5f42bd915b68925aff9f367febe94c5e14cd4b42#L0L516">516</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/5f42bd915b68925aff9f367febe94c5e14cd4b42#L0L517">517</a></span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/5f42bd915b68925aff9f367febe94c5e14cd4b42#L0L518">518</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/5f42bd915b68925aff9f367febe94c5e14cd4b42#L0L519">519</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/5f42bd915b68925aff9f367febe94c5e14cd4b42#L0L520">520</a></span>
<span class="diff-line-number-hunk-header" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/5f42bd915b68925aff9f367febe94c5e14cd4b42#L0L658">...</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/5f42bd915b68925aff9f367febe94c5e14cd4b42#L0L659">659</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/5f42bd915b68925aff9f367febe94c5e14cd4b42#L0L660">660</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/5f42bd915b68925aff9f367febe94c5e14cd4b42#L0L661">661</a></span>
<span class="diff-line-number-deleted" style="background-color: #ffaaaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/5f42bd915b68925aff9f367febe94c5e14cd4b42#L0L662">662</a></span>
<span class="diff-line-number-deleted" style="background-color: #ffaaaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/5f42bd915b68925aff9f367febe94c5e14cd4b42#L0L663">663</a></span>
<span class="diff-line-number-deleted" style="background-color: #ffaaaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/5f42bd915b68925aff9f367febe94c5e14cd4b42#L0L664">664</a></span>
<span class="diff-line-number-deleted" style="background-color: #ffaaaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/5f42bd915b68925aff9f367febe94c5e14cd4b42#L0L665">665</a></span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/5f42bd915b68925aff9f367febe94c5e14cd4b42#L0L666">666</a></span>
<span class="diff-line-number-deleted" style="background-color: #ffaaaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/5f42bd915b68925aff9f367febe94c5e14cd4b42#L0L667">667</a></span>
<span class="diff-line-number-deleted" style="background-color: #ffaaaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/5f42bd915b68925aff9f367febe94c5e14cd4b42#L0L668">668</a></span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/5f42bd915b68925aff9f367febe94c5e14cd4b42#L0L669">669</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/5f42bd915b68925aff9f367febe94c5e14cd4b42#L0L670">670</a></span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/5f42bd915b68925aff9f367febe94c5e14cd4b42#L0L671">671</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/5f42bd915b68925aff9f367febe94c5e14cd4b42#L0L672">672</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/5f42bd915b68925aff9f367febe94c5e14cd4b42#L0L673">673</a></span>
<span class="diff-line-number-hunk-header" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/5f42bd915b68925aff9f367febe94c5e14cd4b42#L0L674">...</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/5f42bd915b68925aff9f367febe94c5e14cd4b42#L0L675">675</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/5f42bd915b68925aff9f367febe94c5e14cd4b42#L0L676">676</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/5f42bd915b68925aff9f367febe94c5e14cd4b42#L0L677">677</a></span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/5f42bd915b68925aff9f367febe94c5e14cd4b42#L0L678">678</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/5f42bd915b68925aff9f367febe94c5e14cd4b42#L0L679">679</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/5f42bd915b68925aff9f367febe94c5e14cd4b42#L0L680">680</a></span>
</pre>
            </th>
            <th class="diff-line-number" style="border: 1px solid #aaa">
<pre style="border: 0; font-family: Consolas, Menlo, &quot;Liberation Mono&quot;, Courier, monospace; line-height: 1.2; margin: 0; padding: 0.5em; white-space: normal; width: auto"><span class="diff-line-number-hunk-header" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/5f42bd915b68925aff9f367febe94c5e14cd4b42#L0R477">...</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/5f42bd915b68925aff9f367febe94c5e14cd4b42#L0R478">478</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/5f42bd915b68925aff9f367febe94c5e14cd4b42#L0R479">479</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/5f42bd915b68925aff9f367febe94c5e14cd4b42#L0R480">480</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/5f42bd915b68925aff9f367febe94c5e14cd4b42#L0R481">481</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/5f42bd915b68925aff9f367febe94c5e14cd4b42#L0R482">482</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/5f42bd915b68925aff9f367febe94c5e14cd4b42#L0R483">483</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/5f42bd915b68925aff9f367febe94c5e14cd4b42#L0R484">484</a></span>
<span class="diff-line-number-hunk-header" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/5f42bd915b68925aff9f367febe94c5e14cd4b42#L0R515">...</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/5f42bd915b68925aff9f367febe94c5e14cd4b42#L0R516">516</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/5f42bd915b68925aff9f367febe94c5e14cd4b42#L0R517">517</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/5f42bd915b68925aff9f367febe94c5e14cd4b42#L0R518">518</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/5f42bd915b68925aff9f367febe94c5e14cd4b42#L0R519">519</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/5f42bd915b68925aff9f367febe94c5e14cd4b42#L0R520">520</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/5f42bd915b68925aff9f367febe94c5e14cd4b42#L0R521">521</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/5f42bd915b68925aff9f367febe94c5e14cd4b42#L0R522">522</a></span>
<span class="diff-line-number-hunk-header" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/5f42bd915b68925aff9f367febe94c5e14cd4b42#L0R660">...</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/5f42bd915b68925aff9f367febe94c5e14cd4b42#L0R661">661</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/5f42bd915b68925aff9f367febe94c5e14cd4b42#L0R662">662</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/5f42bd915b68925aff9f367febe94c5e14cd4b42#L0R663">663</a></span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/5f42bd915b68925aff9f367febe94c5e14cd4b42#L0R664">664</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/5f42bd915b68925aff9f367febe94c5e14cd4b42#L0R665">665</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/5f42bd915b68925aff9f367febe94c5e14cd4b42#L0R666">666</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/5f42bd915b68925aff9f367febe94c5e14cd4b42#L0R667">667</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/5f42bd915b68925aff9f367febe94c5e14cd4b42#L0R668">668</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/5f42bd915b68925aff9f367febe94c5e14cd4b42#L0R669">669</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/5f42bd915b68925aff9f367febe94c5e14cd4b42#L0R670">670</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/5f42bd915b68925aff9f367febe94c5e14cd4b42#L0R671">671</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/5f42bd915b68925aff9f367febe94c5e14cd4b42#L0R672">672</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/5f42bd915b68925aff9f367febe94c5e14cd4b42#L0R673">673</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/5f42bd915b68925aff9f367febe94c5e14cd4b42#L0R674">674</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/5f42bd915b68925aff9f367febe94c5e14cd4b42#L0R675">675</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/5f42bd915b68925aff9f367febe94c5e14cd4b42#L0R676">676</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/5f42bd915b68925aff9f367febe94c5e14cd4b42#L0R677">677</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/5f42bd915b68925aff9f367febe94c5e14cd4b42#L0R678">678</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/5f42bd915b68925aff9f367febe94c5e14cd4b42#L0R679">679</a></span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/5f42bd915b68925aff9f367febe94c5e14cd4b42#L0R680">680</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/5f42bd915b68925aff9f367febe94c5e14cd4b42#L0R681">681</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/5f42bd915b68925aff9f367febe94c5e14cd4b42#L0R682">682</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/5f42bd915b68925aff9f367febe94c5e14cd4b42#L0R683">683</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/5f42bd915b68925aff9f367febe94c5e14cd4b42#L0R684">684</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/5f42bd915b68925aff9f367febe94c5e14cd4b42#L0R685">685</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/5f42bd915b68925aff9f367febe94c5e14cd4b42#L0R686">686</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/5f42bd915b68925aff9f367febe94c5e14cd4b42#L0R687">687</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/5f42bd915b68925aff9f367febe94c5e14cd4b42#L0R688">688</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/5f42bd915b68925aff9f367febe94c5e14cd4b42#L0R689">689</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/5f42bd915b68925aff9f367febe94c5e14cd4b42#L0R690">690</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/5f42bd915b68925aff9f367febe94c5e14cd4b42#L0R691">691</a></span>
<span class="diff-line-number-hunk-header" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/5f42bd915b68925aff9f367febe94c5e14cd4b42#L0R692">...</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/5f42bd915b68925aff9f367febe94c5e14cd4b42#L0R693">693</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/5f42bd915b68925aff9f367febe94c5e14cd4b42#L0R694">694</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/5f42bd915b68925aff9f367febe94c5e14cd4b42#L0R695">695</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/5f42bd915b68925aff9f367febe94c5e14cd4b42#L0R696">696</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/5f42bd915b68925aff9f367febe94c5e14cd4b42#L0R697">697</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/5f42bd915b68925aff9f367febe94c5e14cd4b42#L0R698">698</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/5f42bd915b68925aff9f367febe94c5e14cd4b42#L0R699">699</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga/commit/5f42bd915b68925aff9f367febe94c5e14cd4b42#L0R700">700</a></span>
</pre>
            </th>
            <td class="diff-content" style="border: 1px solid #aaa">
<pre style="border: 0; font-family: Consolas, Menlo, &quot;Liberation Mono&quot;, Courier, monospace; line-height: 1.2; margin: 0; padding: 0.5em; white-space: normal; width: auto"><span class="diff-hunk-header" style="background-color: #eaf2f5; color: #999999; display: block; white-space: pre">@@ -478,6 +478,7 @@ <span class="diff-context" style="background-color: #ffffaa; color: #000000">typedef struct {</span></span>
<span class="diff-not-changed" style="display: block; white-space: pre">   } get;</span>
<span class="diff-not-changed" style="display: block; white-space: pre">   grn_bool is_begin;</span>
<span class="diff-not-changed" style="display: block; white-space: pre">   grn_bool is_end;</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+  grn_bool is_first_token;</span>
<span class="diff-not-changed" style="display: block; white-space: pre">   grn_bool is_overlapping;</span>
<span class="diff-not-changed" style="display: block; white-space: pre">   const char *next;</span>
<span class="diff-not-changed" style="display: block; white-space: pre">   const char *end;</span>
<span class="diff-hunk-header" style="background-color: #eaf2f5; color: #999999; display: block; white-space: pre">@@ -515,6 +516,7 @@ <span class="diff-context" style="background-color: #ffffaa; color: #000000">regexp_init(grn_ctx *ctx, int nargs, grn_obj **args, grn_user_data *user_data)</span></span>
<span class="diff-not-changed" style="display: block; white-space: pre"> </span>
<span class="diff-not-changed" style="display: block; white-space: pre">   tokenizer-&gt;is_begin = GRN_TRUE;</span>
<span class="diff-not-changed" style="display: block; white-space: pre">   tokenizer-&gt;is_end   = GRN_FALSE;</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+  tokenizer-&gt;is_first_token = GRN_TRUE;</span>
<span class="diff-not-changed" style="display: block; white-space: pre">   tokenizer-&gt;is_overlapping = GRN_FALSE;</span>
<span class="diff-not-changed" style="display: block; white-space: pre"> </span>
<span class="diff-not-changed" style="display: block; white-space: pre">   grn_string_get_normalized(ctx, tokenizer-&gt;query-&gt;normalized_query,</span>
<span class="diff-hunk-header" style="background-color: #eaf2f5; color: #999999; display: block; white-space: pre">@@ -659,15 +661,31 @@ <span class="diff-context" style="background-color: #ffffaa; color: #000000">regexp_next(grn_ctx *ctx, int nargs, grn_obj **args, grn_user_data *user_data)</span></span>
<span class="diff-not-changed" style="display: block; white-space: pre">   }</span>
<span class="diff-not-changed" style="display: block; white-space: pre">   tokenizer-&gt;is_overlapping = (n_characters &gt; 1);</span>
<span class="diff-not-changed" style="display: block; white-space: pre"> </span>
<span class="diff-deleted" style="background-color: #ffaaaa; color: #000000; display: block; white-space: pre">-  if (tokenizer-&gt;next == end) {</span>
<span class="diff-deleted" style="background-color: #ffaaaa; color: #000000; display: block; white-space: pre">-    tokenizer-&gt;is_end = GRN_TRUE;</span>
<span class="diff-deleted" style="background-color: #ffaaaa; color: #000000; display: block; white-space: pre">-    if (mode == GRN_TOKEN_GET) {</span>
<span class="diff-deleted" style="background-color: #ffaaaa; color: #000000; display: block; white-space: pre">-      if (!tokenizer-&gt;get.have_end) {</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+  if (mode == GRN_TOKEN_GET) {</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+    if ((end - tokenizer-&gt;next) &lt; ngram_unit) {</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+      if (tokenizer-&gt;get.have_end) {</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+        if (tokenizer-&gt;next == end) {</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+          tokenizer-&gt;is_end = GRN_TRUE;</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+        }</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+        if (status &amp; GRN_TOKEN_UNMATURED) {</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+          if (tokenizer-&gt;is_first_token) {</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+            status |= GRN_TOKEN_FORCE_PREFIX;</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+          } else {</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+            status |= GRN_TOKEN_SKIP;</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+          }</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+        }</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+      } else {</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+        tokenizer-&gt;is_end = GRN_TRUE;</span>
<span class="diff-not-changed" style="display: block; white-space: pre">         status |= GRN_TOKEN_LAST | GRN_TOKEN_REACH_END;</span>
<span class="diff-deleted" style="background-color: #ffaaaa; color: #000000; display: block; white-space: pre">-      } else if (status &amp; GRN_TOKEN_UNMATURED) {</span>
<span class="diff-deleted" style="background-color: #ffaaaa; color: #000000; display: block; white-space: pre">-        status |= GRN_TOKEN_FORCE_PREFIX;</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+        if (status &amp; GRN_TOKEN_UNMATURED) {</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+          status |= GRN_TOKEN_FORCE_PREFIX;</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+        }</span>
<span class="diff-not-changed" style="display: block; white-space: pre">       }</span>
<span class="diff-not-changed" style="display: block; white-space: pre">     }</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+  } else {</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+    if (tokenizer-&gt;next == end) {</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+      tokenizer-&gt;is_end = GRN_TRUE;</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+    }</span>
<span class="diff-not-changed" style="display: block; white-space: pre">   }</span>
<span class="diff-not-changed" style="display: block; white-space: pre"> </span>
<span class="diff-not-changed" style="display: block; white-space: pre">   grn_tokenizer_token_push(ctx,</span>
<span class="diff-hunk-header" style="background-color: #eaf2f5; color: #999999; display: block; white-space: pre">@@ -675,6 +693,8 @@ <span class="diff-context" style="background-color: #ffffaa; color: #000000">regexp_next(grn_ctx *ctx, int nargs, grn_obj **args, grn_user_data *user_data)</span></span>
<span class="diff-not-changed" style="display: block; white-space: pre">                            GRN_TEXT_VALUE(buffer),</span>
<span class="diff-not-changed" style="display: block; white-space: pre">                            GRN_TEXT_LEN(buffer),</span>
<span class="diff-not-changed" style="display: block; white-space: pre">                            status);</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+  tokenizer-&gt;is_first_token = GRN_FALSE;</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+</span>
<span class="diff-not-changed" style="display: block; white-space: pre">   return NULL;</span>
<span class="diff-not-changed" style="display: block; white-space: pre"> }</span>
<span class="diff-not-changed" style="display: block; white-space: pre"> </span>
</pre>
            </td>
          </tr>
        </tbody>
      </table>
    </div>
  </body>
</html>