<!DOCTYPE html>
<html>
  <head>
  </head>
  <body>
    <dl style="line-height: 1.5; margin-left: 2em">
      <dt style="clear: both; float: left; font-weight: bold; width: 8em">Author</dt>
      <dd style="margin-left: 8.5em">Kouhei Sutou &lt;kou****@clear*****&gt;</dd>
      <dt style="clear: both; float: left; font-weight: bold; width: 8em">Date</dt>
      <dd style="margin-left: 8.5em">2015-01-26 15:51:40 +0900 (Mon, 26 Jan 2015)</dd>
      <dt style="clear: both; float: left; font-weight: bold; width: 8em">New Revision</dt>
      <dd style="margin-left: 8.5em"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1">9c03966d2fc23cb9785ee1d479ff42d41c7038c1</a></dd>
      <dt style="clear: both; float: left; font-weight: bold; width: 8em">Message</dt>
      <dd style="margin-left: 8.5em"><pre style="border: 1px solid #aaa; font-family: Consolas, Menlo, &quot;Liberation Mono&quot;, Courier, monospace; line-height: 1.2; padding: 0.5em; width: auto">tokenize: support mode and token_filters</pre></dd>
      <dt style="clear: both; float: left; font-weight: bold; width: 8em">Modified files</dt>
      <dd style="margin-left: 8.5em">
        <ul>
          <li><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#diff-0">lib/groonga/command/tokenize.rb</a></li>
          <li><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#diff-1">test/command/test-tokenize.rb</a></li>
        </ul>
      </dd>
    </dl>

    <div class="diff-section" style="clear: both">
      <table style="border-collapse: collapse; border: 1px solid #aaa">
        <thead>
          <tr class="diff-header" style="border: 1px solid #aaa">
            <td colspan="3">
<pre style="border: 0; font-family: Consolas, Menlo, &quot;Liberation Mono&quot;, Courier, monospace; line-height: 1.2; margin: 0; padding: 0.5em; white-space: normal; width: auto"><span class="diff-header" style="background-color: #eaf2f5; color: #999999; display: block; white-space: pre">  Modified: lib/groonga/command/tokenize.rb (+20 -1)</span>
<span class="diff-header-mark" style="background-color: #eaf2f5; color: #999999; display: block; white-space: pre">===================================================================</span>
</pre>
            </td>
          </tr>
        </thead>
        <tbody>
          <tr>
            <th class="diff-line-number" style="border: 1px solid #aaa">
<pre style="border: 0; font-family: Consolas, Menlo, &quot;Liberation Mono&quot;, Courier, monospace; line-height: 1.2; margin: 0; padding: 0.5em; white-space: normal; width: auto"><span class="diff-line-number-hunk-header" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L0L32">...</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L0L33">33</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L0L34">34</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L0L35">35</a></span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L0L36">36</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L0L37">37</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L0L38">38</a></span>
<span class="diff-line-number-hunk-header" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L0L54">...</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L0L55">55</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L0L56">56</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L0L57">57</a></span>
<span class="diff-line-number-deleted" style="background-color: #ffaaaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L0L58">58</a></span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L0L59">59</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L0L60">60</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L0L61">61</a></span>
<span class="diff-line-number-hunk-header" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L0L63">...</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L0L64">64</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L0L65">65</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L0L66">66</a></span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L0L67">67</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L0L68">68</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L0L69">69</a></span>
</pre>
            </th>
            <th class="diff-line-number" style="border: 1px solid #aaa">
<pre style="border: 0; font-family: Consolas, Menlo, &quot;Liberation Mono&quot;, Courier, monospace; line-height: 1.2; margin: 0; padding: 0.5em; white-space: normal; width: auto"><span class="diff-line-number-hunk-header" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L0R32">...</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L0R33">33</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L0R34">34</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L0R35">35</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L0R36">36</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L0R37">37</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L0R38">38</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L0R39">39</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L0R40">40</a></span>
<span class="diff-line-number-hunk-header" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L0R56">...</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L0R57">57</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L0R58">58</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L0R59">59</a></span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L0R60">60</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L0R61">61</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L0R62">62</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L0R63">63</a></span>
<span class="diff-line-number-hunk-header" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L0R65">...</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L0R66">66</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L0R67">67</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L0R68">68</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L0R69">69</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L0R70">70</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L0R71">71</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L0R72">72</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L0R73">73</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L0R74">74</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L0R75">75</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L0R76">76</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L0R77">77</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L0R78">78</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L0R79">79</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L0R80">80</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L0R81">81</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L0R82">82</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L0R83">83</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L0R84">84</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L0R85">85</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L0R86">86</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L0R87">87</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L0R88">88</a></span>
</pre>
            </th>
            <td class="diff-content" style="border: 1px solid #aaa">
<pre style="border: 0; font-family: Consolas, Menlo, &quot;Liberation Mono&quot;, Courier, monospace; line-height: 1.2; margin: 0; padding: 0.5em; white-space: normal; width: auto"><span class="diff-hunk-header" style="background-color: #eaf2f5; color: #999999; display: block; white-space: pre">@@ -33,6 +33,8 @@ <span class="diff-context" style="background-color: #ffffaa; color: #000000">module Groonga</span></span>
<span class="diff-not-changed" style="display: block; white-space: pre">             :string,</span>
<span class="diff-not-changed" style="display: block; white-space: pre">             :normalizer,</span>
<span class="diff-not-changed" style="display: block; white-space: pre">             :flags,</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+            :mode,</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+            :token_filters,</span>
<span class="diff-not-changed" style="display: block; white-space: pre">           ]</span>
<span class="diff-not-changed" style="display: block; white-space: pre">         end</span>
<span class="diff-not-changed" style="display: block; white-space: pre">       end</span>
<span class="diff-hunk-header" style="background-color: #eaf2f5; color: #999999; display: block; white-space: pre">@@ -55,7 +57,7 @@ <span class="diff-context" style="background-color: #ffffaa; color: #000000">module Groonga</span></span>
<span class="diff-not-changed" style="display: block; white-space: pre">         self[:normalizer]</span>
<span class="diff-not-changed" style="display: block; white-space: pre">       end</span>
<span class="diff-not-changed" style="display: block; white-space: pre"> </span>
<span class="diff-deleted" style="background-color: #ffaaaa; color: #000000; display: block; white-space: pre">-      # @return [Array&lt;String&gt;] An array of flags specified in `flags`</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+      # @return [Array&lt;String&gt;] An array of flag specified in `flags`</span>
<span class="diff-not-changed" style="display: block; white-space: pre">       #   parameter value. This array is extracted by parsing `flags`</span>
<span class="diff-not-changed" style="display: block; white-space: pre">       #   parameter value. If `flags` parameter value is nil or empty,</span>
<span class="diff-not-changed" style="display: block; white-space: pre">       #   an empty array is returned.</span>
<span class="diff-hunk-header" style="background-color: #eaf2f5; color: #999999; display: block; white-space: pre">@@ -64,6 +66,23 @@ <span class="diff-context" style="background-color: #ffffaa; color: #000000">module Groonga</span></span>
<span class="diff-not-changed" style="display: block; white-space: pre">       def flags</span>
<span class="diff-not-changed" style="display: block; white-space: pre">         @flags ||= (self[:flags] || &quot;&quot;).split(/\s*[| ]\s*/)</span>
<span class="diff-not-changed" style="display: block; white-space: pre">       end</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+      # @return [String] `mode` parameter value.</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+      #</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+      # @since 1.1.0</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+      def mode</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+        self[:mode]</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+      end</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+      # @return [Array&lt;String&gt;] An array of token filter specified in</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+      #   `token_filters` parameter value. This array is extracted by</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+      #   parsing `token_filters` parameter value. If `token_filters`</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+      #   parameter value is nil or empty, an empty array is returned.</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+      #</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+      # @since 1.1.0</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+      def token_filters</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+        @token_filters ||= (self[:token_filters] || &quot;&quot;).split(/\s*,\s*/)</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+      end</span>
<span class="diff-not-changed" style="display: block; white-space: pre">     end</span>
<span class="diff-not-changed" style="display: block; white-space: pre">   end</span>
<span class="diff-not-changed" style="display: block; white-space: pre"> end</span>
</pre>
            </td>
          </tr>
        </tbody>
      </table>
      <table style="border-collapse: collapse; border: 1px solid #aaa">
        <thead>
          <tr class="diff-header" style="border: 1px solid #aaa">
            <td colspan="3">
<pre style="border: 0; font-family: Consolas, Menlo, &quot;Liberation Mono&quot;, Courier, monospace; line-height: 1.2; margin: 0; padding: 0.5em; white-space: normal; width: auto"><span class="diff-header" style="background-color: #eaf2f5; color: #999999; display: block; white-space: pre">  Modified: test/command/test-tokenize.rb (+40 -1)</span>
<span class="diff-header-mark" style="background-color: #eaf2f5; color: #999999; display: block; white-space: pre">===================================================================</span>
</pre>
            </td>
          </tr>
        </thead>
        <tbody>
          <tr>
            <th class="diff-line-number" style="border: 1px solid #aaa">
<pre style="border: 0; font-family: Consolas, Menlo, &quot;Liberation Mono&quot;, Courier, monospace; line-height: 1.2; margin: 0; padding: 0.5em; white-space: normal; width: auto"><span class="diff-line-number-hunk-header" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L1L0">...</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L1L1">1</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L1L2">2</a></span>
<span class="diff-line-number-deleted" style="background-color: #ffaaaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L1L3">3</a></span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L1L4">4</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L1L5">5</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L1L6">6</a></span>
<span class="diff-line-number-hunk-header" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L1L29">...</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L1L30">30</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L1L31">31</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L1L32">32</a></span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L1L33">33</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L1L34">34</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L1L35">35</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L1L36">36</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L1L37">37</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L1L38">38</a></span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L1L39">39</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L1L40">40</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L1L41">41</a></span>
<span class="diff-line-number-hunk-header" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L1L42">...</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L1L43">43</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L1L44">44</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L1L45">45</a></span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L1L46">46</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L1L47">47</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L1L48">48</a></span>
<span class="diff-line-number-hunk-header" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L1L94">...</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L1L95">95</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L1L96">96</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L1L97">97</a></span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L1L98">98</a></span>
</pre>
            </th>
            <th class="diff-line-number" style="border: 1px solid #aaa">
<pre style="border: 0; font-family: Consolas, Menlo, &quot;Liberation Mono&quot;, Courier, monospace; line-height: 1.2; margin: 0; padding: 0.5em; white-space: normal; width: auto"><span class="diff-line-number-hunk-header" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L1R0">...</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L1R1">1</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L1R2">2</a></span>
<span class="diff-line-number-nothing" style="display: block; white-space: pre">&nbsp;</span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L1R3">3</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L1R4">4</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L1R5">5</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L1R6">6</a></span>
<span class="diff-line-number-hunk-header" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L1R29">...</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L1R30">30</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L1R31">31</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L1R32">32</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L1R33">33</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L1R34">34</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L1R35">35</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L1R36">36</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L1R37">37</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L1R38">38</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L1R39">39</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L1R40">40</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L1R41">41</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L1R42">42</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L1R43">43</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L1R44">44</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L1R45">45</a></span>
<span class="diff-line-number-hunk-header" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L1R46">...</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L1R47">47</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L1R48">48</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L1R49">49</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L1R50">50</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L1R51">51</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L1R52">52</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L1R53">53</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L1R54">54</a></span>
<span class="diff-line-number-hunk-header" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L1R100">...</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L1R101">101</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L1R102">102</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L1R103">103</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L1R104">104</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L1R105">105</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L1R106">106</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L1R107">107</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L1R108">108</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L1R109">109</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L1R110">110</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L1R111">111</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L1R112">112</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L1R113">113</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L1R114">114</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L1R115">115</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L1R116">116</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L1R117">117</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L1R118">118</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L1R119">119</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L1R120">120</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L1R121">121</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L1R122">122</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L1R123">123</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L1R124">124</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L1R125">125</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L1R126">126</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L1R127">127</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L1R128">128</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L1R129">129</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L1R130">130</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L1R131">131</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L1R132">132</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L1R133">133</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L1R134">134</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L1R135">135</a></span>
<span class="diff-line-number-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L1R136">136</a></span>
<span class="diff-line-number-not-changed" style="display: block; white-space: pre"><a href="https://github.com/groonga/groonga-command/commit/9c03966d2fc23cb9785ee1d479ff42d41c7038c1#L1R137">137</a></span>
</pre>
            </th>
            <td class="diff-content" style="border: 1px solid #aaa">
<pre style="border: 0; font-family: Consolas, Menlo, &quot;Liberation Mono&quot;, Courier, monospace; line-height: 1.2; margin: 0; padding: 0.5em; white-space: normal; width: auto"><span class="diff-hunk-header" style="background-color: #eaf2f5; color: #999999; display: block; white-space: pre">@@ -1,6 +1,6 @@</span>
<span class="diff-not-changed" style="display: block; white-space: pre"> # -*- coding: utf-8 -*-</span>
<span class="diff-not-changed" style="display: block; white-space: pre"> #</span>
<span class="diff-deleted" style="background-color: #ffaaaa; color: #000000; display: block; white-space: pre">-# Copyright (C) 2013  Kouhei Sutou &lt;kou****@clear*****&gt;</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+# Copyright (C) 2013-2015  Kouhei Sutou &lt;kou****@clear*****&gt;</span>
<span class="diff-not-changed" style="display: block; white-space: pre"> #</span>
<span class="diff-not-changed" style="display: block; white-space: pre"> # This library is free software; you can redistribute it and/or</span>
<span class="diff-not-changed" style="display: block; white-space: pre"> # modify it under the terms of the GNU Lesser General Public</span>
<span class="diff-hunk-header" style="background-color: #eaf2f5; color: #999999; display: block; white-space: pre">@@ -30,12 +30,16 @@ <span class="diff-context" style="background-color: #ffffaa; color: #000000">class TokenizeCommandTest &lt; Test::Unit::TestCase</span></span>
<span class="diff-not-changed" style="display: block; white-space: pre">       string     = &quot;groonga ruby linux&quot;</span>
<span class="diff-not-changed" style="display: block; white-space: pre">       normalizer = &quot;NormalizerAuto&quot;</span>
<span class="diff-not-changed" style="display: block; white-space: pre">       flags      = &quot;NONE&quot;</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+      mode       = &quot;ADD&quot;</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+      token_filters = &quot;TokenFilterStem&quot;</span>
<span class="diff-not-changed" style="display: block; white-space: pre"> </span>
<span class="diff-not-changed" style="display: block; white-space: pre">       ordered_arguments = [</span>
<span class="diff-not-changed" style="display: block; white-space: pre">         tokenizer,</span>
<span class="diff-not-changed" style="display: block; white-space: pre">         string,</span>
<span class="diff-not-changed" style="display: block; white-space: pre">         normalizer,</span>
<span class="diff-not-changed" style="display: block; white-space: pre">         flags,</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+        mode,</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+        token_filters</span>
<span class="diff-not-changed" style="display: block; white-space: pre">       ]</span>
<span class="diff-not-changed" style="display: block; white-space: pre">       command = tokenize_command({}, ordered_arguments)</span>
<span class="diff-not-changed" style="display: block; white-space: pre">       assert_equal({</span>
<span class="diff-hunk-header" style="background-color: #eaf2f5; color: #999999; display: block; white-space: pre">@@ -43,6 +47,8 @@ <span class="diff-context" style="background-color: #ffffaa; color: #000000">class TokenizeCommandTest &lt; Test::Unit::TestCase</span></span>
<span class="diff-not-changed" style="display: block; white-space: pre">                      :string     =&gt; string,</span>
<span class="diff-not-changed" style="display: block; white-space: pre">                      :normalizer =&gt; normalizer,</span>
<span class="diff-not-changed" style="display: block; white-space: pre">                      :flags      =&gt; flags,</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+                     :mode       =&gt; mode,</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+                     :token_filters =&gt; token_filters,</span>
<span class="diff-not-changed" style="display: block; white-space: pre">                    },</span>
<span class="diff-not-changed" style="display: block; white-space: pre">                    command.arguments)</span>
<span class="diff-not-changed" style="display: block; white-space: pre">     end</span>
<span class="diff-hunk-header" style="background-color: #eaf2f5; color: #999999; display: block; white-space: pre">@@ -95,4 +101,37 @@ <span class="diff-context" style="background-color: #ffffaa; color: #000000">class TokenizeCommandTest &lt; Test::Unit::TestCase</span></span>
<span class="diff-not-changed" style="display: block; white-space: pre">       assert_equal([&quot;NONE&quot;, &quot;ENABLE_TOKENIZED_DELIMITER&quot;], command.flags)</span>
<span class="diff-not-changed" style="display: block; white-space: pre">     end</span>
<span class="diff-not-changed" style="display: block; white-space: pre">   end</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+  class ModeTest &lt; self</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+    def test_reader</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+      command = tokenize_command(:mode =&gt; &quot;ADD&quot;)</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+      assert_equal(&quot;ADD&quot;, command.mode)</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+    end</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+  end</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+  class TokenFiltersTest &lt; self</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+    def test_nil</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+      command = tokenize_command</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+      assert_equal([], command.token_filters)</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+    end</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+    def test_empty</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+      command = tokenize_command(:token_filters =&gt; &quot;&quot;)</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+      assert_equal([], command.token_filters)</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+    end</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+    def test_comma_separator</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+      token_filters = &quot;TokenFilterStem,TokenFilterStopWord&quot;</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+      command = tokenize_command(:token_filters =&gt; token_filters)</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+      assert_equal([&quot;TokenFilterStem&quot;, &quot;TokenFilterStopWord&quot;],</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+                   command.token_filters)</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+    end</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+    def test_spaces_around_separator</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+      token_filters = &quot;TokenFilterStem , TokenFilterStopWord&quot;</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+      command = tokenize_command(:token_filters =&gt; token_filters)</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+      assert_equal([&quot;TokenFilterStem&quot;, &quot;TokenFilterStopWord&quot;],</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+                   command.token_filters)</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+    end</span>
<span class="diff-added" style="background-color: #aaffaa; color: #000000; display: block; white-space: pre">+  end</span>
<span class="diff-not-changed" style="display: block; white-space: pre"> end</span>
</pre>
            </td>
          </tr>
        </tbody>
      </table>
    </div>
  </body>
</html>